From f83e02b79a8396784a11daa28402d55897aa65a0 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:20 -0400 Subject: [PATCH 01/38] replace capi --- .../artifacts/Bootstrap.ps1 | 10 +- .../artifacts/DevOpsLogonScript.ps1 | 76 +- .../gitops_scripts/ResetBookstore.ps1 | 4 +- .../artifacts/installK3s.sh | 216 +- .../artifacts/longhorn.yaml | 4571 +++++++++++++++++ .../bicep/clientVm/clientVm.bicep | 7 +- .../bicep/kubernetes/aks.bicep | 2 +- .../bicep/kubernetes/ubuntuRancher.bicep | 107 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 150 + azure_jumpstart_arcbox/bicep/main.bicep | 55 +- 10 files changed, 5040 insertions(+), 158 deletions(-) create mode 100644 azure_jumpstart_arcbox/artifacts/longhorn.yaml create mode 100644 azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep diff --git a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 index f4466a3811..90266d939a 100644 --- a/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 +++ b/azure_jumpstart_arcbox/artifacts/Bootstrap.ps1 @@ -21,7 +21,7 @@ param ( [string]$POSTGRES_SERVICE_TYPE, [string]$stagingStorageAccountName, [string]$workspaceName, - [string]$capiArcDataClusterName, + [string]$k3sArcDataClusterName, [string]$k3sArcClusterName, [string]$aksArcClusterName, [string]$aksdrArcClusterName, @@ -31,7 +31,8 @@ param ( [string]$rdpPort, [string]$sshPort, [string]$vmAutologon, - [string]$addsDomainName + [string]$addsDomainName, + [string]$customLocationRPOID ) [System.Environment]::SetEnvironmentVariable('adminUsername', $adminUsername, [System.EnvironmentVariableTarget]::Machine) @@ -51,7 +52,7 @@ param ( [System.Environment]::SetEnvironmentVariable('POSTGRES_SERVICE_TYPE', $POSTGRES_SERVICE_TYPE, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('stagingStorageAccountName', $stagingStorageAccountName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('workspaceName', $workspaceName, [System.EnvironmentVariableTarget]::Machine) -[System.Environment]::SetEnvironmentVariable('capiArcDataClusterName', $capiArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('k3sArcDataClusterName', $k3sArcDataClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('k3sArcClusterName', $k3sArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('githubUser', $githubUser, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('templateBaseUrl', $templateBaseUrl, [System.EnvironmentVariableTarget]::Machine) @@ -60,6 +61,7 @@ param ( [System.Environment]::SetEnvironmentVariable('addsDomainName', $addsDomainName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksArcClusterName', $aksArcClusterName, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('aksdrArcClusterName', $aksdrArcClusterName, [System.EnvironmentVariableTarget]::Machine) +[System.Environment]::SetEnvironmentVariable('customLocationRPOID', $customLocationRPOID, [System.EnvironmentVariableTarget]::Machine) [System.Environment]::SetEnvironmentVariable('ArcBoxDir', "C:\ArcBox", [System.EnvironmentVariableTarget]::Machine) @@ -242,6 +244,7 @@ if ($flavor -eq "DevOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/icons/bookstore.ico") -OutFile $Env:ArcBoxIconDir\bookstore.ico Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/devops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\devops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/devops.dsc.yml") -OutFile $Env:ArcBoxDscDir\devops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # DataOps @@ -269,6 +272,7 @@ if ($flavor -eq "DataOps") { Invoke-WebRequest ($templateBaseUrl + "artifacts/testDefenderForSQL.ps1") -OutFile $Env:ArcBoxDir\testDefenderForSQL.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/tests/dataops.tests.ps1") -OutFile $Env:ArcBoxTestsDir\dataops.tests.ps1 Invoke-WebRequest ($templateBaseUrl + "artifacts/dsc/dataops.dsc.yml") -OutFile $Env:ArcBoxDscDir\dataops.dsc.yml + Invoke-WebRequest ($templateBaseUrl + "artifacts/longhorn.yaml") -OutFile $Env:ArcBoxDir\longhorn.yaml } # Full diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b60a8d6f8b..406795706e 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -29,44 +29,49 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:capiArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "CAPI" | Where-Object { $_ -ne "" } -$Env:capiArcDataClusterName=$Env:capiArcDataClusterName -replace "`n","" +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" + +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" az login --identity --tenant $spnTenantId az account set -s $env:subscriptionId -# Downloading CAPI Kubernetes cluster kubeconfig file -Write-Header "Downloading CAPI K8s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/config" +# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file +Write-Header "Downloading ArcBox-DataSvc-K3s K8s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading Rancher K3s cluster kubeconfig file -Write-Header "Downloading K3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" -$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +# Downloading ArcBox-DataSvc-K3s log file +Write-Header "Downloading ArcBox-DataSvc-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Downloading 'installCAPI.log' log file -Write-Header "Downloading CAPI Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-capi/installCAPI.log" +# Downloading ArcBox-K3s cluster kubeconfig file +Write-Header "Downloading ArcBox-K3s Kubeconfig" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/config" +$context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installCAPI.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" +$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +kubectx -# Downloading 'installK3s.log' log file -Write-Header "Downloading K3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s.log" +# Downloading ArcBox-K3s log file +Write-Header "Downloading ArcBox-K3s Install Logs" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installK3s.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from CAPI and Rancher K3s -Write-Header "Merging CAPI & K3s Kubeconfigs" +# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -92,13 +97,18 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v +# Longhorn setup for RWX-capable storage class +Write-Header "Creating longhorn storage" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +Start-Sleep -Seconds 30 + # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --version $osmReleaseVersion ` @@ -128,7 +138,7 @@ Write-Header "Applying GitOps Configs" # Create GitOps config for NGINX Ingress Controller Write-Host "Creating GitOps config for NGINX Ingress Controller" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-nginx ` --namespace $ingressNamespace ` @@ -141,7 +151,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore application Write-Host "Creating GitOps config for Bookstore application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore ` --cluster-type connectedClusters ` @@ -152,7 +162,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore RBAC Write-Host "Creating GitOps config for Bookstore RBAC" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-rbac ` --cluster-type connectedClusters ` @@ -165,7 +175,7 @@ az k8s-configuration flux create ` # Create GitOps config for Bookstore Traffic Split Write-Host "Creating GitOps config for Bookstore Traffic Split" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-bookstore-osm ` --cluster-type connectedClusters ` @@ -178,7 +188,7 @@ az k8s-configuration flux create ` # Create GitOps config for Hello-Arc application Write-Host "Creating GitOps config for Hello-Arc application" az k8s-configuration flux create ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --name config-helloarc ` --namespace hello-arc ` @@ -212,7 +222,7 @@ az k8s-extension create ` --name 'akvsecretsprovider' ` --extension-type Microsoft.AzureKeyVaultSecretsProvider ` --scope cluster ` - --cluster-name $Env:capiArcDataClusterName ` + --cluster-name $Env:k3sArcDataClusterName ` --resource-group $Env:resourceGroup ` --cluster-type connectedClusters ` --release-namespace kube-system ` @@ -270,8 +280,8 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" -# Creating CAPI Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Hello-Arc.lnk" +# Creating K3s Hello Arc Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "https://$certdns" @@ -279,8 +289,8 @@ $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() -# Creating CAPI Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\CAPI Bookstore.lnk" +# Creating K3s Bookstore Icon on Desktop +$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index a304f2a278..68facfccaf 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,8 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-capi -kubectx arcbox-capi +# Switch kubectl context to arcbox-k3s +kubectx arcbox-datasvc-k3s ############################ # - Deploy Ingress for Reset diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 2ef7db3bcb..04df2fa835 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -14,24 +14,30 @@ echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $deployBastion:$9 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh + sed -i '2s/^/export adminUsername=/' vars.sh sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export vmName=/' vars.sh -sed -i '7s/^/export location=/' vars.sh -sed -i '8s/^/export stagingStorageAccountName=/' vars.sh -sed -i '9s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '10s/^/export deployBastion=/' vars.sh +sed -i '6s/^/export subscriptionId=/' vars.sh +sed -i '7s/^/export vmName=/' vars.sh +sed -i '8s/^/export location=/' vars.sh +sed -i '9s/^/export stagingStorageAccountName=/' vars.sh +sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh sed -i '11s/^/export templateBaseUrl=/' vars.sh +sed -i '12s/^/export storageContainerName=/' vars.sh +sed -i '13s/^/export k3sControlPlane=/' vars.sh +# Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! chmod +x vars.sh @@ -44,79 +50,139 @@ sudo curl -v -o /etc/profile.d/welcomeK3s.sh ${templateBaseUrl}artifacts/welcome sudo -u $adminUsername mkdir -p /home/${adminUsername}/jumpstart_logs while sleep 1; do sudo -s rsync -a /var/lib/waagent/custom-script/download/0/installK3s.log /home/${adminUsername}/jumpstart_logs/installK3s.log; done & -# Installing Rancher K3s cluster (single control plane) -echo "" -publicIp=$(hostname -i) -sudo mkdir ~/.kube -sudo -u $adminUsername mkdir /home/${adminUsername}/.kube -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --node-external-ip ${publicIp} --bind-address ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} sh - -sudo chmod 644 /etc/rancher/k3s/k3s.yaml -sudo kubectl config rename-context default arcbox-k3s --kubeconfig /etc/rancher/k3s/k3s.yaml -sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config -sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging -sudo chown -R $adminUsername /home/${adminUsername}/.kube/ -sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging - -# Installing Helm 3 -sudo snap install helm --classic - -echo "" -echo "Making sure Rancher K3s cluster is ready..." -echo "" -sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null -sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' -echo "" - # Installing Azure CLI & Azure Arc extensions curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -sudo -u $adminUsername az extension add --name connectedk8s -sudo -u $adminUsername az extension add --name k8s-configuration -sudo -u $adminUsername az extension add --name k8s-extension - echo "" echo "Log in to Azure" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password $SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -subscriptionId=$(sudo -u $adminUsername az account show --query id --output tsv) -sudo -u $adminUsername az account set -s $subscriptionId -az -v echo "" +sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az account set --subscription $subscriptionId +az -v -# Registering Azure resource providers -sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait -sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait - -sudo service sshd restart - -# Onboard the cluster to Azure Arc -resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) -workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) -sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --only-show-errors - -# Enabling Container Insights and Microsoft Defender for Containers cluster extensions -echo "" -sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors -echo "" -sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors +if [[ "$k3sControlPlane" == "true" ]]; then + + # Installing Azure Arc extensions + echo "" + echo "Installing Azure Arc extensions" + echo "" + sudo -u $adminUsername az extension add --name connectedk8s + sudo -u $adminUsername az extension add --name k8s-configuration + sudo -u $adminUsername az extension add --name k8s-extension + + # Installing Rancher K3s cluster (single control plane) + echo "" + echo "Installing Rancher K3s cluster" + echo "" + publicIp=$(hostname -i) + sudo mkdir ~/.kube + sudo -u $adminUsername mkdir /home/${adminUsername}/.kube + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --node-ip ${publicIp} --node-external-ip ${publicIp} --bind-address ${publicIp} --tls-san ${publicIp}" INSTALL_K3S_VERSION=v${K3S_VERSION} K3S_KUBECONFIG_MODE="644" sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: K3s installation failed" + exit 1 + fi + # Renaming default context to k3s cluster name + context=$(echo $storageContainerName | sed 's/-[^-]*$//') + sudo kubectl config rename-context default $context --kubeconfig /etc/rancher/k3s/k3s.yaml + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config + sudo cp /etc/rancher/k3s/k3s.yaml /home/${adminUsername}/.kube/config.staging + sudo chown -R $adminUsername /home/${adminUsername}/.kube/ + sudo chown -R staginguser /home/${adminUsername}/.kube/config.staging + + # Installing Helm 3 + echo "" + echo "Installing Helm" + echo "" + sudo snap install helm --classic + if [[ $? -ne 0 ]]; then + echo "ERROR: Helm installation failed" + exit 1 + fi + + echo "" + echo "Making sure Rancher K3s cluster is ready..." + echo "" + sudo kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null + sudo kubectl get nodes -o wide | expand | awk 'length($0) > length(longest) { longest = $0 } { lines[NR] = $0 } END { gsub(/./, "=", longest); print "/=" longest "=\\"; n = length(longest); for(i = 1; i <= NR; ++i) { printf("| %s %*s\n", lines[i], n - length(lines[i]) + 1, "|"); } print "\\=" longest "=/" }' + + # Copying Rancher K3s kubeconfig file to staging storage account + echo "" + echo "Copying Rancher K3s kubeconfig file to staging storage account" + echo "" + localPath="/home/$adminUsername/.kube/config" + k3sClusterNodeConfig="/home/$adminUsername/k3sClusterNodeConfig.yaml" + echo "k3sNodeToken: $(sudo cat /var/lib/rancher/k3s/server/node-token)" >> $k3sClusterNodeConfig + echo "k3sClusterIp: $publicIp" >> $k3sClusterNodeConfig + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath + sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig + + # Registering Azure resource providers + echo "" + echo "Registering Azure resource providers" + echo "" + sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + + # sudo service sshd restart + # Onboard the cluster to Azure Arc + echo "" + echo "Onboarding the cluster to Azure Arc" + echo "" + resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) + workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + + # Enabling Container Insights and Microsoft Defender for Containers cluster extensions + echo "" + echo "Enabling Container Insights and Microsoft Defender for Containers cluster extensions" + echo "" + sudo -u $adminUsername az k8s-extension create -n "azuremonitor-containers" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureMonitor.Containers --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + sudo -u $adminUsername az k8s-extension create -n "azure-defender" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.AzureDefender.Kubernetes --configuration-settings logAnalyticsWorkspaceResourceID=$workspaceResourceId --only-show-errors + + # Enabling Azure Policy for Kubernetes on the cluster + echo "" + echo "Enabling Azure Policy for Kubernetes on the cluster" + echo "" + sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors + +else + # Downloading k3s control plane details + echo "" + echo "Downloading k3s control plane details" + echo "" + k3sClusterNodeConfig="k3sClusterNodeConfig.yaml" + sudo -u $adminUsername az extension add --upgrade -n storage-preview + storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') + storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') + sudo -u $adminUsername az storage azcopy blob download --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source "$k3sClusterNodeConfig" --destination "/home/$adminUsername/$k3sClusterNodeConfig" + + # Installing Rancher K3s cluster (single worker node) + echo "" + echo "Installing Rancher K3s cluster node" + echo "" + k3sNodeToken=$(grep 'k3sNodeToken' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + k3sClusterIp=$(grep 'k3sClusterIp' "/home/$adminUsername/$k3sClusterNodeConfig" | awk '{print $2}') + curl -sfL https://get.k3s.io | K3S_URL=https://${k3sClusterIp}:6443 K3S_TOKEN=${k3sNodeToken} sh - + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to add k3s worker nodes" + exit 1 + fi + + sudo service sshd restart +fi -# Enabling Azure Policy for Kubernetes on the cluster +# Uploading this script log to staging storage for ease of troubleshooting echo "" -sudo -u $adminUsername az k8s-extension create --name "arc-azurepolicy" --cluster-name $vmName --resource-group $resourceGroup --cluster-type connectedClusters --extension-type Microsoft.PolicyInsights --only-show-errors - -# Copying Rancher K3s kubeconfig file to staging storage account +echo "Uploading the script logs to staging storage" echo "" -sudo -u $adminUsername az extension add --upgrade -n storage-preview -storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') -storageContainerName="staging-k3s" -localPath="/home/$adminUsername/.kube/config" -storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') -sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath - -# Uploading this script log to staging storage for ease of troubleshooting log="/home/${adminUsername}/jumpstart_logs/installK3s.log" -sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log +sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $log --destination "installK3s-$vmName.log" \ No newline at end of file diff --git a/azure_jumpstart_arcbox/artifacts/longhorn.yaml b/azure_jumpstart_arcbox/artifacts/longhorn.yaml new file mode 100644 index 0000000000..b03ab89440 --- /dev/null +++ b/azure_jumpstart_arcbox/artifacts/longhorn.yaml @@ -0,0 +1,4571 @@ +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + default-setting.yaml: |- + priority-class: longhorn-critical +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup + properties: + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. Required + type: boolean + required: + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + size: + description: The backing image size. + format: int64 + type: integer + state: + description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instance manager + properties: + dataEngine: + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instanceEngines: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + instances: + additionalProperties: + properties: + spec: + properties: + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + conditions: + additionalProperties: + type: boolean + type: object + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas' + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + description: Allow scheduling replicas on the node. + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + description: The status of the disks on the node. + nullable: true + type: object + region: + description: The Region of the node. + type: string + snapshotCheckStatus: + description: The status of the snapshot integrity check. + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + description: The Zone of the node. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim" + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + description: 'Deprecated: Replaced by field `spec.evictionRequested`.' + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + description: The value of the setting. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + description: Share manager image used for creating a share manager pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + description: NFS endpoint that can access the mounted filesystem of the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + properties: + volumeBackupPolicy: + description: The create volume backup policy Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: longhorn-system + path: /v1/webhook/conversion + port: 9501 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backendStoreDriver: + description: 'Deprecated: Replaced by field `dataEngine`.' + type: string + backingImage: + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + description: 'Deprecated: Replaced by field `image`.' + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineReplicaRebuilding: + description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not + enum: + - ignored + - disabled + - enabled + type: string + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + offlineReplicaRebuildingRequired: + type: boolean + ownerID: + type: string + pendingNodeID: + description: Deprecated. + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: LoadBalancer + selector: + app: longhorn-ui + ports: + - name: http + port: 8888 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: conversion-webhook + port: 9501 + targetPort: conversion-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-engine-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + name: longhorn-replica-manager + namespace: longhorn-system +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "longhornio/longhorn-engine:v1.6.0" + - --instance-manager-image + - "longhornio/longhorn-instance-manager:v1.6.0" + - --share-manager-image + - "longhornio/longhorn-share-manager:v1.6.0" + - --backing-image-manager-image + - "longhornio/backing-image-manager:v1.6.0" + - --support-bundle-manager-image + - "longhornio/support-bundle-kit:v0.0.33" + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9501 + name: conversion-wh + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9501 + scheme: HTTPS + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: longhornio/longhorn-manager:v1.6.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: longhornio/longhorn-manager:v1.6.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "longhornio/longhorn-manager:v1.6.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "longhornio/csi-attacher:v4.4.2" + - name: CSI_PROVISIONER_IMAGE + value: "longhornio/csi-provisioner:v3.6.2" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "longhornio/csi-node-driver-registrar:v2.9.2" + - name: CSI_RESIZER_IMAGE + value: "longhornio/csi-resizer:v1.9.2" + - name: CSI_SNAPSHOTTER_IMAGE + value: "longhornio/csi-snapshotter:v6.3.2" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "longhornio/livenessprobe:v2.11.0" + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.6.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: longhornio/longhorn-ui:v1.6.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# \ No newline at end of file diff --git a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep index 51a643c2ca..c199296e18 100644 --- a/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep +++ b/azure_jumpstart_arcbox/bicep/clientVm/clientVm.bicep @@ -2,7 +2,7 @@ param vmName string = 'ArcBox-Client' @description('The name of the Cluster API workload cluster to be connected as an Azure Arc-enabled Kubernetes cluster') -param capiArcDataClusterName string = 'ArcBox-CAPI-Data' +param k3sArcDataClusterName string = 'ArcBox-K3s-Data' @description('Username for the Virtual Machine') param windowsAdminUsername string = 'arcdemo' @@ -102,6 +102,9 @@ param aksdrArcClusterName string = 'ArcBox-AKS-DR-Data' @description('Domain name for the jumpstart environment') param addsDomainName string = 'jumpstart.local' +@description('The custom location RPO ID') +param customLocationRPOID string + var bastionName = 'ArcBox-Bastion' var publicIpAddressName = deployBastion == false ? '${vmName}-PIP' : '${bastionName}-PIP' @@ -206,7 +209,7 @@ resource vmBootstrap 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = fileUris: [ uri(templateBaseUrl, 'artifacts/Bootstrap.ps1') ] - commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -capiArcDataClusterName ${capiArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName}' + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminUsername ${windowsAdminUsername} -adminPassword ${windowsAdminPassword} -spnClientId ${spnClientId} -spnClientSecret ${spnClientSecret} -spnTenantId ${spnTenantId} -spnAuthority ${spnAuthority} -subscriptionId ${subscription().subscriptionId} -resourceGroup ${resourceGroup().name} -azdataUsername ${azdataUsername} -azdataPassword ${azdataPassword} -acceptEula ${acceptEula} -registryUsername ${registryUsername} -registryPassword ${registryPassword} -arcDcName ${arcDcName} -azureLocation ${location} -mssqlmiName ${mssqlmiName} -POSTGRES_NAME ${postgresName} -POSTGRES_WORKER_NODE_COUNT ${postgresWorkerNodeCount} -POSTGRES_DATASIZE ${postgresDatasize} -POSTGRES_SERVICE_TYPE ${postgresServiceType} -stagingStorageAccountName ${stagingStorageAccountName} -workspaceName ${workspaceName} -templateBaseUrl ${templateBaseUrl} -flavor ${flavor} -k3sArcDataClusterName ${k3sArcDataClusterName} -k3sArcClusterName ${k3sArcClusterName} -aksArcClusterName ${aksArcClusterName} -aksdrArcClusterName ${aksdrArcClusterName} -githubUser ${githubUser} -vmAutologon ${vmAutologon} -rdpPort ${rdpPort} -addsDomainName ${addsDomainName} -customLocationRPOID ${customLocationRPOID}' } } } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep index dc0cf1ad5c..ba5e3b8f0b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/aks.bicep @@ -50,7 +50,7 @@ param enableRBAC bool = true param osType string = 'Linux' @description('The version of Kubernetes') -param kubernetesVersion string = '1.26.6' +param kubernetesVersion string = '1.27.9' var serviceCidr_primary = '10.20.64.0/19' var dnsServiceIP_primary = '10.20.64.10' diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 342a423213..d1cc5183e6 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -49,35 +49,66 @@ param templateBaseUrl string @description('Choice to deploy Bastion to connect to the client VM') param deployBastion bool = false -var publicIpAddressName = '${vmName}-PIP' -var networkInterfaceName = '${vmName}-NIC' -var osDiskType = 'Premium_LRS' -var PublicIPNoBastion = { - id: publicIpAddress.id -} +@description('Storage account container name for artifacts') +param storageContainerName string +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string -resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { - name: networkInterfaceName - location: azureLocation - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - subnet: { - id: subnetId - } - privateIPAllocationMethod: 'Dynamic' - publicIPAddress: deployBastion== false ? PublicIPNoBastion : null - } - } - ] - } -} +@description('The number of IP addresses to create') +param numberOfIPAddresses int = 7 -resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ - name: publicIpAddressName +var publicIpAddressName = '${vmName}-PIP' +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +// var PublicIPNoBastion = { +// id: publicIpAddress.id +// } +var k3sControlPlane = 'true' // deploy single-node k3s control plane +var diskSize = (flavor == 'DataOps') ? 512 : 64 + + +// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { +// name: networkInterfaceName +// location: azureLocation +// properties: { +// ipConfigurations: [ +// { +// name: 'ipconfig1' +// properties: { +// subnet: { +// id: subnetId +// } +// privateIPAllocationMethod: 'Dynamic' +// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null +// } +// } +// ] +// } +// } + +// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ +// name: publicIpAddressName +// location: azureLocation +// properties: { +// publicIPAllocationMethod: 'Static' +// publicIPAddressVersion: 'IPv4' +// idleTimeoutInMinutes: 4 +// } +// sku: { +// name: 'Basic' +// } +// } + +// Create multiple public IP addresses if deployBastion is false +resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { + name: '${publicIpAddressName}${i}' location: azureLocation properties: { publicIPAllocationMethod: 'Static' @@ -87,6 +118,27 @@ resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(d sku: { name: 'Basic' } +}] + +// Create multiple NIC IP configurations and assign the public IP to the IP configuration if deployBastion is false +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [for i in range(1, numberOfIPAddresses): { + name: 'ipconfig${i}' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIpAddresses[i-1].id + } + primary: i == 1 ? true : false + } + }] + } } resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { @@ -105,6 +157,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { managedDisk: { storageAccountType: osDiskType } + diskSizeGB: diskSize } imageReference: { publisher: 'canonical' @@ -149,7 +202,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${deployBastion}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep new file mode 100644 index 0000000000..7b93acdea6 --- /dev/null +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -0,0 +1,150 @@ +@description('The name of you Virtual Machine') +param vmName string = 'ArcBox-K3s-Node' + +@description('Username for the Virtual Machine') +param adminUsername string = 'arcdemo' + +@description('SSH Key for the Virtual Machine. SSH key is recommended over password') +@secure() +param sshRSAPublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version') +@allowed([ + '22_04-lts-gen2' +]) +param ubuntuOSVersion string = '22_04-lts-gen2' + +@description('Location for all resources.') +param azureLocation string = resourceGroup().location + +@description('Resource Id of the subnet in the virtual network') +param subnetId string + +param resourceTags object = { + Project: 'jumpstart_arcbox' +} + +@description('Azure service principal client id') +param spnClientId string + +@description('Azure service principal client secret') +@secure() +param spnClientSecret string + +@description('Azure AD tenant id for your service principal') +param spnTenantId string + +@description('Name for the staging storage account using to hold kubeconfig. This value is passed into the template as an output from mgmtStagingStorage.json') +param stagingStorageAccountName string + +@description('Name of the Log Analytics workspace used with cluster extensions') +param logAnalyticsWorkspace string + +@description('The base URL used for accessing artifacts and automation artifacts') +param templateBaseUrl string + +@description('The flavor of ArcBox you want to deploy. Valid values are: \'Full\', \'ITPro\'') +@allowed([ + 'Full' + 'ITPro' + 'DevOps' + 'DataOps' +]) +param flavor string + +@description('Storage account container name for artifacts') +param storageContainerName string + +var networkInterfaceName = '${vmName}-NIC' +var osDiskType = 'Premium_LRS' +var vmSize = (flavor == 'DevOps') ? 'Standard_B2ms' : 'Standard_B8ms' +var diskSize = (flavor == 'DataOps') ? 512 : 64 + +resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { + name: networkInterfaceName + location: azureLocation + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnetId + } + privateIPAllocationMethod: 'Dynamic' + } + } + ] + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { + name: vmName + location: azureLocation + tags: resourceTags + properties: { + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + name: '${vmName}-OSDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + diskSizeGB: diskSize + } + imageReference: { + publisher: 'canonical' + offer: '0001-com-ubuntu-server-jammy' + sku: ubuntuOSVersion + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: networkInterface.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + linuxConfiguration: { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: sshRSAPublicKey + } + ] + } + } + } + } +} + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } +} + +output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 5287d63ee8..3efc686c0f 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'microsoft' +param githubAccount string = 'zaidmohd' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'microsoft' +param githubUser string = 'zaidmohd' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' @@ -60,15 +60,18 @@ param guid string = substring(newGuid(),0,4) @description('Azure location to deploy all resources') param location string = resourceGroup().location -var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' +@description('The custom location RPO ID') +param customLocationRPOID string -var capiArcDataClusterName = 'ArcBox-CAPI-Data-${guid}' -var k3sArcDataClusterName = 'ArcBox-K3s-${guid}' +var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = 'ArcBox-AKS-Data-${guid}' var aksDrArcDataClusterName = 'ArcBox-AKS-DR-Data-${guid}' +var k3sArcDataClusterName = 'ArcBox-DataSvc-K3s-${guid}' +var k3sArcClusterName = 'ArcBox-K3s-${guid}' +var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster -module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { - name: 'ubuntuCAPIDeployment' +module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -80,16 +83,35 @@ module ubuntuCAPIDeployment 'kubernetes/ubuntuCapi.bicep' = if (flavor == 'Full' subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location + vmName : k3sArcDataClusterName + storageContainerName: toLower(k3sArcDataClusterName) flavor: flavor - capiArcDataClusterName : capiArcDataClusterName + } +} + +module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps') { + name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}' + params: { + sshRSAPublicKey: sshRSAPublicKey + spnClientId: spnClientId + spnClientSecret: spnClientSecret + spnTenantId: spnTenantId + stagingStorageAccountName: stagingStorageAccountDeployment.outputs.storageAccountName + logAnalyticsWorkspace: logAnalyticsWorkspaceName + templateBaseUrl: templateBaseUrl + subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId + azureLocation: location + flavor: flavor + vmName : '${k3sArcDataClusterName}-Node-0${i}' + storageContainerName: toLower(k3sArcDataClusterName) } dependsOn: [ - updateVNetDNSServers + ubuntuRancherK3sDataSvcDeployment ] -} +}] -module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { - name: 'ubuntuRancherDeployment' +module ubuntuRancherK3sDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == 'Full' || flavor == 'DevOps') { + name: 'ubuntuRancherK3sDeployment' params: { sshRSAPublicKey: sshRSAPublicKey spnClientId: spnClientId @@ -101,7 +123,9 @@ module ubuntuRancherDeployment 'kubernetes/ubuntuRancher.bicep' = if (flavor == subnetId: mgmtArtifactsAndPolicyDeployment.outputs.subnetId deployBastion: deployBastion azureLocation: location - vmName : k3sArcDataClusterName + vmName : k3sArcClusterName + storageContainerName: toLower(k3sArcClusterName) + flavor: flavor } } @@ -122,13 +146,14 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = { deployBastion: deployBastion githubUser: githubUser location: location - k3sArcClusterName : k3sArcDataClusterName - capiArcDataClusterName : capiArcDataClusterName + k3sArcDataClusterName : k3sArcDataClusterName + k3sArcClusterName : k3sArcClusterName aksArcClusterName : aksArcDataClusterName aksdrArcClusterName : aksDrArcDataClusterName vmAutologon: vmAutologon rdpPort: rdpPort addsDomainName: addsDomainName + customLocationRPOID: customLocationRPOID } dependsOn: [ updateVNetDNSServers From 25238a37a0bfb14122f638591d6e641360d5cdd5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 14:59:55 -0400 Subject: [PATCH 02/38] fix az login --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 406795706e..402eaace64 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file From 4f0538379e6b342d384eb6e8a58b1cfa272f2818 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 29 May 2024 21:46:43 -0400 Subject: [PATCH 03/38] update to managed identity --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/installK3s.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index ba98390998..7084717a84 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -31,7 +31,7 @@ Connect-AzAccount -Identity -Tenant $env:spntenantId -Subscription $env:subscrip # Required for CLI commands Write-Header "Az CLI Login" -az login --identity --tenant $spnTenantId +az login --identity az account set -s $env:subscriptionId # Retrieve Azure Key Vault secrets and store as runtime environment variables diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 04df2fa835..a1eb599b00 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -56,7 +56,8 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID +sudo -u $adminUsername az login --identity sudo -u $adminUsername az account set --subscription $subscriptionId az -v From 1334251d44a1ac02dd74b4d94ca608c2349d9d4a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 14:37:59 -0400 Subject: [PATCH 04/38] enable system assigned identity --- .../bicep/kubernetes/ubuntuRancher.bicep | 13 +++++++++++++ .../bicep/kubernetes/ubuntuRancherNodes.bicep | 13 ++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index d1cc5183e6..1ad353169b 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -145,6 +145,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -209,3 +212,13 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } } + +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 7b93acdea6..f76002a3a1 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -82,6 +82,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { name: vmName location: azureLocation tags: resourceTags + identity: { + type: 'SystemAssigned' + } properties: { hardwareProfile: { vmSize: vmSize @@ -147,4 +150,12 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 } } -output privateIP string = networkInterface.properties.ipConfigurations[0].properties.privateIPAddress +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} From e67a6362fe57f823b0c51e3c37a4779214faf563 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 30 May 2024 16:40:28 -0400 Subject: [PATCH 05/38] remove debugging --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index a1eb599b00..71f520aaf0 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -140,7 +140,7 @@ if [[ "$k3sControlPlane" == "true" ]]; then echo "" resourceGroup=$(sudo -u $adminUsername az resource list --query "[?name=='$stagingStorageAccountName']".[resourceGroup] --resource-type "Microsoft.Storage/storageAccounts" -o tsv) workspaceResourceId=$(sudo -u $adminUsername az resource show --resource-group $resourceGroup --name $logAnalyticsWorkspace --resource-type "Microsoft.OperationalInsights/workspaces" --query id -o tsv) - sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' --debug + sudo -u $adminUsername az connectedk8s connect --name $vmName --resource-group $resourceGroup --location $location --tags 'Project=jumpstart_arcbox' # Enabling Container Insights and Microsoft Defender for Containers cluster extensions echo "" From af8d36b85a901e3afc3d1c73f57a3211802491c4 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:47:41 -0400 Subject: [PATCH 06/38] comment resource provider, this will be part of pre-req --- azure_jumpstart_arcbox/artifacts/installK3s.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 71f520aaf0..c41d24fd68 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -123,15 +123,15 @@ if [[ "$k3sControlPlane" == "true" ]]; then sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig - # Registering Azure resource providers - echo "" - echo "Registering Azure resource providers" - echo "" - sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait - sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait + # # Registering Azure resource providers + # echo "" + # echo "Registering Azure resource providers" + # echo "" + # sudo -u $adminUsername az provider register --namespace 'Microsoft.Kubernetes' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.KubernetesConfiguration' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.PolicyInsights' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.ExtendedLocation' --wait + # sudo -u $adminUsername az provider register --namespace 'Microsoft.AzureArcData' --wait # sudo service sshd restart # Onboard the cluster to Azure Arc From d5e59bb726c23340f067ebb4ddaa42cb4184c70a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 21:59:50 -0400 Subject: [PATCH 07/38] update ip address --- .../bicep/kubernetes/ubuntuRancher.bicep | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 1ad353169b..9eef2d56ff 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -61,50 +61,12 @@ param storageContainerName string ]) param flavor string -@description('The number of IP addresses to create') -param numberOfIPAddresses int = 7 - var publicIpAddressName = '${vmName}-PIP' var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' -// var PublicIPNoBastion = { -// id: publicIpAddress.id -// } var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 - - -// resource networkInterface 'Microsoft.Network/networkInterfaces@2022-01-01' = { -// name: networkInterfaceName -// location: azureLocation -// properties: { -// ipConfigurations: [ -// { -// name: 'ipconfig1' -// properties: { -// subnet: { -// id: subnetId -// } -// privateIPAllocationMethod: 'Dynamic' -// publicIPAddress: deployBastion== false ? PublicIPNoBastion : null -// } -// } -// ] -// } -// } - -// resource publicIpAddress 'Microsoft.Network/publicIpAddresses@2022-01-01' = if(deployBastion == false){ -// name: publicIpAddressName -// location: azureLocation -// properties: { -// publicIPAllocationMethod: 'Static' -// publicIPAddressVersion: 'IPv4' -// idleTimeoutInMinutes: 4 -// } -// sku: { -// name: 'Basic' -// } -// } +var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From c6cba69df6b3f9400bdeb249fc1c0bb31ac1ae16 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 4 Jun 2024 22:26:14 -0400 Subject: [PATCH 08/38] remove kv extension --- .../artifacts/DevOpsLogonScript.ps1 | 76 +++++++++---------- .../artifacts/devops_ingress/bookbuyer.yaml | 53 ------------- .../artifacts/devops_ingress/bookstore.yaml | 53 ------------- .../artifacts/devops_ingress/hello-arc.yaml | 53 ------------- 4 files changed, 38 insertions(+), 197 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 402eaace64..c805bd1c70 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -10,7 +10,7 @@ $osmCLIReleaseVersion = "v1.2.3" $osmMeshName = "osm" $ingressNamespace = "ingress-nginx" -$certname = "ingress-cert" +# $certname = "ingress-cert" $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -198,43 +198,43 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Header "Installing KeyVault Extension" - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" - -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcDataClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Replace Variable values +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Header "Installing KeyVault Extension" + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" + +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcDataClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Replace Variable values Get-ChildItem -Path $Env:ArcBoxKVDir | ForEach-Object { - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName - (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName + # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:spnTenantId | Set-Content -Path $_.FullName } Write-Header "Creating Ingress Controller" @@ -242,8 +242,8 @@ Write-Header "Creating Ingress Controller" # Deploy Ingress resources for Bookstore and Hello-Arc App foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) { # Create the Kubernetes secret with the service principal credentials - kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret - kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret + # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml" @@ -284,7 +284,7 @@ Write-Header "Creating Desktop Icons" $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b847169b4..1b611819cc 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index 26c8238a15..f02c21a829 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b372aaaf4b..b35c1315a3 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,52 +1,3 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: {JS_CERTNAME} - key: tls.key - - objectName: {JS_CERTNAME} - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: {JS_KEYVAULTNAME} - objects: | - array: - - | - objectName: {JS_CERTNAME} - objectType: secret - tenantId: {JS_TENANTID} ---- -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -55,10 +6,6 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - {JS_HOST} - secretName: ingress-tls-csi rules: - host: {JS_HOST} http: From f846010461152e01fe03b57949485c883074cb21 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 14:45:01 -0400 Subject: [PATCH 09/38] add kubevip for service ip --- .../artifacts/DevOpsLogonScript.ps1 | 130 +++++++++++++++++- 1 file changed, 127 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index c805bd1c70..33f46a174b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -15,6 +15,12 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" +$clusters = @( + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } +) + Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log # Required for azcopy and Get-AzResource @@ -97,10 +103,128 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# Longhorn setup for RWX-capable storage class -Write-Header "Creating longhorn storage" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +foreach ($cluster in $clusters) { + +Write-Header "Configuring kube-vip on K3s cluster" +kubectx $cluster.context +kubectl apply -f https://kube-vip.io/manifests/rbac.yaml + +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv + +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +"@ + +$kubeVipDaemonset | kubectl apply -f - + +# Kube vip cloud controller +kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} +$lowestServiceIp = $sortedIps[0] +$highestServiceIp = $sortedIps[-1] + +kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp +Start-Sleep -Seconds 30 + +Write-Header "Creating longhorn storage on K3scluster" +kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig Start-Sleep -Seconds 30 +Write-Host "`n" +} + +# # Longhorn setup for RWX-capable storage class +# Write-Header "Creating longhorn storage" +# kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" +# Start-Sleep -Seconds 30 # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" From 541d52610400af5e965b1f485ac17719eadb575d Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 5 Jun 2024 22:36:12 -0400 Subject: [PATCH 10/38] fix kube config --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 33f46a174b..5ecffbc78d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -16,9 +16,9 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = $Env:k3sArcDataClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = $Env:k3sArcClusterName.ToLower() ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -207,7 +207,8 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv +$nicName = $cluster.clusterName + "-NIC" +$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] $highestServiceIp = $sortedIps[-1] @@ -228,6 +229,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" +kubectx $clusters[0].context az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 905f5550b97e064d4dccd407a5acd6cd37256195 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 00:06:29 -0400 Subject: [PATCH 11/38] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5ecffbc78d..6487d5e756 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -109,6 +109,7 @@ Write-Header "Configuring kube-vip on K3s cluster" kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +$nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" @@ -207,7 +208,6 @@ $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -$nicName = $cluster.clusterName + "-NIC" $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 246b7d32b0396d75ee24c8dde30bb68c293d87f5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 13:41:17 -0400 Subject: [PATCH 12/38] fix nic name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 6487d5e756..7025e8d340 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -110,7 +110,7 @@ kubectx $cluster.context kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $cluster.clusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv +$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv $kubeVipDaemonset = @" apiVersion: apps/v1 From 2cfb09b9d54cd8406e9aabdefc4deaef99115712 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:21:02 -0400 Subject: [PATCH 13/38] fix kube config --- .../artifacts/DevOpsLogonScript.ps1 | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 7025e8d340..3562707cac 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "arcbox-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } - [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "arcbox-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DevOpsLogonScript.log @@ -76,17 +76,17 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s -Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" -Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" -$Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" -kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp -kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" -Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" -$Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx +# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s +# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" +# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" +# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp +# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config" +# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s" +# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config" +# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" +# kubectx # Download OSM binaries Write-Header "Downloading OSM Binaries" @@ -106,7 +106,8 @@ az -v foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" -kubectx $cluster.context +$Env:KUBECONFIG=$cluster.kubeConfig +kubectx kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" @@ -229,7 +230,8 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -kubectx $clusters[0].context +$Env:KUBECONFIG=$cluster[0].kubeConfig +kubectx az k8s-extension create ` --name $osmMeshName ` --extension-type Microsoft.openservicemesh ` From 18cd0b27d09495302b64e538da0f89cf8e1faba7 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 14:48:42 -0400 Subject: [PATCH 14/38] Add role assignment dependency --- .../bicep/kubernetes/ubuntuRancher.bicep | 23 +++++++++++-------- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 9eef2d56ff..b06ca94334 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -156,6 +156,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -173,14 +183,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index f76002a3a1..50e84b71c9 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -131,6 +131,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2022-03-01' = { } } +// Add role assignment for the VM: Owner role +resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') + scope: resourceGroup() + properties: { + principalId: vm.identity.principalId + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + } +} + resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { parent: vm name: 'installscript_k3s' @@ -148,14 +158,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 ] } } -} - -// Add role assignment for the VM: Owner role -resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(vm.id, 'Microsoft.Authorization/roleAssignments', 'Owner') - scope: resourceGroup() - properties: { - principalId: vm.identity.principalId - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - } + dependsOn: [ + vmRoleAssignment_Owner + ] } From 8d81c2b9ad28e903a3628b4d1b266082f5731ecc Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 16:59:58 -0400 Subject: [PATCH 15/38] add kubeconfig --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 3562707cac..a82e55e9d5 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -230,7 +230,7 @@ Write-Host "`n" # "Create OSM Kubernetes extension instance" Write-Header "Creating OSM K8s Extension Instance" -$Env:KUBECONFIG=$cluster[0].kubeConfig +$Env:KUBECONFIG=$clusters[0].kubeConfig kubectx az k8s-extension create ` --name $osmMeshName ` From 866e104854f5c88d7ab76d118053e1758cb16c7e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 18:48:44 -0400 Subject: [PATCH 16/38] update https --- azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 index 59edcc6b9b..6354717a4c 100644 --- a/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 +++ b/azure_jumpstart_arcbox/artifacts/BookStoreLaunch.ps1 @@ -1,2 +1,2 @@ -Start-Process -FilePath msedge -ArgumentList '--new-window https://arcbox.devops.com/bookbuyer https://arcbox.devops.com/bookstore https://arcbox.devops.com/bookstore-v2' +Start-Process -FilePath msedge -ArgumentList '--new-window http://arcbox.devops.com/bookbuyer http://arcbox.devops.com/bookstore http://arcbox.devops.com/bookstore-v2' [Environment]::Exit(1) From e3e4d9131a83d1960d2fa0ea6d575114cc34e995 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 19:07:55 -0400 Subject: [PATCH 17/38] remove kube vip rbac url --- .../artifacts/DevOpsLogonScript.ps1 | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index a82e55e9d5..86db5e3ecd 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -68,7 +68,6 @@ $sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceTyp $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config-k3s" $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config" -kubectx # Downloading ArcBox-K3s log file Write-Header "Downloading ArcBox-K3s Install Logs" @@ -108,11 +107,58 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +"@ +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet $kubeVipDaemonset = @" apiVersion: apps/v1 kind: DaemonSet @@ -203,12 +249,12 @@ status: numberMisscheduled: 0 numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml +# Set kube-vip range-global for kubernetes services $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} $lowestServiceIp = $sortedIps[0] From 0866319aac17f396cd83016ddd1ac1083d558f4b Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 6 Jun 2024 22:34:23 -0400 Subject: [PATCH 18/38] remove spn details --- .../artifacts/installK3s.sh | 58 +++++++++++-------- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- .../bicep/kubernetes/ubuntuRancherNodes.bicep | 2 +- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index c41d24fd68..6aa4a68f3e 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -11,31 +11,31 @@ sudo echo "staginguser:ArcPassw0rd" | sudo chpasswd # Injecting environment variables echo '#!/bin/bash' >> vars.sh echo $adminUsername:$1 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh -echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh -echo $subscriptionId:$5 | awk '{print substr($1,2); }' >> vars.sh -echo $vmName:$6 | awk '{print substr($1,2); }' >> vars.sh -echo $location:$7 | awk '{print substr($1,2); }' >> vars.sh -echo $stagingStorageAccountName:$8 | awk '{print substr($1,2); }' >> vars.sh -echo $logAnalyticsWorkspace:$9 | awk '{print substr($1,2); }' >> vars.sh -echo $templateBaseUrl:${10} | awk '{print substr($1,2); }' >> vars.sh -echo $storageContainerName:${11} | awk '{print substr($1,2); }' >> vars.sh -echo $k3sControlPlane:${12} | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_ID:$2 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_CLIENT_SECRET:$3 | awk '{print substr($1,2); }' >> vars.sh +# echo $SPN_TENANT_ID:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $subscriptionId:$2 | awk '{print substr($1,2); }' >> vars.sh +echo $vmName:$3 | awk '{print substr($1,2); }' >> vars.sh +echo $location:$4 | awk '{print substr($1,2); }' >> vars.sh +echo $stagingStorageAccountName:$5 | awk '{print substr($1,2); }' >> vars.sh +echo $logAnalyticsWorkspace:$6 | awk '{print substr($1,2); }' >> vars.sh +echo $templateBaseUrl:$7 | awk '{print substr($1,2); }' >> vars.sh +echo $storageContainerName:$8 | awk '{print substr($1,2); }' >> vars.sh +echo $k3sControlPlane:$9 | awk '{print substr($1,2); }' >> vars.sh sed -i '2s/^/export adminUsername=/' vars.sh -sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh -sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh -sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh -sed -i '6s/^/export subscriptionId=/' vars.sh -sed -i '7s/^/export vmName=/' vars.sh -sed -i '8s/^/export location=/' vars.sh -sed -i '9s/^/export stagingStorageAccountName=/' vars.sh -sed -i '10s/^/export logAnalyticsWorkspace=/' vars.sh -sed -i '11s/^/export templateBaseUrl=/' vars.sh -sed -i '12s/^/export storageContainerName=/' vars.sh -sed -i '13s/^/export k3sControlPlane=/' vars.sh +# sed -i '3s/^/export SPN_CLIENT_ID=/' vars.sh +# sed -i '4s/^/export SPN_CLIENT_SECRET=/' vars.sh +# sed -i '5s/^/export SPN_TENANT_ID=/' vars.sh +sed -i '3s/^/export subscriptionId=/' vars.sh +sed -i '4s/^/export vmName=/' vars.sh +sed -i '5s/^/export location=/' vars.sh +sed -i '6s/^/export stagingStorageAccountName=/' vars.sh +sed -i '7s/^/export logAnalyticsWorkspace=/' vars.sh +sed -i '8s/^/export templateBaseUrl=/' vars.sh +sed -i '9s/^/export storageContainerName=/' vars.sh +sed -i '10s/^/export k3sControlPlane=/' vars.sh # Set k3 deployment variables export K3S_VERSION="1.28.2+k3s1" # Do not change! @@ -56,8 +56,18 @@ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "" echo "Log in to Azure" echo "" -# sudo -u $adminUsername az login --service-principal --username $SPN_CLIENT_ID --password=$SPN_CLIENT_SECRET --tenant $SPN_TENANT_ID -sudo -u $adminUsername az login --identity +for i in {1..5}; do + sudo -u $adminUsername az login --identity + if [[ $? -eq 0 ]]; then + break + fi + sleep 15 + if [[ $i -eq 5 ]]; then + echo "Error: Failed to login to Azure after 5 retries" + exit 1 + fi +done + sudo -u $adminUsername az account set --subscription $subscriptionId az -v diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index b06ca94334..7943d98335 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -177,7 +177,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName} ${k3sControlPlane}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 50e84b71c9..a351cedcc3 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -152,7 +152,7 @@ resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-0 autoUpgradeMinorVersion: true settings: {} protectedSettings: { - commandToExecute: 'bash installK3s.sh ${adminUsername} ${spnClientId} ${spnClientSecret} ${spnTenantId} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' fileUris: [ '${templateBaseUrl}artifacts/installK3s.sh' ] From 7eb8d7c734d2943cc980e4921dc8fee5bf5c7cf5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 7 Jun 2024 00:56:44 -0400 Subject: [PATCH 19/38] update k3s scripts --- .../artifacts/devops_ingress/bookbuyer.yaml | 4 +- .../artifacts/devops_ingress/bookstore.yaml | 4 +- .../artifacts/devops_ingress/hello-arc.yaml | 4 +- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 194 +++++++++--------- .../gitops_scripts/ResetBookstore.ps1 | 24 +-- 5 files changed, 107 insertions(+), 123 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml index 1b611819cc..feb77ea40a 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookbuyer.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml index f02c21a829..7cf3770a2e 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/bookstore.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml index b35c1315a3..1dc452b013 100644 --- a/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml +++ b/azure_jumpstart_arcbox/artifacts/devops_ingress/hello-arc.yaml @@ -1,11 +1,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: nginx rules: - host: {JS_HOST} http: diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 693e94e4a9..d001b41df5 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -9,7 +9,7 @@ $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" $ingressNamespace = "ingress-nginx" -$certname = "k3s-ingress-cert" +# $certname = "k3s-ingress-cert" $certdns = "arcbox.k3sdevops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" @@ -57,113 +57,109 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml -################################################ -# - Install Key Vault Extension / Create Ingress -################################################ - -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword - -Write-Host "Importing the TLS certificate to Key Vault" -az keyvault certificate import ` - --vault-name $Env:keyVaultName ` - --password "arcbox" ` - --name $certname ` - --file "$Env:TempDir\$certname.pfx" +# ################################################ +# # - Install Key Vault Extension / Create Ingress +# ################################################ + +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword + +# Write-Host "Importing the TLS certificate to Key Vault" +# az keyvault certificate import ` +# --vault-name $Env:keyVaultName ` +# --password "arcbox" ` +# --name $certname ` +# --file "$Env:TempDir\$certname.pfx" -Write-Host "Installing Azure Key Vault Kubernetes extension instance" -az k8s-extension create ` - --name 'akvsecretsprovider' ` - --extension-type Microsoft.AzureKeyVaultSecretsProvider ` - --scope cluster ` - --cluster-name $Env:k3sArcClusterName ` - --resource-group $Env:resourceGroup ` - --cluster-type connectedClusters ` - --release-namespace kube-system ` - --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' - -# Create the Kubernetes secret with the service principal credentials -kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret -kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - -# Deploy SecretProviderClass -$secretProvider = @" -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: azure-kv-sync-tls -spec: - provider: azure - secretObjects: # secretObjects defines the desired state of synced K8s secret objects - - secretName: ingress-tls-csi - type: kubernetes.io/tls - data: - - objectName: "$certname" - key: tls.key - - objectName: "$certname" - key: tls.crt - parameters: - usePodIdentity: "false" - keyvaultName: $Env:keyVaultName - objects: | - array: - - | - objectName: "$certname" - objectType: secret - tenantId: "$Env:spnTenantId" -"@ - -Write-Host "Creating Secret Provider Class" -$secretProvider | kubectl apply -n $k3sNamespace -f - - -# Create the pod with volume referencing the secrets-store.csi.k8s.io driver -$appConsumer = @" -apiVersion: v1 -kind: Pod -metadata: - name: busybox-secrets-sync -spec: - containers: - - name: busybox - image: k8s.gcr.io/e2e-test-images/busybox:1.29 - command: - - "/bin/sleep" - - "10000" - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-kv-sync-tls" - nodePublishSecretRef: - name: secrets-store-creds -"@ - -Write-Host "Deploying App referencing the secret" -$appConsumer | kubectl apply -n $k3sNamespace -f - +# Write-Host "Installing Azure Key Vault Kubernetes extension instance" +# az k8s-extension create ` +# --name 'akvsecretsprovider' ` +# --extension-type Microsoft.AzureKeyVaultSecretsProvider ` +# --scope cluster ` +# --cluster-name $Env:k3sArcClusterName ` +# --resource-group $Env:resourceGroup ` +# --cluster-type connectedClusters ` +# --release-namespace kube-system ` +# --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true' + +# # Create the Kubernetes secret with the service principal credentials +# kubectl create secret generic secrets-store-creds --namespace $k3sNamespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret +# kubectl --namespace $k3sNamespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true + +# # Deploy SecretProviderClass +# $secretProvider = @" +# apiVersion: secrets-store.csi.x-k8s.io/v1 +# kind: SecretProviderClass +# metadata: +# name: azure-kv-sync-tls +# spec: +# provider: azure +# secretObjects: # secretObjects defines the desired state of synced K8s secret objects +# - secretName: ingress-tls-csi +# type: kubernetes.io/tls +# data: +# - objectName: "$certname" +# key: tls.key +# - objectName: "$certname" +# key: tls.crt +# parameters: +# usePodIdentity: "false" +# keyvaultName: $Env:keyVaultName +# objects: | +# array: +# - | +# objectName: "$certname" +# objectType: secret +# tenantId: "$Env:spnTenantId" +# "@ + +# Write-Host "Creating Secret Provider Class" +# $secretProvider | kubectl apply -n $k3sNamespace -f - + +# # Create the pod with volume referencing the secrets-store.csi.k8s.io driver +# $appConsumer = @" +# apiVersion: v1 +# kind: Pod +# metadata: +# name: busybox-secrets-sync +# spec: +# containers: +# - name: busybox +# image: k8s.gcr.io/e2e-test-images/busybox:1.29 +# command: +# - "/bin/sleep" +# - "10000" +# volumeMounts: +# - name: secrets-store-inline +# mountPath: "/mnt/secrets-store" +# readOnly: true +# volumes: +# - name: secrets-store-inline +# csi: +# driver: secrets-store.csi.k8s.io +# readOnly: true +# volumeAttributes: +# secretProviderClass: "azure-kv-sync-tls" +# nodePublishSecretRef: +# name: secrets-store-creds +# "@ + +# Write-Host "Deploying App referencing the secret" +# $appConsumer | kubectl apply -n $k3sNamespace -f - # Deploy an Ingress Resource referencing the Secret created by the CSI driver $ingressController = @" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-tls + name: ingress annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -188,7 +184,7 @@ Add-Content -Path $Env:windir\System32\drivers\etc\hosts -Value "`n`t$ip`t$certd $shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$certdns" $shortcut.IconLocation="$Env:ArcBoxIconDir\arc.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 68facfccaf..30015437ab 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -19,13 +19,9 @@ kind: Ingress metadata: name: ingress-reset-bookbuyer annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -49,13 +45,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -78,13 +70,9 @@ kind: Ingress metadata: name: ingress-reset-bookstore-v2 annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: /reset spec: - tls: - - hosts: - - "$certdns" - secretName: ingress-tls-csi + ingressClassName: nginx rules: - host: "$certdns" http: @@ -104,6 +92,6 @@ $ingressBookstorev2 | kubectl apply -n bookstore -f - # - Invoke Reset API #################### -Invoke-WebRequest -Uri "https://$certdns/bookbuyer/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore/reset" -UseBasicParsing -Invoke-WebRequest -Uri "https://$certdns/bookstore-v2/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookbuyer/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore/reset" -UseBasicParsing +Invoke-WebRequest -Uri "http://$certdns/bookstore-v2/reset" -UseBasicParsing From 0a0b5bbc69a409469522b12c5569d7ed9d99c120 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 10 Jun 2024 13:23:42 -0400 Subject: [PATCH 20/38] k3s reset script changes --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 1 - .../artifacts/gitops_scripts/K3sGitOps.ps1 | 7 ++++--- .../artifacts/gitops_scripts/K3sRBAC.ps1 | 7 ++++--- .../artifacts/gitops_scripts/ResetBookstore.ps1 | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 86db5e3ecd..8c85b6dafa 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -107,7 +107,6 @@ foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig kubectx -# kubectl apply -f https://kube-vip.io/manifests/rbac.yaml $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index d001b41df5..5824acc47c 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -16,8 +16,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sGitOps.log -Write-Host "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -25,7 +25,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index 6064c1fc36..ecdb1319df 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -7,8 +7,8 @@ $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" Start-Transcript -Path $Env:ArcBoxLogsDir\K3sRBAC.log -# echo "Login to Az CLI using the service principal" -az login --service-principal --username $Env:spnClientID --password=$Env:spnClientSecret --tenant $Env:spnTenantId +Write-Host "Login to Az CLI using the managed identity" +az login --identity # Making extension install dynamic az config set extension.use_dynamic_install=yes_without_prompt @@ -16,7 +16,8 @@ Write-Host "`n" az -v # Switch kubectl context to arcbox-k3s -kubectx arcbox-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config-k3s" +kubectx ############################# # - Apply GitOps Configs diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index 30015437ab..dc246f9f66 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,8 +4,9 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-k3s -kubectx arcbox-datasvc-k3s +# Switch kubectl context to arcbox-datasvc-k3s +$Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" +kubectx ############################ # - Deploy Ingress for Reset From f368c43549c3315964d4336f3552dbd3cfea4ceb Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 11:18:13 -0400 Subject: [PATCH 21/38] Fix resource name --- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 | 2 +- azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 5824acc47c..3da9a15048 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -3,7 +3,7 @@ $Env:ToolsDir = "C:\Tools" $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 index ecdb1319df..1a2d2416bb 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sRBAC.ps1 @@ -1,5 +1,5 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "ArcBox-K3s" | Where-Object { $_ -ne "" } $Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n","" $k3sNamespace = "hello-arc" @@ -24,7 +24,7 @@ kubectx ############################# # Create GitOps config for Hello-Arc RBAC -echo "Creating GitOps config for Hello-Arc RBAC" +Write-Host "Creating GitOps config for Hello-Arc RBAC" az k8s-configuration flux create ` --cluster-name $Env:k3sArcClusterName ` --resource-group $Env:resourceGroup ` From d00752ea2ae310a7695da89310a9a45cd904bf26 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 12:36:39 -0400 Subject: [PATCH 22/38] update icon names --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 8c85b6dafa..51c021ceb4 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -454,7 +454,7 @@ New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Edge\ExtensionInstallFo Write-Header "Creating Desktop Icons" # Creating K3s Hello Arc Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Hello-Arc.lnk" +$shortcutLocation = "$Env:Public\Desktop\Hello-Arc.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "http://$certdns" @@ -463,7 +463,7 @@ $shortcut.WindowStyle = 3 $shortcut.Save() # Creating K3s Bookstore Icon on Desktop -$shortcutLocation = "$Env:Public\Desktop\K3s Bookstore.lnk" +$shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) $shortcut.TargetPath = "pwsh.exe" From 984145308258c5c4688854fa13f3494e443d791e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 21:56:29 -0400 Subject: [PATCH 23/38] fix format --- .../artifacts/DevOpsLogonScript.ps1 | 313 +++++++++--------- 1 file changed, 158 insertions(+), 155 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 51c021ceb4..57b170bbd8 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,170 +102,173 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -foreach ($cluster in $clusters) { - -Write-Header "Configuring kube-vip on K3s cluster" -$Env:KUBECONFIG=$cluster.kubeConfig -kubectx - -$nicName = $cluster.clusterName + "-NIC" -$k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - -# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml -$kubeVipRBAC = @" -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io +# foreach ($cluster in $clusters) { + +$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { + $cluster = $_ + + Write-Header "Configuring kube-vip on K3s cluster" + $Env:KUBECONFIG=$cluster.kubeConfig + kubectx + + $nicName = $cluster.clusterName + "-NIC" + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv + + # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml + $kubeVipRBAC = @" + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ -$kubeVipRBAC | kubectl apply -f - - -# Apply kube-vip DaemonSet -$kubeVipDaemonset = @" -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: + $kubeVipRBAC | kubectl apply -f - + + # Apply kube-vip DaemonSet + $kubeVipDaemonset = @" + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system + spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ -$kubeVipDaemonset | kubectl apply -f - + $kubeVipDaemonset | kubectl apply -f - -# Kube vip cloud controller -kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + # Kube vip cloud controller + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml -# Set kube-vip range-global for kubernetes services -$serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv -$sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} -$lowestServiceIp = $sortedIps[0] -$highestServiceIp = $sortedIps[-1] + # Set kube-vip range-global for kubernetes services + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] -kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp -Start-Sleep -Seconds 30 + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 -Write-Header "Creating longhorn storage on K3scluster" -kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig -Start-Sleep -Seconds 30 -Write-Host "`n" + Write-Header "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" } # # Longhorn setup for RWX-capable storage class From ac20698cb90967c49f5267527ddcf224c8b314e3 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Wed, 12 Jun 2024 23:43:34 -0400 Subject: [PATCH 24/38] fix format --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 57b170bbd8..e45fc2f83b 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -102,10 +102,7 @@ az config set extension.use_dynamic_install=yes_without_prompt Write-Host "`n" az -v -# foreach ($cluster in $clusters) { - -$clusters | Foreach-Object -ThrottleLimit 5 -Parallel { - $cluster = $_ +foreach ($cluster in $clusters) { Write-Header "Configuring kube-vip on K3s cluster" $Env:KUBECONFIG=$cluster.kubeConfig @@ -158,6 +155,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { name: kube-vip namespace: kube-system "@ + $kubeVipRBAC | kubectl apply -f - # Apply kube-vip DaemonSet @@ -251,6 +249,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { numberMisscheduled: 0 numberReady: 0 "@ + $kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller From 26e47c0a8bf918ca68025803308830dc4731dd66 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 18:35:11 -0400 Subject: [PATCH 25/38] fix inline kubectl format --- .../artifacts/DevOpsLogonScript.ps1 | 295 ++++++++++-------- 1 file changed, 162 insertions(+), 133 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index e45fc2f83b..5b363a155d 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -111,146 +111,146 @@ foreach ($cluster in $clusters) { $nicName = $cluster.clusterName + "-NIC" $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $nicName --query "[?primary == ``true``].privateIPAddress" -otsv - # Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml - $kubeVipRBAC = @" - apiVersion: v1 - kind: ServiceAccount - metadata: - name: kube-vip - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 +# Apply kube-vip RBAC manifests https://kube-vip.io/manifests/rbac.yaml +$kubeVipRBAC = @" +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role - rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:kube-vip-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role - subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system "@ - $kubeVipRBAC | kubectl apply -f - - - # Apply kube-vip DaemonSet - $kubeVipDaemonset = @" - apiVersion: apps/v1 - kind: DaemonSet - metadata: - creationTimestamp: null - labels: +$kubeVipRBAC | kubectl apply -f - + +# Apply kube-vip DaemonSet +$kubeVipDaemonset = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.7.0 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - name: kube-vip-ds - namespace: kube-system - spec: - selector: - matchLabels: + template: + metadata: + creationTimestamp: null + labels: app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.7.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: eth0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: "$k3sVIP" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.7.0 - imagePullPolicy: Always - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} - status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 + app.kubernetes.io/version: v0.7.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: eth0 + - name: vip_cidr + value: "32" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: "$k3sVIP" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.7.0 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 "@ - $kubeVipDaemonset | kubectl apply -f - +$kubeVipDaemonset | kubectl apply -f - # Kube vip cloud controller kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml @@ -373,6 +373,35 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcDataClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcDataClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcDataClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcDataClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 7995479064833d64a02b5107d59ef852d4605b71 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 13 Jun 2024 20:41:22 -0400 Subject: [PATCH 26/38] add retry for k3s gitops --- .../artifacts/gitops_scripts/K3sGitOps.ps1 | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 index 3da9a15048..8d29d8cf63 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/K3sGitOps.ps1 @@ -58,6 +58,36 @@ az k8s-configuration flux create ` --branch main --sync-interval 3s ` --kustomization name=helloarc path=./hello-arc/yaml +$configs = $(az k8s-configuration flux list --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup --query "[].name" -otsv) + +foreach ($configName in $configs) { + Write-Host "Checking GitOps configuration $configName on $Env:k3sArcClusterName" + $retryCount = 0 + $maxRetries = 5 + do { + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Compliant") { + Write-Host "GitOps configuration $configName is ready on $Env:k3sArcClusterName" + } + else { + if ($configStatus.ComplianceState -ne "Non-compliant") { + Start-Sleep -Seconds 60 + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 60 + $configStatus = $(az k8s-configuration flux show --name $configName --cluster-name $Env:k3sArcClusterName --cluster-type connectedClusters --resource-group $Env:resourceGroup -o json 2>$null) | convertFrom-JSON + if ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -lt $maxRetries) { + $retryCount++ + } + } + elseif ($configStatus.ComplianceState -eq "Non-compliant" -and $retryCount -eq $maxRetries) { + Write-Host "GitOps configuration $configName has failed on $Env:k3sArcClusterName. Exiting..." + break + } + } + } until ($configStatus.ComplianceState -eq "Compliant") +} + # ################################################ # # - Install Key Vault Extension / Create Ingress # ################################################ From 5cf9318d8ca94c8755ee45a3c6538596348e2de2 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 14 Jun 2024 00:06:54 -0400 Subject: [PATCH 27/38] update github account --- azure_jumpstart_arcbox/bicep/main.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 3efc686c0f..abee87be15 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -40,7 +40,7 @@ param logAnalyticsWorkspaceName string param flavor string = 'Full' @description('Target GitHub account') -param githubAccount string = 'zaidmohd' +param githubAccount string = 'microsoft' @description('Target GitHub branch') param githubBranch string = 'arcbox_3.0' @@ -49,7 +49,7 @@ param githubBranch string = 'arcbox_3.0' param deployBastion bool = false @description('User github account where they have forked https://github.com/microsoft/azure-arc-jumpstart-apps') -param githubUser string = 'zaidmohd' +param githubUser string = 'microsoft' @description('Active directory domain services domain name') param addsDomainName string = 'jumpstart.local' From 124622bedb27dfa45898fe4a7c75f7a673646970 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 4 Jul 2024 14:11:23 -0400 Subject: [PATCH 28/38] fix custom location issue --- .../artifacts/DataOpsAppDRScript.ps1 | 4 +- .../artifacts/DataOpsAppScript.ps1 | 8 +- .../artifacts/DataOpsLogonScript.ps1 | 79 +++++++++---------- .../artifacts/DataOpsTestAppScript.ps1 | 8 +- .../artifacts/DeployAPIM.ps1 | 2 +- .../artifacts/DeployPostgreSQL.ps1 | 2 +- .../artifacts/DeploySQLMI.ps1 | 4 +- .../artifacts/DeploySQLMIADAuth.ps1 | 4 +- .../artifacts/dataController.parameters.json | 4 +- 9 files changed, 57 insertions(+), 58 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index 34ee60b070..164e2df25f 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -21,7 +21,7 @@ Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Dom Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 # Deploy the App and service -$appCAPI = @" +$appK3s = @" apiVersion: apps/v1 kind: Deployment metadata: @@ -67,7 +67,7 @@ spec: "@ Write-Header "Deploying App Resource" -$appCAPI | kubectl apply -n $appNamespace -f - +$appK3s | kubectl apply -n $appNamespace -f - # Deploy an Ingress Resource for the app $appIngress = @" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index c473b7b1df..64fa16f002 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -38,7 +38,7 @@ foreach ($cluster in @('k3s', 'aks-dr')) { helm install dataops-ingress nginx-stable/nginx-ingress } -# Switch kubectl context to capi +# Switch kubectl context to k3s kubectx $sqlInstance Write-Header "Adding CName Record for App" @@ -52,7 +52,7 @@ Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Dom Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 # Deploy the App and service -$appCAPI = @" +$appK3s = @" apiVersion: apps/v1 kind: Deployment metadata: @@ -98,7 +98,7 @@ spec: "@ Write-Header "Deploying App Resource" -$appCAPI | kubectl apply -n $appNamespace -f - +$appK3s | kubectl apply -n $appNamespace -f - # Deploy an Ingress Resource for the app $appIngress = @" @@ -135,7 +135,7 @@ Do { $podStatus = $(if(kubectl get pods -n $appNamespace | Select-String "web-app" | Select-String "Running" -Quiet){"Ready!"}Else{"Nope"}) } while ($podStatus -eq "Nope") -# Creating CAPI Bookstore Arc Icon on Desktop +# Creating K3s Bookstore Arc Icon on Desktop $shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index d6428c6abd..3964a08b30 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -5,9 +5,9 @@ $Env:ArcBoxIconDir = "C:\ArcBox\Icons" $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$Env:k3sArcDataClusterName-dc" ; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } - [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } - [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$Env:k3sArcDataClusterName-dc" ; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + # [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } + # [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsLogonScript.log @@ -96,11 +96,11 @@ Write-Host "`n" # Downloading k3s Kubernetes cluster kubeconfig file Write-Header "Downloading k3s Kubeconfig" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/config" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" $context = (Get-AzStorageAccount -ResourceGroupName $Env:resourceGroup).Context -$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Object -Permission racwdlup +$sas = New-AzStorageAccountSASToken -Context $context -Service Blob -ResourceType Container,Object -Permission racwdlup $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config" $addsDomainNetBiosName = $Env:addsDomainName.Split(".")[0] @@ -108,9 +108,9 @@ azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:admin # Downloading 'installk3s.log' log file Write-Header "Downloading k3s Install Logs" -$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/staging-k3s/installK3s-$Env:k3sArcDataClusterName.log" +$sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\installk3s.log" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" Start-Sleep -Seconds 10 @@ -157,20 +157,19 @@ foreach ($cluster in $clusters) { } foreach ($cluster in $clusters) { - if ($cluster.context -eq 'k3s') { - Write-Host "Enabling custom-locations feature on k3s cluster" - az connectedk8s enable-features -n $cluster.clusterName ` - -g $Env:resourceGroup ` - --custom-locations-oid $Env:customLocationRPOID ` - --features cluster-connect custom-locations ` - --kube-config $cluster.kubeConfig --only-show-errors - - Write-Header "Configuring kube-vip on K3s cluster" - kubectx k3s - $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $Env:k3sArcDataClusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv - - Write-Host "Assignin kube-vip-role on k3s cluster" - + if ($cluster.context -eq 'k3s') { + Write-Host "Enabling custom-locations feature on k3s cluster" + az connectedk8s enable-features -n $cluster.clusterName ` + -g $Env:resourceGroup ` + --custom-locations-oid $Env:customLocationRPOID ` + --features cluster-connect custom-locations ` + --kube-config $cluster.kubeConfig --only-show-errors + + Write-Header "Configuring kube-vip on K3s cluster" + kubectx k3s + $k3sVIP = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $Env:k3sArcDataClusterName-NIC --query "[?primary == ``true``].privateIPAddress" -otsv + +Write-Host "Assignin kube-vip-role on k3s cluster" $kubeVipRBAC = @" apiVersion: v1 kind: ServiceAccount @@ -215,7 +214,7 @@ subjects: namespace: kube-system "@ - $kubeVipRBAC | kubectl apply -f - +$kubeVipRBAC | kubectl apply -f - $kubeVipDaemonset = @" apiVersion: apps/v1 @@ -308,24 +307,23 @@ status: numberReady: 0 "@ - Write-Host "Deploying Kube vip cloud controller on k3s cluster" - $kubeVipDaemonset | kubectl apply -f - + $kubeVipDaemonset | kubectl apply -f - - # Kube vip cloud controller - kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + Write-Host "Deploying Kube vip cloud controller on k3s cluster" + kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml - $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $Env:k3sArcDataClusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv - $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} - $lowestServiceIp = $sortedIps[0] - $highestServiceIp = $sortedIps[-1] + $serviceIpRange = az network nic ip-config list --resource-group $Env:resourceGroup --nic-name $Env:k3sArcDataClusterName-NIC --query "[?primary == ``false``].privateIPAddress" -otsv + $sortedIps = $serviceIpRange | Sort-Object {[System.Version]$_} + $lowestServiceIp = $sortedIps[0] + $highestServiceIp = $sortedIps[-1] - kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp - Start-Sleep -Seconds 30 + kubectl create configmap -n kube-system kubevip --from-literal range-global=$lowestServiceIp-$highestServiceIp + Start-Sleep -Seconds 30 - Write-Host "Creating longhorn storage on K3scluster" - kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig - Start-Sleep -Seconds 30 - Write-Host "`n" + Write-Host "Creating longhorn storage on K3scluster" + kubectl apply -f "$Env:ArcBoxDir\longhorn.yaml" --kubeconfig $cluster.kubeConfig + Start-Sleep -Seconds 30 + Write-Host "`n" } } @@ -334,9 +332,9 @@ Stop-Transcript # - Deploying data services on k3s cluster ################################################ -Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'k3s Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-k3s"" ; Start-Sleep -Seconds 5; Clear-Host }" -Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aks"" ; Start-Sleep -Seconds 5; Clear-Host }" -Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aksdr"" ; Start-Sleep -Seconds 5; Clear-Host }" +Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'k3s Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-datasvc-k3s"" ; Start-Sleep -Seconds 5; Clear-Host }" +# Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aks"" ; Start-Sleep -Seconds 5; Clear-Host }" +# Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aksdr"" ; Start-Sleep -Seconds 5; Clear-Host }" Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { @@ -397,6 +395,7 @@ $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { (Get-Content -Path $dataControllerParams) -replace 'subscriptionId-stage', $Env:subscriptionId | Set-Content -Path $dataControllerParams (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsWorkspaceId-stage', $workspaceId | Set-Content -Path $dataControllerParams (Get-Content -Path $dataControllerParams) -replace 'logAnalyticsPrimaryKey-stage', $workspaceKey | Set-Content -Path $dataControllerParams + (Get-Content -Path $dataControllerParams) -replace 'storageClass-stage', $cluster.storageClassName | Set-Content -Path $dataControllerParams Write-Host "Deploying arc data controller on $clusterName" Write-Host "`n" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 index 7a990d962e..d3f8bcf3b6 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsTestAppScript.ps1 @@ -1,14 +1,14 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $appNamespace = "arc" -$sqlInstance = "capi" +$sqlInstance = "k3s" Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsTestAppScript.log -# Switch kubectl context to capi +# Switch kubectl context to k3s kubectx $sqlInstance # Deploy the App and service -$appCAPI = @" +$appK3s = @" apiVersion: apps/v1 kind: Deployment metadata: @@ -37,7 +37,7 @@ spec: "@ Write-Header "Deploying DB Connect Test App" -$appCAPI | kubectl apply -n $appNamespace -f - +$appK3s | kubectl apply -n $appNamespace -f - Do { Write-Host "Waiting for App pod, hold tight..." diff --git a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 index 769c56ac20..04fd862977 100644 --- a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 @@ -28,7 +28,7 @@ az config set extension.use_dynamic_install=yes_without_prompt ################################################ # Retrive SQL Managed Instances ################################################ -kubectx arcbox-capi +kubectx arcbox-datasvc-k3s kubectl get nodes # Retrieving SQL MI connection endpoints diff --git a/azure_jumpstart_arcbox/artifacts/DeployPostgreSQL.ps1 b/azure_jumpstart_arcbox/artifacts/DeployPostgreSQL.ps1 index c3a368be5a..5346834513 100644 --- a/azure_jumpstart_arcbox/artifacts/DeployPostgreSQL.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeployPostgreSQL.ps1 @@ -27,7 +27,7 @@ $coordinatorCoresLimit = "4" $coordinatorMemoryLimit = "8Gi" # Storage -$StorageClassName = "managed-premium" +$StorageClassName = "longhorn" $dataStorageSize = "5Gi" $logsStorageSize = "5Gi" diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMI.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMI.ps1 index 1bd90713fc..e24ac92b34 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMI.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMI.ps1 @@ -13,7 +13,7 @@ Write-Host "Deploying Azure Arc SQL Managed Instance" Write-Host "`n" $dataControllerId = $(az resource show --resource-group $Env:resourceGroup --name $controllerName --resource-type "Microsoft.AzureArcData/dataControllers" --query id -o tsv) -$customLocationId = $(az customlocation show --name "$Env:capiArcDataClusterName-cl" --resource-group $Env:resourceGroup --query id -o tsv) +$customLocationId = $(az customlocation show --name "$Env:k3sArcDataClusterName-cl" --resource-group $Env:resourceGroup --query id -o tsv) ################################################ # Localize ARM template @@ -28,7 +28,7 @@ $vCoresLimit = "4" $memoryLimit = "8Gi" # Storage -$StorageClassName = "managed-premium" +$StorageClassName = "longhorn" $dataStorageSize = "5Gi" $logsStorageSize = "5Gi" $dataLogsStorageSize = "5Gi" diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index ad612f4017..94d2a5bf5d 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -55,7 +55,7 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'capi-sql'; dataController = "$Env:capiArcDataClusterName-dc"; customLocation = "$Env:capiArcDataClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'capi' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-capi" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$Env:k3sArcDataClusterName-dc"; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } @@ -278,7 +278,7 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel { Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance." # Downloading demo database and restoring onto SQL MI - if ($sqlMIName -eq "capi-sql") { + if ($sqlMIName -eq "k3s-sql") { Write-Host "`n" Write-Host "Downloading AdventureWorks database for MS SQL... (1/2)" kubectl exec $podname -n arc --kubeconfig $sqlInstance.kubeConfig -c arc-sqlmi -- wget https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2019.bak -O /var/opt/mssql/data/AdventureWorks2019.bak 2>&1 | Out-Null diff --git a/azure_jumpstart_arcbox/artifacts/dataController.parameters.json b/azure_jumpstart_arcbox/artifacts/dataController.parameters.json index 2dfaec30b8..49d986f54a 100644 --- a/azure_jumpstart_arcbox/artifacts/dataController.parameters.json +++ b/azure_jumpstart_arcbox/artifacts/dataController.parameters.json @@ -41,13 +41,13 @@ "value": "arc-private-registry" }, "dataStorageClass": { - "value": "managed-premium" + "value": "storageClass-stage" }, "dataStorageSize": { "value": "15Gi" }, "logsStorageClass": { - "value": "managed-premium" + "value": "storageClass-stage" }, "logsStorageSize": { "value": "10Gi" From e356c036454924ae1a4b2fabf6754808bade70ea Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 4 Jul 2024 17:14:40 -0400 Subject: [PATCH 29/38] fix install k3s and k3s nodes --- .../artifacts/installK3s.sh | 1 - .../bicep/kubernetes/ubuntuRancherNodes.bicep | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/installK3s.sh b/azure_jumpstart_arcbox/artifacts/installK3s.sh index 6f9eb3e765..5cf76b5ff7 100644 --- a/azure_jumpstart_arcbox/artifacts/installK3s.sh +++ b/azure_jumpstart_arcbox/artifacts/installK3s.sh @@ -129,7 +129,6 @@ if [[ "$k3sControlPlane" == "true" ]]; then sudo -u $adminUsername az extension add --upgrade -n storage-preview storageAccountRG=$(sudo -u $adminUsername az storage account show --name $stagingStorageAccountName --query 'resourceGroup' | sed -e 's/^"//' -e 's/"$//') storageAccountKey=$(sudo -u $adminUsername az storage account keys list --resource-group $storageAccountRG --account-name $stagingStorageAccountName --query [0].value | sed -e 's/^"//' -e 's/"$//') - storageContainerName="staging-k3s" sudo -u $adminUsername az storage container create -n $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $localPath sudo -u $adminUsername az storage azcopy blob upload --container $storageContainerName --account-name $stagingStorageAccountName --account-key $storageAccountKey --source $k3sClusterNodeConfig diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep index 1683a6118e..9a6b9587ee 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancherNodes.bicep @@ -139,3 +139,25 @@ resource vmRoleAssignment_Owner 'Microsoft.Authorization/roleAssignments@2022-04 roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') } } + +resource vmInstallscriptK3s 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = { + parent: vm + name: 'installscript_k3s' + location: azureLocation + properties: { + publisher: 'Microsoft.Azure.Extensions' + type: 'CustomScript' + typeHandlerVersion: '2.1' + autoUpgradeMinorVersion: true + settings: {} + protectedSettings: { + commandToExecute: 'bash installK3s.sh ${adminUsername} ${subscription().subscriptionId} ${vmName} ${azureLocation} ${stagingStorageAccountName} ${logAnalyticsWorkspace} ${templateBaseUrl} ${storageContainerName}' + fileUris: [ + '${templateBaseUrl}artifacts/installK3s.sh' + ] + } + } + dependsOn: [ + vmRoleAssignment_Owner + ] +} From 443c8f4e2e73fdaac2fb6e5da2bd29b3799d41de Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 4 Jul 2024 21:04:29 -0400 Subject: [PATCH 30/38] fix k3s issue --- azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index 3964a08b30..6f4b98ac70 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -6,8 +6,8 @@ $Env:ArcBoxTestsDir = "$Env:ArcBoxDir\Tests" $clusters = @( [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$Env:k3sArcDataClusterName-dc" ; customLocation = "$Env:k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } - # [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } - # [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } + [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$Env:aksArcClusterName-dc" ; customLocation = "$Env:aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } + [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$Env:aksdrArcClusterName-dc" ; customLocation = "$Env:aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsLogonScript.log @@ -333,8 +333,8 @@ Stop-Transcript ################################################ Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'k3s Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-datasvc-k3s"" ; Start-Sleep -Seconds 5; Clear-Host }" -# Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aks"" ; Start-Sleep -Seconds 5; Clear-Host }" -# Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aksdr"" ; Start-Sleep -Seconds 5; Clear-Host }" +Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aks"" ; Start-Sleep -Seconds 5; Clear-Host }" +Start-Process pwsh.exe -ArgumentList "-NoExit", "-Command", "[System.Console]::Title = 'AKS-DR Cluster'; for (0 -lt 1) { kubectl get pods -n arc --kubeconfig ""C:\Users\$Env:USERNAME\.kube\config-aksdr"" ; Start-Sleep -Seconds 5; Clear-Host }" Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { From a2f4bdd4edf956c9d5e896f63fb43558d1554698 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 11 Jul 2024 10:32:29 -0400 Subject: [PATCH 31/38] update acr --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index f977232516..3dce153182 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -71,7 +71,7 @@ spec: spec: containers: - name: web - image: azurearcjumpstart.azurecr.io/demoapp + image: jumpstartdev.azurecr.io/demoapp ports: - containerPort: 80 volumeMounts: From 3774223039d8bf40c1eeaa165ad855dd0a01cb8a Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Thu, 18 Jul 2024 11:51:50 -0400 Subject: [PATCH 32/38] fix app ingress --- .../artifacts/DataOpsAppDRScript.ps1 | 48 ++++-------------- .../artifacts/DataOpsAppScript.ps1 | 49 ++++--------------- 2 files changed, 20 insertions(+), 77 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index 2e31ae144b..fbbf2a0664 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -10,16 +10,6 @@ Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppDRScript.log # Switch kubectl context to AKS DR kubectx $sqlInstance -Write-Header "Adding CName Record for App" -$dcInfo = Get-ADDomainController -Do -{ - $appIpaddress= kubectl get svc "dataops-ingress-nginx-ingress-controller" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" - Start-Sleep -Seconds 5 -} while ($null -eq $appIpaddress) -Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 - # Deploy the App and service $appK3s = @" apiVersion: apps/v1 @@ -69,34 +59,16 @@ spec: Write-Header "Deploying App Resource" $appK3s | kubectl apply -n $appNamespace -f - -# Deploy an Ingress Resource for the app -$appIngress = @" -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-tls - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - tls: - - hosts: - - "$certdns" - secretName: "$CName-secret" - rules: - - host: "$certdns" - http: - paths: - - pathType: ImplementationSpecific - backend: - service: - name: web-app-service - port: - number: 80 - path: / -"@ -Write-Header "Deploying App Ingress Resource" -$appIngress | kubectl apply -n $appNamespace -f - +Write-Header "Adding CName Record for App" +$dcInfo = Get-ADDomainController +Do +{ + $appIpaddress= kubectl get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" + Start-Sleep -Seconds 5 +} while ($null -eq $appIpaddress) +Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 + Do { Write-Host "Waiting for Web App pod, hold tight..." diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 3dce153182..faf90e5f51 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -41,16 +41,6 @@ foreach ($cluster in @('k3s', 'aks-dr')) { # Switch kubectl context to k3s kubectx $sqlInstance -Write-Header "Adding CName Record for App" -$dcInfo = Get-ADDomainController -Do -{ - $appIpaddress= kubectl get svc "dataops-ingress-nginx-ingress-controller" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" - Start-Sleep -Seconds 5 -} while ($null -eq $appIpaddress) -Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 - # Deploy the App and service $appK3s = @" apiVersion: apps/v1 @@ -90,7 +80,7 @@ metadata: spec: selector: app: web - type: ClusterIP + type: LoadBalancer ports: - protocol: TCP port: 80 @@ -100,34 +90,15 @@ spec: Write-Header "Deploying App Resource" $appK3s | kubectl apply -n $appNamespace -f - -# Deploy an Ingress Resource for the app -$appIngress = @" -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-tls - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - tls: - - hosts: - - "$certdns" - secretName: "$CName-secret" - rules: - - host: "$certdns" - http: - paths: - - pathType: ImplementationSpecific - backend: - service: - name: web-app-service - port: - number: 80 - path: / -"@ -Write-Header "Deploying App Ingress Resource" -$appIngress | kubectl apply -n $appNamespace -f - +Write-Header "Adding CName Record for App" +$dcInfo = Get-ADDomainController +Do +{ + $appIpaddress= kubectl get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" + Start-Sleep -Seconds 5 +} while ($null -eq $appIpaddress) +Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 Do { Write-Host "Waiting for Web App pod, hold tight..." From a9ac9d2ff0f50f5b280b91efb1d94f74807eed99 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Fri, 19 Jul 2024 00:19:57 -0400 Subject: [PATCH 33/38] add namespace --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index faf90e5f51..919987fab1 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -94,7 +94,7 @@ Write-Header "Adding CName Record for App" $dcInfo = Get-ADDomainController Do { - $appIpaddress= kubectl get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" + $appIpaddress= kubectl -n $appNamespace get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" Start-Sleep -Seconds 5 } while ($null -eq $appIpaddress) Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord From a25aff40a9c5bad53a990faa12e21e5908835b2e Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 22 Jul 2024 01:52:25 -0400 Subject: [PATCH 34/38] remove ingress --- .../artifacts/DataOpsAppDRScript.ps1 | 13 ++-- .../artifacts/DataOpsAppScript.ps1 | 63 ++++++++++--------- .../bicep/kubernetes/ubuntuRancher.bicep | 2 +- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index fbbf2a0664..be07d7efbe 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -1,7 +1,7 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$CName = "dataops" -$certdns = "$CName.jumpstart.local" +# $CName = "dataops" +# $certdns = "$CName.jumpstart.local" $appNamespace = "arc" $sqlInstance = "aks-dr" @@ -49,7 +49,7 @@ metadata: spec: selector: app: web - type: ClusterIP + type: LoadBalancer ports: - protocol: TCP port: 80 @@ -59,15 +59,16 @@ spec: Write-Header "Deploying App Resource" $appK3s | kubectl apply -n $appNamespace -f - -Write-Header "Adding CName Record for App" +# Write-Header "Adding CName Record for App" $dcInfo = Get-ADDomainController Do { + Write-Host "Waiting for Web App Service, hold tight..." $appIpaddress= kubectl get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" Start-Sleep -Seconds 5 } while ($null -eq $appIpaddress) -Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 +# Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +# Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 Do { diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 919987fab1..1717a11efc 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -2,41 +2,41 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$CName = "jumpstartbooks" -$certdns = "$CName.jumpstart.local" -$password = "arcbox" +# $CName = "jumpstartbooks" +# $certdns = "$CName.jumpstart.local" +# $password = "arcbox" $appNamespace = "arc" $sqlInstance = "k3s" Start-Transcript -Path $Env:ArcBoxLogsDir\DataOpsAppScript.log -# Add OpenSSL to path environment variable -$openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" -$currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Process) -$newPathVariable = $currentPathVariable + ";" + $openSSL -[Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Process) +# # Add OpenSSL to path environment variable +# $openSSL = "C:\Program Files\FireDaemon OpenSSL 3\bin" +# $currentPathVariable = [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Process) +# $newPathVariable = $currentPathVariable + ";" + $openSSL +# [Environment]::SetEnvironmentVariable("PATH", $newPathVariable, [EnvironmentVariableTarget]::Process) -Write-Host "Generating a TLS Certificate" -$cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" -$certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText -Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword -Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword +# Write-Host "Generating a TLS Certificate" +# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My" +# $certPassword = ConvertTo-SecureString -String $password -Force -AsPlainText +# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$CName.pfx" -Password $certPassword +# Import-PfxCertificate -FilePath "$Env:TempDir\$CName.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword -openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password -openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password -openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password +# openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -nocerts -out "$Env:TempDir\$CName.key" -password pass:$password -passout pass:$password +# openssl pkcs12 -in "$Env:TempDir\$CName.pfx" -clcerts -nokeys -out "$Env:TempDir\$CName.crt" -password pass:$password +# openssl rsa -in "$Env:TempDir\$CName.key" -out "$Env:TempDir\$CName-dec.key" -passin pass:$password -Write-Header "Creating Ingress Controller" -foreach ($cluster in @('k3s', 'aks-dr')) { - # Create K8s Ingress TLS secret - kubectx $cluster - kubectl -n $appNamespace create secret tls "$CName-secret" --key "$Env:TempDir\$CName-dec.key" --cert "$Env:TempDir\$CName.crt" +# Write-Header "Creating Ingress Controller" +# foreach ($cluster in @('k3s', 'aks-dr')) { +# # Create K8s Ingress TLS secret +# kubectx $cluster +# kubectl -n $appNamespace create secret tls "$CName-secret" --key "$Env:TempDir\$CName-dec.key" --cert "$Env:TempDir\$CName.crt" - # Deploy NGINX Ingress Controller - helm repo add nginx-stable https://helm.nginx.com/stable - helm repo update - helm install dataops-ingress nginx-stable/nginx-ingress -} +# # Deploy NGINX Ingress Controller +# helm repo add nginx-stable https://helm.nginx.com/stable +# helm repo update +# helm install dataops-ingress nginx-stable/nginx-ingress +# } # Switch kubectl context to k3s kubectx $sqlInstance @@ -90,15 +90,16 @@ spec: Write-Header "Deploying App Resource" $appK3s | kubectl apply -n $appNamespace -f - -Write-Header "Adding CName Record for App" -$dcInfo = Get-ADDomainController +# Write-Header "Adding CName Record for App" +# $dcInfo = Get-ADDomainController Do { + Write-Host "Waiting for Web App Service, hold tight..." $appIpaddress= kubectl -n $appNamespace get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" Start-Sleep -Seconds 5 } while ($null -eq $appIpaddress) -Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 +# Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +# Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 Do { Write-Host "Waiting for Web App pod, hold tight..." @@ -110,7 +111,7 @@ Do { $shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "https://$certdns" +$shortcut.TargetPath = "http://$appIpaddress" $shortcut.IconLocation="$Env:ArcBoxIconDir\bookstore.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() diff --git a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep index 6ede81154e..daa28ca621 100644 --- a/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep +++ b/azure_jumpstart_arcbox/bicep/kubernetes/ubuntuRancher.bicep @@ -52,7 +52,7 @@ var networkInterfaceName = '${vmName}-NIC' var osDiskType = 'Premium_LRS' var k3sControlPlane = 'true' // deploy single-node k3s control plane var diskSize = (flavor == 'DataOps') ? 512 : 64 -var numberOfIPAddresses = (flavor == 'DataOps') ? 7 : 5 // The number of IP addresses to create +var numberOfIPAddresses = (flavor == 'DataOps') ? 8 : 5 // The number of IP addresses to create // Create multiple public IP addresses if deployBastion is false resource publicIpAddresses 'Microsoft.Network/publicIpAddresses@2022-01-01' = [for i in range(1, numberOfIPAddresses): { From 60d262c06c1d71ff7ed36e13fa07470e982b08a5 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 22 Jul 2024 13:21:42 -0400 Subject: [PATCH 35/38] fix cname record --- .../artifacts/DataOpsAppDRScript.ps1 | 8 ++++---- .../artifacts/DataOpsAppScript.ps1 | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index be07d7efbe..86ab904c20 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -1,7 +1,7 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -# $CName = "dataops" -# $certdns = "$CName.jumpstart.local" +$CName = "dataops" +$certdns = "$CName.jumpstart.local" $appNamespace = "arc" $sqlInstance = "aks-dr" @@ -67,8 +67,8 @@ Do $appIpaddress= kubectl get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" Start-Sleep -Seconds 5 } while ($null -eq $appIpaddress) -# Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -# Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 +Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 Do { diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index 1717a11efc..b349c4b7c7 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -2,8 +2,8 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -# $CName = "jumpstartbooks" -# $certdns = "$CName.jumpstart.local" +$CName = "jumpstartbooks" +$certdns = "$CName.jumpstart.local" # $password = "arcbox" $appNamespace = "arc" $sqlInstance = "k3s" @@ -90,16 +90,16 @@ spec: Write-Header "Deploying App Resource" $appK3s | kubectl apply -n $appNamespace -f - -# Write-Header "Adding CName Record for App" -# $dcInfo = Get-ADDomainController +Write-Header "Adding CName Record for App" +$dcInfo = Get-ADDomainController Do { Write-Host "Waiting for Web App Service, hold tight..." $appIpaddress= kubectl -n $appNamespace get svc "web-app-service" -o jsonpath="{.status.loadBalancer.ingress[0].ip}" Start-Sleep -Seconds 5 } while ($null -eq $appIpaddress) -# Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord -# Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 +Add-DnsServerResourceRecord -ComputerName $dcInfo.HostName -ZoneName $dcInfo.Domain -A -Name "$CName-$sqlInstance" -AllowUpdateAny -IPv4Address $appIpaddress -TimeToLive 01:00:00 -AgeRecord +Add-DnsServerResourceRecordCName -Name $CName -ComputerName $dcInfo.HostName -HostNameAlias "$CName-$sqlInstance.jumpstart.local" -ZoneName jumpstart.local -TimeToLive 00:05:00 Do { Write-Host "Waiting for Web App pod, hold tight..." From b667fca08d9e40f243b8848fa750638fef9f53ad Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 22 Jul 2024 18:51:24 -0400 Subject: [PATCH 36/38] cname --- azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 | 7 ++++--- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index 86ab904c20..ff603d1916 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -1,7 +1,8 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$CName = "dataops" -$certdns = "$CName.jumpstart.local" +$CName = "dataops.jumpstart.local" +# $CName = "dataops" +# $certdns = "$CName.jumpstart.local" $appNamespace = "arc" $sqlInstance = "aks-dr" @@ -30,7 +31,7 @@ spec: spec: containers: - name: web - image: azurearcjumpstart.azurecr.io/demoapp:dr + image: jumpstartdev.azurecr.io/demoapp:dr ports: - containerPort: 80 volumeMounts: diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index b349c4b7c7..b8b1ac8e96 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -2,8 +2,9 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$CName = "jumpstartbooks" -$certdns = "$CName.jumpstart.local" +$CName = "jumpstartbooks.jumpstart.local" +# $CName = "jumpstartbooks" +# $certdns = "$CName.jumpstart.local" # $password = "arcbox" $appNamespace = "arc" $sqlInstance = "k3s" @@ -111,7 +112,7 @@ Do { $shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "http://$appIpaddress" +$shortcut.TargetPath = "http://$CName" $shortcut.IconLocation="$Env:ArcBoxIconDir\bookstore.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save() From b29426dfc8da328866db9e44a462d8aaf2b0ba04 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 22 Jul 2024 21:14:37 -0400 Subject: [PATCH 37/38] fix cname issue --- azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 | 3 +-- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 index ff603d1916..b0514181d2 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppDRScript.ps1 @@ -1,7 +1,6 @@ $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" -$CName = "dataops.jumpstart.local" -# $CName = "dataops" +$CName = "dataops" # $certdns = "$CName.jumpstart.local" $appNamespace = "arc" $sqlInstance = "aks-dr" diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index b8b1ac8e96..e77c8ddf7a 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -2,8 +2,7 @@ $Env:ArcBoxDir = "C:\ArcBox" $Env:ArcBoxLogsDir = "C:\ArcBox\Logs" $Env:ArcBoxIconDir = "C:\ArcBox\Icons" -$CName = "jumpstartbooks.jumpstart.local" -# $CName = "jumpstartbooks" +$CName = "jumpstartbooks" # $certdns = "$CName.jumpstart.local" # $password = "arcbox" $appNamespace = "arc" From 125fef319980ef3502f6e8a143105a4ef476c4ec Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Mon, 22 Jul 2024 23:26:06 -0400 Subject: [PATCH 38/38] fix app url --- azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 index e77c8ddf7a..66d6846490 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsAppScript.ps1 @@ -111,7 +111,7 @@ Do { $shortcutLocation = "$Env:Public\Desktop\Bookstore.lnk" $wScriptShell = New-Object -ComObject WScript.Shell $shortcut = $wScriptShell.CreateShortcut($shortcutLocation) -$shortcut.TargetPath = "http://$CName" +$shortcut.TargetPath = "http://$CName.jumpstart.local" $shortcut.IconLocation="$Env:ArcBoxIconDir\bookstore.ico, 0" $shortcut.WindowStyle = 3 $shortcut.Save()