From 295b98e2a285d6f590a5c85382bd263b5a675a9b Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:09:09 -0400 Subject: [PATCH 1/5] update k3s clsuters name --- .../artifacts/DataOpsLogonScript.ps1 | 8 ++++---- .../artifacts/DeployAPIM.ps1 | 2 +- .../artifacts/DeploySQLMIADAuth.ps1 | 2 +- .../artifacts/DevOpsLogonScript.ps1 | 20 +++++++++---------- .../gitops_scripts/ResetBookstore.ps1 | 2 +- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 index c997589429..cc3560d07c 100644 --- a/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1 @@ -12,7 +12,7 @@ $aksArcClusterName = ($Env:aksArcClusterName).toLower() $aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower() $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" } ) @@ -107,7 +107,7 @@ Write-Host "`n" # Downloading k3s Kubernetes cluster kubeconfig file Write-Header "Downloading k3s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" -azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" +azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config" $addsDomainNetBiosName = $Env:addsDomainName.Split(".")[0] @@ -132,7 +132,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC kubectx aks="$Env:aksArcClusterName-admin" kubectx aks-dr="$Env:aksdrArcClusterName-admin" -kubectx k3s="$namingPrefix-datasvc-k3s" +kubectx k3s="$namingPrefix-k3s-datasvc" Start-Sleep -Seconds 10 @@ -333,7 +333,7 @@ Stop-Transcript # - Deploying data services on k3s cluster ################################################ -wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' +wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster' Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster" $clusters | Foreach-Object -ThrottleLimit 5 -Parallel { diff --git a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 index 45b4290e10..04d309a532 100644 --- a/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeployAPIM.ps1 @@ -28,7 +28,7 @@ az config set extension.use_dynamic_install=yes_without_prompt ################################################ # Retrive SQL Managed Instances ################################################ -kubectx arcbox-datasvc-k3s +kubectx arcbox-k3s-datasvc kubectl get nodes # Retrieving SQL MI connection endpoints diff --git a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 index a008f2d108..857dc46cdb 100644 --- a/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1 @@ -61,7 +61,7 @@ else { $sqlInstances = @( - [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-datasvc-k3s" } + [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" } [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" } diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index 5e6957f823..fee32318cc 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -18,7 +18,7 @@ $certdns = "arcbox.devops.com" $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps" $clusters = @( - [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-datasvc-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } + [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-datasvc" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" } [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "$namingPrefix-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" } ) @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-DataSvc-K3s" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-k3s-datasvc" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } @@ -48,13 +48,13 @@ Write-Header "Az CLI Login" az login --identity az account set -s $env:subscriptionId -# Downloading ArcBox-DataSvc-K3s Kubernetes cluster kubeconfig file -Write-Header "Downloading $namingPrefix-DataSvc-K3s K8s Kubeconfig" +# Downloading ArcBox-k3s-datasvc Kubernetes cluster kubeconfig file +Write-Header "Downloading $namingPrefix-k3s-datasvc K8s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading ArcBox-DataSvc-K3s log file -Write-Header "Downloading $namingPrefix-DataSvc-K3s Install Logs" +# Downloading ArcBox-k3s-datasvc log file +Write-Header "Downloading $namingPrefix-k3s-datasvc Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" @@ -71,8 +71,8 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-DataSvc-K3s and ArcBox-K3s -# Write-Header "Merging ArcBox-DataSvc-K3s & ArcBox-K3s Kubeconfigs" +# # Merging kubeconfig files from ArcBox-k3s-datasvc and ArcBox-K3s +# Write-Header "Merging ArcBox-k3s-datasvc & ArcBox-K3s Kubeconfigs" # Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" # $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" # kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -263,9 +263,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Switch Kubernetes context to ArcBox-DataSvc-K3s cluster +# Switch Kubernetes context to ArcBox-k3s-datasvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-datasvc-k3s') { + if ($cluster.context -like '*-k3s-datasvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } diff --git a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 index dc246f9f66..4295dba813 100644 --- a/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 +++ b/azure_jumpstart_arcbox/artifacts/gitops_scripts/ResetBookstore.ps1 @@ -4,7 +4,7 @@ $certdns = "arcbox.devops.com" Start-Transcript -Path $Env:ArcBoxLogsDir\ResetBookstore.log -# Switch kubectl context to arcbox-datasvc-k3s +# Switch kubectl context to arcbox-k3s-datasvc $Env:KUBECONFIG="C:\Users\$Env:adminUsername\.kube\config" kubectx diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 747ffd9288..22aea935a4 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -78,7 +78,7 @@ param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' -var k3sArcDataClusterName = '${namingPrefix}-DataSvc-K3s-${guid}' +var k3sArcDataClusterName = '${namingPrefix}-K3s-DataSvc-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster From f82b133f6bfb3a6ea378b8c0d9d4c941af23ccb9 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:16:21 -0400 Subject: [PATCH 2/5] update k3s clsuters name --- .../artifacts/DevOpsLogonScript.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index fee32318cc..cc6338bbae 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-k3s-datasvc" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-DataSvc" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" } @@ -48,13 +48,13 @@ Write-Header "Az CLI Login" az login --identity az account set -s $env:subscriptionId -# Downloading ArcBox-k3s-datasvc Kubernetes cluster kubeconfig file -Write-Header "Downloading $namingPrefix-k3s-datasvc K8s Kubeconfig" +# Downloading ArcBox-K3s-DataSvc Kubernetes cluster kubeconfig file +Write-Header "Downloading $namingPrefix-K3s-DataSvc K8s Kubeconfig" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config" azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:USERNAME\.kube\config" -# Downloading ArcBox-k3s-datasvc log file -Write-Header "Downloading $namingPrefix-k3s-datasvc Install Logs" +# Downloading ArcBox-K3s-DataSvc log file +Write-Header "Downloading $namingPrefix-K3s-DataSvc Install Logs" $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*" $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" @@ -71,8 +71,8 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E $sourceFile = $sourceFile + "?" + $sas azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "$Env:ArcBoxLogsDir\" --include-pattern "*.log" -# # Merging kubeconfig files from ArcBox-k3s-datasvc and ArcBox-K3s -# Write-Header "Merging ArcBox-k3s-datasvc & ArcBox-K3s Kubeconfigs" +# # Merging kubeconfig files from ArcBox-K3s-DataSvc and ArcBox-K3s +# Write-Header "Merging ArcBox-K3s-DataSvc & ArcBox-K3s Kubeconfigs" # Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup" # $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s" # kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp @@ -263,9 +263,9 @@ $kubeVipDaemonset | kubectl apply -f - Write-Host "`n" } -# Switch Kubernetes context to ArcBox-k3s-datasvc cluster +# Switch Kubernetes context to ArcBox-K3s-DataSvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-k3s-datasvc') { + if ($cluster.context -like '*-K3s-DataSvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } From 7ec06e1b4ddbeb9dc939c16e5d22fb1f0bfecc20 Mon Sep 17 00:00:00 2001 From: Zaid Mohammad Date: Tue, 13 Aug 2024 11:19:22 -0400 Subject: [PATCH 3/5] update k3s clsuters name --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index cc6338bbae..b31d89ea3e 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -265,7 +265,7 @@ $kubeVipDaemonset | kubectl apply -f - # Switch Kubernetes context to ArcBox-K3s-DataSvc cluster foreach ($cluster in $clusters) { - if ($cluster.context -like '*-K3s-DataSvc') { + if ($cluster.context -like '*-k3s-datasvc') { $Env:KUBECONFIG=$cluster.kubeConfig kubectx } From a95e4ef4701305b368eac21d2d85750eef47c7a6 Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 11:59:29 -0400 Subject: [PATCH 4/5] Updated main.bicep to keep naming standard consistent --- azure_jumpstart_arcbox/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/bicep/main.bicep b/azure_jumpstart_arcbox/bicep/main.bicep index 22aea935a4..d297befebf 100644 --- a/azure_jumpstart_arcbox/bicep/main.bicep +++ b/azure_jumpstart_arcbox/bicep/main.bicep @@ -78,7 +78,7 @@ param namingPrefix string = 'ArcBox' var templateBaseUrl = 'https://raw.githubusercontent.com/${githubAccount}/azure_arc/${githubBranch}/azure_jumpstart_arcbox/' var aksArcDataClusterName = '${namingPrefix}-AKS-Data-${guid}' var aksDrArcDataClusterName = '${namingPrefix}-AKS-DR-Data-${guid}' -var k3sArcDataClusterName = '${namingPrefix}-K3s-DataSvc-${guid}' +var k3sArcDataClusterName = '${namingPrefix}-K3s-Data-${guid}' var k3sArcClusterName = '${namingPrefix}-K3s-${guid}' var k3sClusterNodesCount = 3 // Number of nodes to deploy in the K3s cluster From 9831b4174eddc0a20a0b817799f208b4dd5e067f Mon Sep 17 00:00:00 2001 From: Venkata Chintala <29983008+chintalavr@users.noreply.github.com> Date: Tue, 13 Aug 2024 15:48:42 -0400 Subject: [PATCH 5/5] Update DevOpsLogonScript.ps1 --- azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 index b31d89ea3e..fd5f2e30b1 100644 --- a/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 +++ b/azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1 @@ -37,7 +37,7 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden)) $Env:AZURE_CONFIG_DIR = $cliDir.FullName -$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-DataSvc" | Where-Object { $_ -ne "" } +$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-Data" | Where-Object { $_ -ne "" } $Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n","" $Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" }