AKS - Azure CNI and Calico Network Policy (#48) (#96)

Azure · Nov 30, 2021 · 52b3df1 · 52b3df1
1 parent b2f31a0
commit 52b3df1
Show file tree

Hide file tree

Showing 26 changed files with 1,055 additions and 102 deletions.
diff --git a/...rs/aks-kubenet/aks-kubenet-with-cmk.bicep → ...sources/containers/aks/aks-with-cmk.bicep b/...rs/aks-kubenet/aks-kubenet-with-cmk.bicep → ...sources/containers/aks/aks-with-cmk.bicep
@@ -13,6 +13,12 @@ param name string
 @description('Azure Kubernetes Service Version.')
 param version string
 
+@description('Azure Kubernetes Service Network Plugin; Kubenet (kubenet) | Azure CNI (azure) .')
+param networkPlugin string
+
+@description('Azure Kubernetes Service Network Policy; for Kubenet: calico | For Azure CNI: azure or calico .')
+param networkPolicy string 
+
 @description('Key/Value pair of tags.')
 param tags object = {}
 
@@ -58,18 +64,18 @@ param dnsPrefix string
 @description('Private DNS Zone Resource Id.')
 param privateDNSZoneId string
 
-// Kubernetes Networking
-@description('Pod CIDR.  Default: 11.0.0.0/16')
-param podCidr string = '11.0.0.0/16'
+// Kubernetes Networking 
+@description('Pod CIDR.')
+param podCidr string 
 
-@description('Service CIDR.  Default: 20.0.0.0/16')
-param serviceCidr string = '20.0.0.0/16'
+@description('Service CIDR.')
+param serviceCidr string
 
-@description('DNS Service IP. Default: 20.0.0.10')
-param dnsServiceIP string = '20.0.0.10'
+@description('DNS Service IP.')
+param dnsServiceIP string
 
-@description('Docker Bridge CIDR.  Default: 30.0.0.1/16')
-param dockerBridgeCidr string = '30.0.0.1/16'
+@description('Docker Bridge CIDR.')
+param dockerBridgeCidr string 
 
 // Container Insights
 @description('Log Analytics Workspace Resource Id.  Default: blank')
@@ -86,6 +92,15 @@ param akvResourceGroupName string
 @description('Azure Key Vault Name.  Required when useCMK=true.')
 param akvName string
 
+var networkProfile =  {
+  networkPlugin: networkPlugin
+  podCidr: podCidr
+  serviceCidr: serviceCidr
+  dnsServiceIP: dnsServiceIP
+  dockerBridgeCidr: dockerBridgeCidr
+  networkPolicy: networkPolicy
+}
+
 resource akv 'Microsoft.KeyVault/vaults@2021-04-01-preview' existing = {
   scope: resourceGroup(akvResourceGroupName)
   name: akvName
@@ -126,7 +141,7 @@ module diskEncryptionSetRoleAssignmentForCMK '../../iam/resource/key-vault-role-
   }
 }
 
-resource akskubenet 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
+resource aks 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
   dependsOn: [
     diskEncryptionSetRoleAssignmentForCMK
   ]
@@ -139,13 +154,7 @@ resource akskubenet 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
     kubernetesVersion: version
     dnsPrefix: dnsPrefix
     enableRBAC: true
-    networkProfile: {
-      networkPlugin: 'kubenet'
-      podCidr: podCidr
-      serviceCidr: serviceCidr
-      dnsServiceIP: dnsServiceIP
-      dockerBridgeCidr: dockerBridgeCidr
-    }
+    networkProfile: networkProfile
     agentPoolProfiles: [
       {
         count: systemNodePoolMinNodeCount

diff --git a/...aks-kubenet/aks-kubenet-without-cmk.bicep → ...rces/containers/aks/aks-without-cmk.bicep b/...aks-kubenet/aks-kubenet-without-cmk.bicep → ...rces/containers/aks/aks-without-cmk.bicep
@@ -13,6 +13,12 @@ param name string
 @description('Azure Kubernetes Service Version.')
 param version string
 
+@description('Azure Kubernetes Service Network Plugin; Kubenet (kubenet) | Azure CNI (azure) .')
+param networkPlugin string
+
+@description('Azure Kubernetes Service Network Policy; for Kubenet: calico | For Azure CNI: azure or calico .')
+param networkPolicy string 
+
 @description('Key/Value pair of tags.')
 param tags object = {}
 
@@ -58,18 +64,18 @@ param dnsPrefix string
 @description('Private DNS Zone Resource Id.')
 param privateDNSZoneId string
 
-// Kubernetes Networking
-@description('Pod CIDR.  Default: 11.0.0.0/16')
-param podCidr string = '11.0.0.0/16'
+// Kubernetes Networking 
+@description('Pod CIDR.')
+param podCidr string 
 
-@description('Service CIDR.  Default: 20.0.0.0/16')
-param serviceCidr string = '20.0.0.0/16'
+@description('Service CIDR.')
+param serviceCidr string
 
-@description('DNS Service IP. Default: 20.0.0.10')
-param dnsServiceIP string = '20.0.0.10'
+@description('DNS Service IP.')
+param dnsServiceIP string
 
-@description('Docker Bridge CIDR.  Default: 30.0.0.1/16')
-param dockerBridgeCidr string = '30.0.0.1/16'
+@description('Docker Bridge CIDR.')
+param dockerBridgeCidr string 
 
 // Container Insights
 @description('Log Analytics Workspace Resource Id.  Default: blank')
@@ -79,7 +85,17 @@ param containerInsightsLogAnalyticsResourceId string = ''
 @description('Enable encryption at host (double encryption).  Default: true')
 param enableEncryptionAtHost bool = true
 
-resource akskubenet 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
+
+var networkProfile =  {
+  networkPlugin: networkPlugin
+  podCidr: podCidr
+  serviceCidr: serviceCidr
+  dnsServiceIP: dnsServiceIP
+  dockerBridgeCidr: dockerBridgeCidr
+  networkPolicy: networkPolicy
+}
+
+resource aks 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
   name: name
   location: resourceGroup().location
   tags: tags
@@ -88,13 +104,7 @@ resource akskubenet 'Microsoft.ContainerService/managedClusters@2021-07-01' = {
     kubernetesVersion: version
     dnsPrefix: dnsPrefix
     enableRBAC: true
-    networkProfile: {
-      networkPlugin: 'kubenet'
-      podCidr: podCidr
-      serviceCidr: serviceCidr
-      dnsServiceIP: dnsServiceIP
-      dockerBridgeCidr: dockerBridgeCidr
-    }
+    networkProfile: networkProfile
     agentPoolProfiles: [
       {
         count: systemNodePoolMinNodeCount

diff --git a/...sources/containers/aks-kubenet/main.bicep → azresources/containers/aks/main.bicep b/...sources/containers/aks-kubenet/main.bicep → azresources/containers/aks/main.bicep
@@ -13,6 +13,12 @@ param name string
 @description('Azure Kubernetes Service Version.')
 param version string
 
+@description('Azure Kubernetes Service Network Plugin; Kubenet (kubenet) | Azure CNI (azure) .')
+param networkPlugin string
+
+@description('Azure Kubernetes Service Network Policy; for Kubenet: calico | For Azure CNI: azure or calico .')
+param networkPolicy string 
+
 @description('Key/Value pair of tags.')
 param tags object = {}
 
@@ -55,18 +61,18 @@ param dnsPrefix string
 @description('Private DNS Zone Resource Id.')
 param privateDNSZoneId string
 
-// Kubernetes Networking
-@description('Pod CIDR.  Default: 11.0.0.0/16')
-param podCidr string = '11.0.0.0/16'
+// Kubernetes Networking 
+@description('Pod CIDR.')
+param podCidr string 
 
-@description('Service CIDR.  Default: 20.0.0.0/16')
-param serviceCidr string = '20.0.0.0/16'
+@description('Service CIDR.')
+param serviceCidr string
 
-@description('DNS Service IP. Default: 20.0.0.10')
-param dnsServiceIP string = '20.0.0.10'
+@description('DNS Service IP.')
+param dnsServiceIP string
 
-@description('Docker Bridge CIDR.  Default: 30.0.0.1/16')
-param dockerBridgeCidr string = '30.0.0.1/16'
+@description('Docker Bridge CIDR.')
+param dockerBridgeCidr string 
 
 // Container Insights
 @description('Log Analytics Workspace Resource Id.  Default: blank')
@@ -126,7 +132,7 @@ module rbacNetworkContributor '../../iam/resource/virtual-network-role-assignmen
   }
 }
 
-module aksWithoutCMK 'aks-kubenet-without-cmk.bicep' = if (!useCMK) {
+module aksWithoutCMK 'aks-without-cmk.bicep' = if (!useCMK) {
   dependsOn: [
     rbacPrivateDnsZoneContributor
     rbacNetworkContributor
@@ -162,6 +168,9 @@ module aksWithoutCMK 'aks-kubenet-without-cmk.bicep' = if (!useCMK) {
     dnsServiceIP: dnsServiceIP
     dockerBridgeCidr: dockerBridgeCidr
 
+    networkPlugin: networkPlugin
+    networkPolicy: networkPolicy
+
     privateDNSZoneId: privateDNSZoneId
 
     containerInsightsLogAnalyticsResourceId: containerInsightsLogAnalyticsResourceId
@@ -170,7 +179,7 @@ module aksWithoutCMK 'aks-kubenet-without-cmk.bicep' = if (!useCMK) {
   }
 }
 
-module aksWithCMK 'aks-kubenet-with-cmk.bicep' = if (useCMK) {
+module aksWithCMK 'aks-with-cmk.bicep' = if (useCMK) {
   dependsOn: [
     rbacPrivateDnsZoneContributor
     rbacNetworkContributor
@@ -206,6 +215,9 @@ module aksWithCMK 'aks-kubenet-with-cmk.bicep' = if (useCMK) {
     dnsServiceIP: dnsServiceIP
     dockerBridgeCidr: dockerBridgeCidr
 
+    networkPlugin: networkPlugin
+    networkPolicy: networkPolicy
+
     privateDNSZoneId: privateDNSZoneId
 
     containerInsightsLogAnalyticsResourceId: containerInsightsLogAnalyticsResourceId

diff --git a/...dingZones/DevTest/8c6e48a4-4c73-4a1f-9f95-9447804f2c98_machinelearning_canadacentral.json b/...dingZones/DevTest/8c6e48a4-4c73-4a1f-9f95-9447804f2c98_machinelearning_canadacentral.json
@@ -82,7 +82,13 @@
     },
     "aks": {
       "value": {
-        "version": "1.21.2"
+        "version": "1.21.2",
+        "networkPlugin": "kubenet" ,
+        "networkPolicy": "calico",
+        "podCidr": "11.0.0.0/16",
+        "serviceCidr": "20.0.0.0/16" ,
+        "dnsServiceIP": "20.0.0.10",
+        "dockerBridgeCidr": "30.0.0.1/16"
       }
     },
     "sqldb": {

diff --git a/...dingZones/DevTest/ec6c5689-db04-4f1e-b76d-834a51dd0e27_machinelearning_canadacentral.json b/...dingZones/DevTest/ec6c5689-db04-4f1e-b76d-834a51dd0e27_machinelearning_canadacentral.json
@@ -89,7 +89,13 @@
     },
     "aks": {
       "value": {
-        "version": "1.21.2"
+        "version": "1.21.2",
+        "networkPlugin": "kubenet" ,
+        "networkPolicy": "calico",
+        "podCidr": "11.0.0.0/16",
+        "serviceCidr": "20.0.0.0/16" ,
+        "dnsServiceIP": "20.0.0.10",
+        "dockerBridgeCidr": "30.0.0.1/16"
       }
     },
     "sqldb": {

diff --git a/...dingZones/DevTest/f08c3057-1713-4a6f-b7e6-0df355b60c30_machinelearning_canadacentral.json b/...dingZones/DevTest/f08c3057-1713-4a6f-b7e6-0df355b60c30_machinelearning_canadacentral.json
@@ -89,7 +89,13 @@
     },
     "aks": {
       "value": {
-        "version": "1.21.2"
+        "version": "1.21.2",
+        "networkPlugin": "kubenet" ,
+        "networkPolicy": "calico",
+        "podCidr": "11.0.0.0/16",
+        "serviceCidr": "20.0.0.0/16" ,
+        "dnsServiceIP": "20.0.0.10",
+        "dockerBridgeCidr": "30.0.0.1/16"
       }
     },
     "sqldb": {

diff --git a/...dingZones/DevTest/f459218a-e8bb-49c9-b768-ee6828a144aa_machinelearning_canadacentral.json b/...dingZones/DevTest/f459218a-e8bb-49c9-b768-ee6828a144aa_machinelearning_canadacentral.json
@@ -89,7 +89,13 @@
     },
     "aks": {
       "value": {
-        "version": "1.21.2"
+        "version": "1.21.2",
+        "networkPlugin": "kubenet" ,
+        "networkPolicy": "calico",
+        "podCidr": "11.0.0.0/16",
+        "serviceCidr": "20.0.0.0/16" ,
+        "dnsServiceIP": "20.0.0.10",
+        "dockerBridgeCidr": "30.0.0.1/16"
       }
     },
     "sqldb": {

diff --git a/docs/archetypes/machinelearning.md b/docs/archetypes/machinelearning.md
@@ -2,19 +2,22 @@
 
 ## Table of Contents
 
-* [Overview](#overview)
-* [Data Flow](#data-flow)
-* [Access Control](#access-control)
-* [Networking and Security Configuration](#networking-and-security-configuration)
-* [Customer Managed Keys](#customer-managed-keys)
-* [Secrets](#secrets)
-* [Logging](#logging)
-* [Testing](#testing)
-* [Azure Deployment](#azure-deployment)
-  * [Schema Definition](#schema-definition)
-  * [Deployment Scenarios](#deployment-scenarios)
-  * [Example Deployment Parameters](#example-deployment-parameters)
-  * [Deployment Instructions](#deployment-instructions)
+- [Archetype: Machine Learning](#archetype-machine-learning)
+  - [Table of Contents](#table-of-contents)
+  - [Overview](#overview)
+  - [Data Flow](#data-flow)
+  - [Access Control](#access-control)
+  - [Networking and Security Configuration](#networking-and-security-configuration)
+  - [Customer Managed Keys](#customer-managed-keys)
+  - [Secrets](#secrets)
+  - [Logging](#logging)
+  - [Testing](#testing)
+    - [Test Scenarios](#test-scenarios)
+  - [Azure Deployment](#azure-deployment)
+    - [Schema Definition](#schema-definition)
+    - [Deployment Scenarios](#deployment-scenarios)
+    - [Example Deployment Parameters](#example-deployment-parameters)
+    - [Deployment Instructions](#deployment-instructions)
 
 ## Overview
 
@@ -61,8 +64,8 @@ Subscription can be moved to a target Management Group through Azure ARM Templat
 | Azure Machine Learning | Deploys Azure Machine Learning Service. | 
 | Azure Databricks | Deploys an Azure Databricks instance.  *There aren't any parameters for customization.* |
 | Azure Data Factory | Deploys an Azure Data Factory instance with Managed Virtual Network and Managed Integrated Runtime.  *There aren't any parameters for customization.* |
-| Azure Kubernetes Services | Deploys an AKS with Kubenet network policy that will be used for deploying machine learning models. |
-| Azure Container Registry | Deploys an Azure Container Registry to store machine learning models as container images.  ACR is used when deploying pods to AKS. *There aren't any parameters for customization.* |
+| Azure Kubernetes Services | Deploys an AKS Cluster that will be used for deploying machine learning models, with option to choose either: Network Plugin: Kubenet + Network Policy: Calico **OR** Network Plugin: Azure CNI + Network Policy: Calico (Network Policy) **OR** Network Plugin: Azure CNI + Network Policy: Azure (Network Policy) .|
+| Azure Container Registry | Deploys an Azure Container Registry to store machine learning models as container images.  ACR is used when deploying pods to AKS. *There aren't any parameters for customization. |
 | Application Insights | Deploys an Application Insights instance that is used by Azure Machine Learning instance.  *There aren't any parameters for customization.* |
 
 ## Data Flow
@@ -121,7 +124,7 @@ Once the machine learning archetype is deployed and available to use, access con
 | Azure Machine Learning | No public workspace access | Private endpoint on `amlWorkspace` + DNS registration to either hub or spoke | `privateEndpoints`|
 | Azure Storage Account for Azure ML | Network ACL deny | Private endpoint on `blob`, `file` + DNS registration to either hub or spoke | `privateEndpoints`|
 | Azure Data Factory | Public network access disabled, Azure integration runtime with managed virtual network | Private endpoint on `dataFactory` + DNS registration to either hub or spoke | `privateEndpoints`|
-| Azure Kubernetes Service | Private cluster, network profile with kubenet | N/A | `aks`|
+| Azure Kubernetes Service | Private cluster, network profile set with either kubenet or Azure CNI | N/A | `aks`|
 | Azure Container Registry | Network ACL deny, public network access disabled | Private endpoint on `registry` + DNS registration to either hub or spoke | `privateEndpoints`|
 | Azure Application Insights | N/A | N/A | N/A |
 
@@ -236,6 +239,9 @@ Reference implementation uses parameter files with `object` parameters to consol
 | Deployment with SQL DB using SQL authentication | [tests/schemas/lz-machinelearning/SQLDB-sqlAuth.json](../../tests/schemas/lz-machinelearning/SQLDB-sqlAuth.json) | `parameters.sqldb.value.aadAuthenticationOnly` is false & `parameters.sqldb.value.sqlAuthenticationUsername` filled in. |
 | Deployment with SQL DB using mixed mode authentication | [tests/schemas/lz-machinelearning/SQLDB-mixedAuth.json](../../tests/schemas/lz-machinelearning/SQLDB-mixedAuth.json) | `parameters.sqldb.value.aadAuthenticationOnly` is false,  `parameters.sqldb.value.aad*` fields filled in & `parameters.sqldb.value.sqlAuthenticationUsername` filled in. |
 | Deployment without customer managed keys | [tests/schemas/lz-machinelearning/WithoutCMK.json](../../tests/schemas/lz-machinelearning/WithoutCMK.json) | `parameters.useCMK.value` is false. |
+| Deployment with AKS using Network Plugin: Kubenet + Network Policy: Calico | [tests/schemas/lz-machinelearning/AKS-Kubenet-Calico.json](../../tests/schemas/lz-machinelearning/AKS-Kubenet-Calico.json) | `parameters.aks.value.networkPlugin`  equals ***kubenet***, `parameters.aks.value.networkPlugin`  equals ***calico***, `parameters.aks.value.podCidr` is filled, `parameters.aks.value.serviceCidr` is filled, `parameters.aks.value.dnsServiceIP` is filled and `parameters.aks.value.dockerBridgeCidr`  is filled |
+| Deployment with AKS using Network Plugin: Azure CNI + Network Policy: Calico | [tests/schemas/lz-machinelearning/AKS-AzureCNI-Calico.json](../../tests/schemas/lz-machinelearning/AKS-AzureCNI-Calico.json) | `parameters.aks.value.networkPlugin`  equals ***azure***, `parameters.aks.value.networkPlugin`  equals ***calico***, `parameters.aks.value.podCidr` is ***empty***, `parameters.aks.value.serviceCidr` is filled, `parameters.aks.value.dnsServiceIP` is filled and `parameters.aks.value.dockerBridgeCidr`  is filled |
+| Deployment with AKS using Network Plugin: Azure CNI + Network Policy: Azure | [tests/schemas/lz-machinelearning/AKS-AzureCNI-AzureNP.json](../../tests/schemas/lz-machinelearning/AKS-AzureCNI-AzureNP.json) | `parameters.aks.value.networkPlugin`  equals ***azure***, `parameters.aks.value.networkPlugin`  equals ***azure***, `parameters.aks.value.podCidr` is ***empty***, `parameters.aks.value.serviceCidr` is filled, `parameters.aks.value.dnsServiceIP` is filled and `parameters.aks.value.dockerBridgeCidr`  is filled |
 
 ### Example Deployment Parameters
 
@@ -353,7 +359,13 @@ This example configures:
     },
     "aks": {
       "value": {
-        "version": "1.21.2"
+        "version": "1.21.2",
+        "networkPlugin": "kubenet" ,
+        "networkPolicy": "calico",
+        "podCidr": "11.0.0.0/16",
+        "serviceCidr": "20.0.0.0/16" ,
+        "dnsServiceIP": "20.0.0.10",
+        "dockerBridgeCidr": "30.0.0.1/16"
       }
     },
     "sqldb": {

diff --git a/landingzones/lz-machinelearning/lz.bicep b/landingzones/lz-machinelearning/lz.bicep
@@ -342,14 +342,21 @@ module databricks '../../azresources/analytics/databricks/main.bicep' = {
   }
 }
 
-module aksKubnet '../../azresources/containers/aks-kubenet/main.bicep' = {
-  name: 'deploy-aksKubnet'
+module aksCluster '../../azresources/containers/aks/main.bicep' = {
+  name: 'deploy-aks-${aks.networkPlugin}'
   scope: rgCompute
   params: {
     tags: resourceTags
 
     name: aksName
     version: aks.version
+    networkPlugin: aks.networkPlugin
+    networkPolicy: aks.networkPolicy
+
+    dnsServiceIP: aks.dnsServiceIP
+    dockerBridgeCidr: aks.dockerBridgeCidr
+    podCidr: aks.podCidr
+    serviceCidr: aks.serviceCidr
 
     systemNodePoolEnableAutoScaling: true
     systemNodePoolMinNodeCount: 1