From 43bc9c85fbb9029761f6e1b3eac3944a28e68783 Mon Sep 17 00:00:00 2001 From: teralauritsen_teradata Date: Sun, 19 Nov 2023 21:08:35 -0800 Subject: [PATCH] add all-in-one for azure update jupyter config in all-in-one add persistent storage to all-in-one azure --- deployments/aws/all-in-one.yaml | 4 +- deployments/aws/jupyter.yaml | 2 +- deployments/aws/workspaces.yaml | 2 +- deployments/azure/all-in-one.json | 829 ++++++++++++++++++ deployments/azure/bicep/all-in-one.bicep | 178 ++++ deployments/azure/bicep/resources.bicep | 11 +- deployments/azure/modules/firewall.bicep | 173 +++- deployments/azure/modules/firewall.json | 88 -- deployments/azure/modules/instance.bicep | 60 +- deployments/azure/modules/network.json | 61 -- deployments/azure/policies/workspaces.json | 66 ++ deployments/azure/resources.json | 11 +- .../azure/templates/all-in-one.cloudinit.yaml | 26 + .../azure/templates/jupyter.cloudinit.yaml | 8 +- deployments/azure/templates/jupyter.service | 7 +- .../azure/templates/workspaces.cloudinit.yaml | 8 +- .../azure/templates/workspaces.service | 6 +- 17 files changed, 1303 insertions(+), 237 deletions(-) create mode 100644 deployments/azure/all-in-one.json create mode 100644 deployments/azure/bicep/all-in-one.bicep delete mode 100644 deployments/azure/modules/firewall.json delete mode 100644 deployments/azure/modules/network.json create mode 100644 deployments/azure/policies/workspaces.json create mode 100644 deployments/azure/templates/all-in-one.cloudinit.yaml diff --git a/deployments/aws/all-in-one.yaml b/deployments/aws/all-in-one.yaml index a78bb9e..c3f6d42 100644 --- a/deployments/aws/all-in-one.yaml +++ b/deployments/aws/all-in-one.yaml @@ -173,7 +173,7 @@ Parameters: MaxValue: 1000 UsePersistentVolume: - Description: size of the optional persistent disk to the workspaces server. + Description: Should we use a new or existing volume for persistent data on the workspaces/jupyter server. Type: String AllowedValues: - None @@ -191,7 +191,7 @@ Parameters: MaxValue: 1000 ExistingPersistentVolumeId: - Description: Id of the existing persistent volume to attach. Must be int the same availability zone as the workspaces instance. + Description: Id of the existing persistent volume to attach. Must be in the same availability zone as the workspaces instance. Type: String Default: None diff --git a/deployments/aws/jupyter.yaml b/deployments/aws/jupyter.yaml index 17e8e28..42c737d 100644 --- a/deployments/aws/jupyter.yaml +++ b/deployments/aws/jupyter.yaml @@ -147,7 +147,7 @@ Parameters: MaxValue: 1000 UsePersistentVolume: - Description: size of the optional persistent disk to the jupyter server. + Description: Should we use a new or existing volume for persistent data on the jupyter server. Type: String AllowedValues: - None diff --git a/deployments/aws/workspaces.yaml b/deployments/aws/workspaces.yaml index 004db85..344e450 100644 --- a/deployments/aws/workspaces.yaml +++ b/deployments/aws/workspaces.yaml @@ -146,7 +146,7 @@ Parameters: MaxValue: 1000 UsePersistentVolume: - Description: size of the optional persistent disk to the workspaces server. + Description: Should we use a new or existing volume for persistent data on the workspaces server. Type: String AllowedValues: - None diff --git a/deployments/azure/all-in-one.json b/deployments/azure/all-in-one.json new file mode 100644 index 0000000..cefb038 --- /dev/null +++ b/deployments/azure/all-in-one.json @@ -0,0 +1,829 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "4527859388330076542" + } + }, + "parameters": { + "ResourceGroupName": { + "type": "string", + "defaultValue": "ai-unlimited-workspace", + "metadata": { + "description": "name for the resource group." + } + }, + "WorkspacesName": { + "type": "string", + "metadata": { + "description": "Name for the Workspace service's virtual machine." + } + }, + "PublicKey": { + "type": "securestring", + "metadata": { + "description": "SSH public key value" + } + }, + "OSVersion": { + "type": "string", + "defaultValue": "Ubuntu-2004", + "allowedValues": [ + "Ubuntu-1804", + "Ubuntu-2004", + "Ubuntu-2204" + ], + "metadata": { + "description": "The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version." + } + }, + "InstanceType": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "The Workspace VM type" + } + }, + "Network": { + "type": "string", + "metadata": { + "description": "Name of the network to run the Workspace service in" + } + }, + "Subnet": { + "type": "string", + "metadata": { + "description": "Name of the subnet to run the Workspace service in" + } + }, + "SecurityGroup": { + "type": "string", + "defaultValue": "WorkspacesSecurityGroup", + "metadata": { + "description": "Name of the network security group" + } + }, + "AccessCIDRs": { + "type": "array", + "defaultValue": [ + "0.0.0.0/0" + ], + "metadata": { + "description": "The CIDR ranges that can be used to communicate with the Workspace service instance." + } + }, + "JupyterHttpPort": { + "type": "string", + "defaultValue": "8888", + "metadata": { + "description": "port to access the Jupyter Labs UI." + } + }, + "WorkspacesHttpPort": { + "type": "string", + "defaultValue": "3000", + "metadata": { + "description": "port to access the workspaces service UI." + } + }, + "WorkspacesGrpcPort": { + "type": "string", + "defaultValue": "3282", + "metadata": { + "description": "port to access the workspaces service api." + } + }, + "SourceAppSecGroups": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Source Application Security Groups to access the workspaces service api." + } + }, + "detinationAppSecGroups": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Destination Application Security Groups to give access to workspaces service instance." + } + }, + "RoleDefinitionId": { + "type": "string", + "metadata": { + "description": "GUID of the Workspaces Role" + } + }, + "AllowPublicSSH": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "allow access the workspaces ssh port from the access cidr." + } + }, + "UsePersistentVolume": { + "type": "string", + "defaultValue": "New", + "allowedValues": [ + "New", + "None", + "Existing" + ], + "metadata": { + "description": "should we use a new or existing volume for persistent data on the workspace server." + } + }, + "PersistentVolumeSize": { + "type": "int", + "defaultValue": 100, + "metadata": { + "description": "size of the optional persistent disk to the workspace server." + } + }, + "ExistingPersistentVolume": { + "type": "string", + "defaultValue": "NONE", + "metadata": { + "description": "Name of the existing persistent volume to attach. Must be in the same region and resourcegroup zone as the workspaces server." + } + }, + "WorkspacesVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "Container Version of the Workspace service" + } + }, + "JupyterVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "Container Version of the Jupyter Labs service" + } + }, + "JupyterToken": { + "type": "string", + "defaultValue": "[uniqueString(subscription().id, utcNow())]", + "metadata": { + "description": "Join token for the Jupyter Labs service" + } + } + }, + "variables": { + "$fxv#0": "#cloud-config\nwrite_files:\n- encoding: b64\n content: \"{0}\"\n owner: root:root\n path: /usr/lib/systemd/system/workspaces.service\n permissions: '0640'\n- encoding: b64\n content: \"{1}\"\n owner: root:root\n path: /usr/lib/systemd/system/jupyter.service\n permissions: '0640'\n\nruncmd:\n- mkdir -p /etc/td \n- |\n export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}');\n if [ -n \"${{PERMDISK}}\" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi\n /usr/bin/echo \"LABEL=WORKSPACES /etc/td ext4 defaults 0 2\" >> /etc/fstab\n /usr/bin/mount -a\n- while [ $(systemctl status docker | grep \"active (running)\" | wc -l) -lt 1 ]; do sleep 5; done\n- sleep 60\n- systemctl enable workspace.service\n- systemctl start workspaces.service\n- systemctl enable jupyter.service\n- systemctl start jupyter.service\n", + "$fxv#1": "[Unit]\nDescription=workspaces\nAfter=docker.service\nRequires=docker.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited\nExecStartPre=-/usr/bin/mkdir -p /etc/td/workspace\nExecStartPre=-/usr/bin/docker exec %n stop || true\nExecStartPre=-/usr/bin/docker rm %n || true\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e accept_license=Y \\\n -e PLATFORM=azure \\\n -e ARM_USE_MSI=true \\\n -e ARM_SUBSCRIPTION_ID={5} \\\n -e ARM_TENANT_ID={6} \\\n -v /etc/td/workspace:/etc/td \\\n -p {3}:3000 \\\n -p {4}:3282 \\\n --network ai_unlimited \\\n --rm --name %n {0}/{1}:{2} workspaces serve -v\n\n[Install]\nWantedBy=multi-user.target", + "$fxv#2": "[Unit]\nDescription=jupyter\nAfter=docker.service\nRequires=docker.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited\nExecStartPre=-/usr/bin/mkdir -p /etc/td/jupyter/{{userdata,ipython}}\nExecStartPre=-/usr/bin/docker exec %n stop || true\nExecStartPre=-/usr/bin/docker rm %n || true\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e accept_license=Y \\\n -e JUPYTER_TOKEN={4} \\\n -v /etc/td/jupyter/userdata:/home/jovyan/JupyterLabRoot/userdata \\\n -v /etc/td/jupyter/ipython:/home/jovyan/.ipython \\\n -p {3}:8888 \\\n --network ai_unlimited \\\n --rm --name %n {0}/{1}:{2}\n\n[Install]\nWantedBy=multi-user.target\n", + "roleAssignmentName": "[guid(subscription().id, parameters('WorkspacesName'), subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), parameters('RoleDefinitionId'))]", + "registry": "teradata", + "workspaceRepository": "ai-unlimited-workspaces", + "jupyterRepository": "ai-unlimited-jupyter", + "cloudInitData": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('WorkspacesVersion'), parameters('WorkspacesHttpPort'), parameters('WorkspacesGrpcPort'), subscription().subscriptionId, subscription().tenantId)), base64(format(variables('$fxv#2'), variables('registry'), variables('jupyterRepository'), parameters('JupyterVersion'), parameters('JupyterHttpPort'), parameters('JupyterToken')))))]" + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('roleAssignmentName')]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', parameters('RoleDefinitionId'))]", + "principalId": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PrincipleId.value]" + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "firewall", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "name": { + "value": "[parameters('SecurityGroup')]" + }, + "accessCidrs": { + "value": "[parameters('AccessCIDRs')]" + }, + "sshAccess": { + "value": "[parameters('AllowPublicSSH')]" + }, + "workspacesHttpPort": { + "value": "[parameters('WorkspacesHttpPort')]" + }, + "workspacesGrpcPort": { + "value": "[parameters('WorkspacesGrpcPort')]" + }, + "jupyterHttpPort": { + "value": "[parameters('JupyterHttpPort')]" + }, + "sourceAppSecGroups": { + "value": "[parameters('SourceAppSecGroups')]" + }, + "detinationAppSecGroups": { + "value": "[parameters('detinationAppSecGroups')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "16212577407509280702" + } + }, + "parameters": { + "location": { + "type": "string" + }, + "name": { + "type": "string" + }, + "accessCidrs": { + "type": "array", + "defaultValue": [] + }, + "sourceAppSecGroups": { + "type": "array", + "defaultValue": [] + }, + "detinationAppSecGroups": { + "type": "array", + "defaultValue": [] + }, + "sshAccess": { + "type": "bool", + "defaultValue": false + }, + "workspacesHttpPort": { + "type": "string", + "defaultValue": "None" + }, + "workspacesGrpcPort": { + "type": "string", + "defaultValue": "None" + }, + "jupyterHttpPort": { + "type": "string", + "defaultValue": "None" + } + }, + "resources": [ + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2022-11-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]" + }, + { + "condition": "[parameters('sshAccess')]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', parameters('name'), format('{0}-ssh-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow ssh to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 700, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + ] + }, + { + "condition": "[not(parameters('sshAccess'))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', parameters('name'), format('{0}-ssh-deny', parameters('name')))]", + "properties": { + "access": "[if(parameters('sshAccess'), 'Allow', 'Deny')]", + "description": "deny ssh to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 700, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + ] + }, + { + "condition": "[not(equals(parameters('workspacesHttpPort'), 'None'))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', parameters('name'), format('{0}-workspace-http-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[parameters('workspacesHttpPort')]", + "direction": "Inbound", + "priority": 701, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + ] + }, + { + "condition": "[not(equals(parameters('workspacesGrpcPort'), 'None'))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', parameters('name'), format('{0}-workspace-grpc-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow grpc to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[parameters('workspacesGrpcPort')]", + "direction": "Inbound", + "priority": 702, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + ] + }, + { + "condition": "[not(equals(parameters('jupyterHttpPort'), 'None'))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', parameters('name'), format('{0}-juptyer-http-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the jupyter instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[parameters('jupyterHttpPort')]", + "direction": "Inbound", + "priority": 703, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + ] + } + ], + "outputs": { + "Id": { + "type": "string", + "value": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" + } + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "workspaces", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "name": { + "value": "[parameters('WorkspacesName')]" + }, + "adminUsername": { + "value": "azureuser" + }, + "sshPublicKey": { + "value": "[parameters('PublicKey')]" + }, + "dnsLabelPrefix": { + "value": "[uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), deployment().name, parameters('WorkspacesName'))]" + }, + "vmSize": { + "value": "[parameters('InstanceType')]" + }, + "subnetId": { + "value": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Network/virtualNetworks/subnets', parameters('Network'), parameters('Subnet'))]" + }, + "networkSecurityGroupID": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall'), '2022-09-01').outputs.Id.value]" + }, + "osVersion": { + "value": "[parameters('OSVersion')]" + }, + "cloudInitData": { + "value": "[variables('cloudInitData')]" + }, + "usePersistentVolume": { + "value": "[parameters('UsePersistentVolume')]" + }, + "persistentVolumeSize": { + "value": "[parameters('PersistentVolumeSize')]" + }, + "existingPersistentVolume": { + "value": "[parameters('ExistingPersistentVolume')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "16549388062424742652" + } + }, + "parameters": { + "location": { + "type": "string" + }, + "name": { + "type": "string" + }, + "adminUsername": { + "type": "string" + }, + "sshPublicKey": { + "type": "string" + }, + "dnsLabelPrefix": { + "type": "string", + "defaultValue": "[toLower(format('$td{0}', uniqueString('ai unlimited', resourceGroup().id)))]" + }, + "vmSize": { + "type": "string" + }, + "subnetId": { + "type": "string" + }, + "networkSecurityGroupID": { + "type": "string" + }, + "osVersion": { + "type": "string" + }, + "usePersistentVolume": { + "type": "string" + }, + "persistentVolumeSize": { + "type": "int" + }, + "existingPersistentVolume": { + "type": "string" + }, + "cloudInitData": { + "type": "string" + } + }, + "variables": { + "imageReference": { + "Ubuntu-1804": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "18_04-lts-gen2", + "version": "latest" + }, + "Ubuntu-2004": { + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-focal", + "sku": "20_04-lts-gen2", + "version": "latest" + }, + "Ubuntu-2204": { + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "publicIPAddressName": "[format('{0}PublicIP', parameters('name'))]", + "networkInterfaceName": "[format('{0}NetInt', parameters('name'))]", + "osDiskType": "Standard_LRS", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[format('/home/{0}/.ssh/authorized_keys', parameters('adminUsername'))]", + "keyData": "[parameters('sshPublicKey')]" + } + ] + } + }, + "trustedExtensionName": "GuestAttestation", + "trustedExtensionPublisher": "Microsoft.Azure.Security.LinuxAttestation", + "trustedExtensionVersion": "1.0", + "trustedMaaTenantName": "GuestAttestation", + "trustedMaaEndpoint": "[substring('emptystring', 0, 0)]", + "dockerExtensionName": "DockerExtension", + "dockerExtensionPublisher": "Microsoft.Azure.Extensions", + "dockerExtensionVersion": "1.1" + }, + "resources": [ + { + "condition": "[equals(parameters('usePersistentVolume'), 'New')]", + "type": "Microsoft.Compute/disks", + "apiVersion": "2023-04-02", + "name": "[format('{0}-disk', parameters('name'))]", + "location": "[parameters('location')]", + "properties": { + "creationData": { + "createOption": "Empty" + }, + "diskSizeGB": "[parameters('persistentVolumeSize')]", + "maxShares": 1, + "osType": "Linux" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2022-11-01", + "name": "[variables('networkInterfaceName')]", + "location": "[parameters('location')]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[parameters('subnetId')]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[parameters('networkSecurityGroupID')]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]" + ] + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2022-11-01", + "name": "[variables('publicIPAddressName')]", + "location": "[parameters('location')]", + "sku": { + "name": "Basic" + }, + "properties": { + "publicIPAllocationMethod": "Static", + "publicIPAddressVersion": "IPv4", + "dnsSettings": { + "domainNameLabel": "[parameters('dnsLabelPrefix')]" + }, + "idleTimeoutInMinutes": 4 + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-03-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "dataDisks": "[if(not(equals(parameters('usePersistentVolume'), 'None')), createArray(), createArray(createObject('lun', 0, 'createOption', 'Attach', 'managedDisk', createObject('id', if(equals(parameters('usePersistentVolume'), 'New'), resourceId('Microsoft.Compute/disks', format('{0}-disk', parameters('name'))), resourceId('Microsoft.Compute/disks', parameters('existingPersistentVolume')))))))]", + "imageReference": "[variables('imageReference')[parameters('osVersion')]]" + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[parameters('name')]", + "adminUsername": "[parameters('adminUsername')]", + "linuxConfiguration": "[variables('linuxConfiguration')]" + }, + "securityProfile": { + "securityType": "TrustedLaunch", + "uefiSettings": { + "secureBootEnabled": true, + "vTpmEnabled": true + } + }, + "userData": "[parameters('cloudInitData')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]", + "[resourceId('Microsoft.Compute/disks', format('{0}-disk', parameters('name')))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-03-01", + "name": "[format('{0}/{1}', parameters('name'), variables('trustedExtensionName'))]", + "location": "[parameters('location')]", + "properties": { + "publisher": "[variables('trustedExtensionPublisher')]", + "type": "[variables('trustedExtensionName')]", + "typeHandlerVersion": "[variables('trustedExtensionVersion')]", + "autoUpgradeMinorVersion": true, + "settings": { + "AttestationConfig": { + "MaaSettings": { + "maaEndpoint": "[variables('trustedMaaEndpoint')]", + "maaTenantName": "[variables('trustedMaaTenantName')]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('name'))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-03-01", + "name": "[format('{0}/{1}', parameters('name'), variables('dockerExtensionName'))]", + "location": "[parameters('location')]", + "properties": { + "publisher": "[variables('dockerExtensionPublisher')]", + "type": "[variables('dockerExtensionName')]", + "typeHandlerVersion": "[variables('dockerExtensionVersion')]", + "autoUpgradeMinorVersion": true + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('name'))]" + ] + } + ], + "outputs": { + "PublicIP": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName')), '2022-11-01').ipAddress]" + }, + "PrivateIP": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName')), '2022-11-01').ipConfigurations[0].properties.privateIPAddress]" + }, + "PrincipleId": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('name')), '2023-03-01', 'full').identity.principalId]" + } + } + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall')]" + ] + } + ], + "outputs": { + "PublicIP": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PublicIP.value]" + }, + "PrivateIP": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PrivateIP.value]" + }, + "WorkspacesPublicHttpAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PublicIP.value, parameters('WorkspacesHttpPort'))]" + }, + "WorkspacesPrivateHttpAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PrivateIP.value, parameters('WorkspacesHttpPort'))]" + }, + "WorkspacesPublicGrpcAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PublicIP.value, parameters('WorkspacesGrpcPort'))]" + }, + "WorkspacesPrivateGrpcAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PrivateIP.value, parameters('WorkspacesGrpcPort'))]" + }, + "JupyterLabPublicHttpAccess": { + "type": "string", + "value": "[format('http://{0}:{1}?token={2}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PublicIP.value, parameters('JupyterHttpPort'), parameters('JupyterToken'))]" + }, + "JupyterLabPrivateHttpAccess": { + "type": "string", + "value": "[format('http://{0}:{1}?token={2}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PrivateIP.value, parameters('JupyterHttpPort'), parameters('JupyterToken'))]" + }, + "sshCommand": { + "type": "string", + "value": "[format('ssh azureuser@{0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'workspaces'), '2022-09-01').outputs.PublicIP.value)]" + }, + "SecurityGroup": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall'), '2022-09-01').outputs.Id.value]" + } + } +} \ No newline at end of file diff --git a/deployments/azure/bicep/all-in-one.bicep b/deployments/azure/bicep/all-in-one.bicep new file mode 100644 index 0000000..e2cf84b --- /dev/null +++ b/deployments/azure/bicep/all-in-one.bicep @@ -0,0 +1,178 @@ +targetScope = 'subscription' + +@description('name for the resource group.') +param ResourceGroupName string = 'ai-unlimited-workspace' + +@description('Name for the Workspace service\'s virtual machine.') +param WorkspacesName string + +@description('SSH public key value') +@secure() +param PublicKey string + +@description('The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version.') +@allowed([ + 'Ubuntu-1804' + 'Ubuntu-2004' + 'Ubuntu-2204' +]) +param OSVersion string = 'Ubuntu-2004' + +@description('The Workspace VM type') +param InstanceType string = 'Standard_D2s_v3' + +@description('Name of the network to run the Workspace service in') +param Network string + +@description('Name of the subnet to run the Workspace service in') +param Subnet string + +@description('Name of the network security group') +param SecurityGroup string = 'WorkspacesSecurityGroup' + +@description('The CIDR ranges that can be used to communicate with the Workspace service instance.') +param AccessCIDRs array = [ '0.0.0.0/0' ] + +@description('port to access the Jupyter Labs UI.') +param JupyterHttpPort string = '8888' + +@description('port to access the workspaces service UI.') +param WorkspacesHttpPort string = '3000' + +@description('port to access the workspaces service api.') +param WorkspacesGrpcPort string = '3282' + +@description('Source Application Security Groups to access the workspaces service api.') +param SourceAppSecGroups array = [] + +@description('Destination Application Security Groups to give access to workspaces service instance.') +param detinationAppSecGroups array = [] + +@description('GUID of the Workspaces Role') +param RoleDefinitionId string + +@description('allow access the workspaces ssh port from the access cidr.') +param AllowPublicSSH bool = true + +@description('should we use a new or existing volume for persistent data on the workspace server.') +@allowed([ 'New', 'None', 'Existing' ]) +param UsePersistentVolume string = 'New' + +@description('size of the optional persistent disk to the workspace server.') +param PersistentVolumeSize int = 100 + +@description('Name of the existing persistent volume to attach. Must be in the same region and resourcegroup zone as the workspaces server.') +param ExistingPersistentVolume string = 'NONE' + +@description('Container Version of the Workspace service') +param WorkspacesVersion string = 'latest' + +@description('Container Version of the Jupyter Labs service') +param JupyterVersion string = 'latest' + +@description('Join token for the Jupyter Labs service') +param JupyterToken string = uniqueString(subscription().id, utcNow()) + +var roleAssignmentName = guid(subscription().id, WorkspacesName, rg.id, RoleDefinitionId) + +var registry = 'teradata' +var workspaceRepository = 'ai-unlimited-workspaces' +var jupyterRepository = 'ai-unlimited-jupyter' + +var cloudInitData = base64( + format( + loadTextContent('../templates/all-in-one.cloudinit.yaml'), + base64( + format( + loadTextContent('../templates/workspaces.service'), + registry, + workspaceRepository, + WorkspacesVersion, + WorkspacesHttpPort, + WorkspacesGrpcPort, + subscription().subscriptionId, + subscription().tenantId + ) + ), + base64( + format( + loadTextContent('../templates/jupyter.service'), + registry, + jupyterRepository, + JupyterVersion, + JupyterHttpPort, + JupyterToken + ) + ) + ) +) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' existing = { + name: ResourceGroupName +} + +resource network 'Microsoft.Network/virtualNetworks@2022-11-01' existing = { + scope: rg + name: Network +} + +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2022-11-01' existing = { + parent: network + name: Subnet +} + +module firewall '../modules/firewall.bicep' = { + scope: rg + name: 'firewall' + params: { + location: rg.location + name: SecurityGroup + accessCidrs: AccessCIDRs + sshAccess: AllowPublicSSH + workspacesHttpPort: WorkspacesHttpPort + workspacesGrpcPort: WorkspacesGrpcPort + jupyterHttpPort: JupyterHttpPort + sourceAppSecGroups: SourceAppSecGroups + detinationAppSecGroups: detinationAppSecGroups + } +} + +module workspaces '../modules/instance.bicep' = { + scope: rg + name: 'workspaces' + params: { + location: rg.location + name: WorkspacesName + adminUsername: 'azureuser' + sshPublicKey: PublicKey + dnsLabelPrefix: uniqueString(rg.id, deployment().name, WorkspacesName) + vmSize: InstanceType + subnetId: subnet.id + networkSecurityGroupID: firewall.outputs.Id + osVersion: OSVersion + cloudInitData: cloudInitData + usePersistentVolume: UsePersistentVolume + persistentVolumeSize: PersistentVolumeSize + existingPersistentVolume: ExistingPersistentVolume + } +} + +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: subscription() + name: roleAssignmentName + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', RoleDefinitionId) + principalId: workspaces.outputs.PrincipleId + } +} + +output PublicIP string = workspaces.outputs.PublicIP +output PrivateIP string = workspaces.outputs.PrivateIP +output WorkspacesPublicHttpAccess string = 'http://${workspaces.outputs.PublicIP}:${WorkspacesHttpPort}' +output WorkspacesPrivateHttpAccess string = 'http://${workspaces.outputs.PrivateIP}:${WorkspacesHttpPort}' +output WorkspacesPublicGrpcAccess string = 'http://${workspaces.outputs.PublicIP}:${WorkspacesGrpcPort}' +output WorkspacesPrivateGrpcAccess string = 'http://${workspaces.outputs.PrivateIP}:${WorkspacesGrpcPort}' +output JupyterLabPublicHttpAccess string = 'http://${workspaces.outputs.PublicIP}:${JupyterHttpPort}?token=${JupyterToken}' +output JupyterLabPrivateHttpAccess string = 'http://${workspaces.outputs.PrivateIP}:${JupyterHttpPort}?token=${JupyterToken}' +output sshCommand string = 'ssh azureuser@${workspaces.outputs.PublicIP}' +output SecurityGroup string = firewall.outputs.Id diff --git a/deployments/azure/bicep/resources.bicep b/deployments/azure/bicep/resources.bicep index 41b5cdc..ec43161 100644 --- a/deployments/azure/bicep/resources.bicep +++ b/deployments/azure/bicep/resources.bicep @@ -4,11 +4,11 @@ targetScope = 'subscription' param name string = 'workspaces' @description('...') -@allowed(['West US']) +@allowed([ 'West US' ]) param location string = 'West US' @description('New network CIDR.') -param networkCidr array = ['10.0.0.0/16'] +param networkCidr array = [ '10.0.0.0/16' ] @description('New subnet CIDR.') param subnetCidr string = '10.0.0.0/24' @@ -18,7 +18,7 @@ resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { location: location } -module network './modules/network.bicep' = { +module network '../modules/network.bicep' = { scope: rg name: 'networkDeployment' params: { @@ -30,7 +30,7 @@ module network './modules/network.bicep' = { } resource roleDef 'Microsoft.Authorization/roleDefinitions@2022-04-01' = { - name: guid(subscription().id, rg.id) + name: guid(subscription().id, rg.id) properties: { roleName: 'Custom Role - Workspaces ${name} Regulus Deployment Permissions' description: 'Subscription level permissions for workspaces to create regulus deployments in there own resource groups' @@ -58,6 +58,9 @@ resource roleDef 'Microsoft.Authorization/roleDefinitions@2022-04-01' = { 'Microsoft.ManagedIdentity/userAssignedIdentities/listAssociatedResources/action' 'Microsoft.ManagedIdentity/userAssignedIdentities/read' 'Microsoft.ManagedIdentity/userAssignedIdentities/write' + 'Microsoft.Network/applicationSecurityGroups/read' + 'Microsoft.Network/applicationSecurityGroups/write' + 'Microsoft.Network/applicationSecurityGroups/joinIpConfiguration/action' 'Microsoft.Network/virtualNetworks/read' 'Microsoft.Network/virtualNetworks/write' 'Microsoft.Network/virtualNetworks/delete' diff --git a/deployments/azure/modules/firewall.bicep b/deployments/azure/modules/firewall.bicep index 9ff20bc..df0d1ec 100644 --- a/deployments/azure/modules/firewall.bicep +++ b/deployments/azure/modules/firewall.bicep @@ -1,55 +1,140 @@ param location string param name string -param accessCidrs array -param sshAccess bool -param httpPort string -param grpcPort string +param accessCidrs array = [] +param sourceAppSecGroups array = [] +param detinationAppSecGroups array = [] +param sshAccess bool = false +param workspacesHttpPort string = 'None' +param workspacesGrpcPort string = 'None' +param jupyterHttpPort string = 'None' resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2022-11-01' = { name: name location: location +} + +resource sshAllow 'Microsoft.Network/networkSecurityGroups/securityRules@2023-04-01' = if (sshAccess) { + name: '${name}-ssh-allow' + parent: networkSecurityGroup + + properties: { + access: 'Allow' + description: 'allow ssh to the workspace instance' + destinationAddressPrefix: '*' // destinationAddressPrefixes: [] + destinationApplicationSecurityGroups: [for secgroup in detinationAppSecGroups: { + id: secgroup + location: location + } + ] + destinationPortRange: '22' // destinationPortRanges: [] + direction: 'Inbound' + priority: 700 + protocol: 'Tcp' + sourceAddressPrefixes: accessCidrs // sourceAddressPrefix: 'string' + sourceApplicationSecurityGroups: [for secgroup in sourceAppSecGroups: { + id: secgroup + location: location + } + ] + sourcePortRange: '*' // sourcePortRanges: [] + } +} + +resource sshDeny 'Microsoft.Network/networkSecurityGroups/securityRules@2023-04-01' = if (!sshAccess) { + name: '${name}-ssh-deny' + parent: networkSecurityGroup + + properties: { + access: sshAccess ? 'Allow' : 'Deny' + description: 'deny ssh to the workspace instance' + destinationAddressPrefix: '*' + destinationPortRange: '22' + direction: 'Inbound' + priority: 700 + protocol: 'Tcp' + sourceAddressPrefix: '*' + sourcePortRange: '*' + } +} + +resource WorkspacesHTTP 'Microsoft.Network/networkSecurityGroups/securityRules@2023-04-01' = if (workspacesHttpPort != 'None') { + name: '${name}-workspace-http-allow' + parent: networkSecurityGroup + properties: { - securityRules: [ - { - name: 'SSH' - properties: { - priority: 700 - protocol: 'Tcp' - access: sshAccess ? 'Allow' : 'Deny' - direction: 'Inbound' - sourceAddressPrefixes: accessCidrs - sourcePortRange: '*' - destinationAddressPrefix: '*' - destinationPortRange: '22' - } - } - { - name: 'HTTP' - properties: { - priority: 701 - protocol: 'Tcp' - access: 'Allow' - direction: 'Inbound' - sourceAddressPrefixes: accessCidrs - sourcePortRange: '*' - destinationAddressPrefix: '*' - destinationPortRange: httpPort - } - } - { - name: 'GRPC' - properties: { - priority: 702 - protocol: 'Tcp' - access: 'Allow' - direction: 'Inbound' - sourceAddressPrefixes: accessCidrs - sourcePortRange: '*' - destinationAddressPrefix: '*' - destinationPortRange: grpcPort - } - } + access: 'Allow' + description: 'allow http to the workspace instance' + destinationAddressPrefix: '*' // destinationAddressPrefixes: [] + destinationApplicationSecurityGroups: [for secgroup in detinationAppSecGroups: { + id: secgroup + location: location + } + ] + destinationPortRange: workspacesHttpPort // destinationPortRanges: [] + direction: 'Inbound' + priority: 701 + protocol: 'Tcp' + sourceAddressPrefixes: accessCidrs // sourceAddressPrefix: 'string' + sourceApplicationSecurityGroups: [for secgroup in sourceAppSecGroups: { + id: secgroup + location: location + } + ] + sourcePortRange: '*' // sourcePortRanges: [] + } +} + +resource WorkspacesGRPC 'Microsoft.Network/networkSecurityGroups/securityRules@2023-04-01' = if (workspacesGrpcPort != 'None') { + name: '${name}-workspace-grpc-allow' + parent: networkSecurityGroup + + properties: { + access: 'Allow' + description: 'allow grpc to the workspace instance' + destinationAddressPrefix: '*' // destinationAddressPrefixes: [] + destinationApplicationSecurityGroups: [for secgroup in detinationAppSecGroups: { + id: secgroup + location: location + } + ] + destinationPortRange: workspacesGrpcPort // destinationPortRanges: [] + direction: 'Inbound' + priority: 702 + protocol: 'Tcp' + sourceAddressPrefixes: accessCidrs // sourceAddressPrefix: 'string' + sourceApplicationSecurityGroups: [for secgroup in sourceAppSecGroups: { + id: secgroup + location: location + } + ] + sourcePortRange: '*' // sourcePortRanges: [] + } +} + +resource JupyterHTTP 'Microsoft.Network/networkSecurityGroups/securityRules@2023-04-01' = if (jupyterHttpPort != 'None') { + name: '${name}-juptyer-http-allow' + parent: networkSecurityGroup + + properties: { + access: 'Allow' + description: 'allow http to the jupyter instance' + destinationAddressPrefix: '*' // destinationAddressPrefixes: [] + destinationApplicationSecurityGroups: [for secgroup in detinationAppSecGroups: { + id: secgroup + location: location + } + ] + destinationPortRange: jupyterHttpPort // destinationPortRanges: [] + direction: 'Inbound' + priority: 703 + protocol: 'Tcp' + sourceAddressPrefixes: accessCidrs // sourceAddressPrefix: 'string' + sourceApplicationSecurityGroups: [for secgroup in sourceAppSecGroups: { + id: secgroup + location: location + } ] + sourcePortRange: '*' // sourcePortRanges: [] } } diff --git a/deployments/azure/modules/firewall.json b/deployments/azure/modules/firewall.json deleted file mode 100644 index d1e2d10..0000000 --- a/deployments/azure/modules/firewall.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.18.4.5664", - "templateHash": "11608979749384228954" - } - }, - "parameters": { - "location": { - "type": "string" - }, - "name": { - "type": "string" - }, - "accessCidrs": { - "type": "array" - }, - "sshAccess": { - "type": "bool" - }, - "httpPort": { - "type": "string" - }, - "grpcPort": { - "type": "string" - } - }, - "resources": [ - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2022-11-01", - "name": "[parameters('name')]", - "location": "[parameters('location')]", - "properties": { - "securityRules": [ - { - "name": "SSH", - "properties": { - "priority": 700, - "protocol": "Tcp", - "access": "[if(parameters('sshAccess'), 'Allow', 'Deny')]", - "direction": "Inbound", - "sourceAddressPrefixes": "[parameters('accessCidrs')]", - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "22" - } - }, - { - "name": "HTTP", - "properties": { - "priority": 701, - "protocol": "Tcp", - "access": "Allow", - "direction": "Inbound", - "sourceAddressPrefixes": "[parameters('accessCidrs')]", - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "[parameters('httpPort')]" - } - }, - { - "name": "GRPC", - "properties": { - "priority": 702, - "protocol": "Tcp", - "access": "Allow", - "direction": "Inbound", - "sourceAddressPrefixes": "[parameters('accessCidrs')]", - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "[parameters('grpcPort')]" - } - } - ] - } - } - ], - "outputs": { - "Id": { - "type": "string", - "value": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('name'))]" - } - } -} \ No newline at end of file diff --git a/deployments/azure/modules/instance.bicep b/deployments/azure/modules/instance.bicep index ec31b97..e97f8d2 100644 --- a/deployments/azure/modules/instance.bicep +++ b/deployments/azure/modules/instance.bicep @@ -2,13 +2,15 @@ param location string param name string param adminUsername string param sshPublicKey string -param dnsLabelPrefix string = toLower('${name}-${uniqueString(resourceGroup().id)}') +param dnsLabelPrefix string = toLower('$td${uniqueString('ai unlimited', resourceGroup().id)}') param vmSize string param subnetId string param networkSecurityGroupID string -param httpPort string -param grpcPort string -param ubuntuOSVersion string +param osVersion string +param usePersistentVolume string +param persistentVolumeSize int +param existingPersistentVolume string +param cloudInitData string var imageReference = { 'Ubuntu-1804': { @@ -55,26 +57,23 @@ var dockerExtensionName = 'DockerExtension' var dockerExtensionPublisher = 'Microsoft.Azure.Extensions' var dockerExtensionVersion = '1.1' -var registry = 'teradata' -var repository = 'regulus-workspaces' -var version = 'devtest' //'latest' -var cloudInitData = base64( - format( - loadTextContent('../templates/workspaces.cloudinit.yaml'), - base64( - format( - loadTextContent('../templates/workspaces.service'), - registry, - repository, - version, - httpPort, - grpcPort, - subscription().subscriptionId, - subscription().tenantId - ) - ) - ) -) +resource existingPersistentDisk 'Microsoft.Compute/disks@2023-04-02' existing = if (usePersistentVolume == 'Existing') { + name: existingPersistentVolume +} + +resource newPersistentDisk 'Microsoft.Compute/disks@2023-04-02' = if (usePersistentVolume == 'New') { + location: location + name: '${name}-disk' + + properties: { + creationData: { + createOption: 'Empty' + } + diskSizeGB: persistentVolumeSize + maxShares: 1 + osType: 'Linux' + } +} resource networkInterface 'Microsoft.Network/networkInterfaces@2022-11-01' = { name: networkInterfaceName @@ -133,7 +132,16 @@ resource vm 'Microsoft.Compute/virtualMachines@2023-03-01' = { storageAccountType: osDiskType } } - imageReference: imageReference[ubuntuOSVersion] + dataDisks: usePersistentVolume != 'None' ? [] : [ + { + lun: 0 + createOption: 'Attach' + managedDisk: { + id: usePersistentVolume == 'New' ? newPersistentDisk.id : existingPersistentDisk.id + } + } + ] + imageReference: imageReference[osVersion] } networkProfile: { networkInterfaces: [ @@ -142,7 +150,7 @@ resource vm 'Microsoft.Compute/virtualMachines@2023-03-01' = { } ] } - + osProfile: { computerName: name adminUsername: adminUsername diff --git a/deployments/azure/modules/network.json b/deployments/azure/modules/network.json deleted file mode 100644 index 6774171..0000000 --- a/deployments/azure/modules/network.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.18.4.5664", - "templateHash": "5866519696252789244" - } - }, - "parameters": { - "networkName": { - "type": "string" - }, - "networkCidr": { - "type": "array" - }, - "subnetCidr": { - "type": "string" - }, - "location": { - "type": "string" - } - }, - "resources": [ - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2022-11-01", - "name": "[parameters('networkName')]", - "location": "[parameters('location')]", - "properties": { - "addressSpace": { - "addressPrefixes": "[parameters('networkCidr')]" - } - } - }, - { - "type": "Microsoft.Network/virtualNetworks/subnets", - "apiVersion": "2022-11-01", - "name": "[format('{0}/{1}', parameters('networkName'), parameters('networkName'))]", - "properties": { - "addressPrefix": "[parameters('subnetCidr')]", - "privateEndpointNetworkPolicies": "Enabled", - "privateLinkServiceNetworkPolicies": "Enabled" - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', parameters('networkName'))]" - ] - } - ], - "outputs": { - "subnetName": { - "type": "string", - "value": "[parameters('networkName')]" - }, - "networkName": { - "type": "string", - "value": "[parameters('networkName')]" - } - } -} \ No newline at end of file diff --git a/deployments/azure/policies/workspaces.json b/deployments/azure/policies/workspaces.json new file mode 100644 index 0000000..fb4f6ba --- /dev/null +++ b/deployments/azure/policies/workspaces.json @@ -0,0 +1,66 @@ +{ + "properties": { + "roleName": "Teradata AI-Unlimited Workspace Deployment Permissions", + "description": "Subscription level permissions for the Workspace service to create AI-Unlimited Engine deployments with their own resource groups", + "assignableScopes": [ + "/subscriptions/" + ], + "permissions": [ + { + "actions": [ + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/sshPublicKeys/read", + "Microsoft.Compute/sshPublicKeys/write", + "Microsoft.Compute/sshPublicKeys/delete", + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/virtualMachines/write", + "Microsoft.Compute/virtualMachines/delete", + "Microsoft.KeyVault/vaults/read", + "Microsoft.KeyVault/vaults/write", + "Microsoft.KeyVault/vaults/delete", + "Microsoft.KeyVault/vaults/accessPolicies/write", + "Microsoft.KeyVault/locations/operationResults/read", + "Microsoft.KeyVault/locations/deletedVaults/purge/action", + "Microsoft.ManagedIdentity/userAssignedIdentities/delete", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + "Microsoft.ManagedIdentity/userAssignedIdentities/listAssociatedResources/action", + "Microsoft.ManagedIdentity/userAssignedIdentities/read", + "Microsoft.ManagedIdentity/userAssignedIdentities/write", + "Microsoft.Network/applicationSecurityGroups/read", + "Microsoft.Network/applicationSecurityGroups/write", + "Microsoft.Network/applicationSecurityGroups/joinIpConfiguration/action", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.Network/virtualNetworks/write", + "Microsoft.Network/virtualNetworks/delete", + "Microsoft.Network/virtualNetworks/subnets/read", + "Microsoft.Network/virtualNetworks/subnets/write", + "Microsoft.Network/virtualNetworks/subnets/delete", + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/networkInterfaces/read", + "Microsoft.Network/networkInterfaces/write", + "Microsoft.Network/networkInterfaces/delete", + "Microsoft.Network/networkInterfaces/join/action", + "Microsoft.Network/networkSecurityGroups/read", + "Microsoft.Network/networkSecurityGroups/write", + "Microsoft.Network/networkSecurityGroups/delete", + "Microsoft.Network/networkSecurityGroups/securityRules/read", + "Microsoft.Network/networkSecurityGroups/securityRules/write", + "Microsoft.Network/networkSecurityGroups/securityRules/delete", + "Microsoft.Network/networkSecurityGroups/join/action", + "Microsoft.Network/publicIPAddresses/read", + "Microsoft.Network/publicIPAddresses/write", + "Microsoft.Network/publicIPAddresses/join/action", + "Microsoft.Network/publicIPAddresses/delete", + "Microsoft.Resources/subscriptions/resourcegroups/read", + "Microsoft.Resources/subscriptions/resourcegroups/write", + "Microsoft.Resources/subscriptions/resourcegroups/delete" + ], + "notActions": [], + "dataActions": [], + "notDataActions": [] + } + ] + } +} \ No newline at end of file diff --git a/deployments/azure/resources.json b/deployments/azure/resources.json index 19c1677..c2370ba 100644 --- a/deployments/azure/resources.json +++ b/deployments/azure/resources.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.19.5.34762", - "templateHash": "5259534896949154499" + "version": "0.23.1.45101", + "templateHash": "9865267277444874121" } }, "parameters": { @@ -81,6 +81,9 @@ "Microsoft.ManagedIdentity/userAssignedIdentities/listAssociatedResources/action", "Microsoft.ManagedIdentity/userAssignedIdentities/read", "Microsoft.ManagedIdentity/userAssignedIdentities/write", + "Microsoft.Network/applicationSecurityGroups/read", + "Microsoft.Network/applicationSecurityGroups/write", + "Microsoft.Network/applicationSecurityGroups/joinIpConfiguration/action", "Microsoft.Network/virtualNetworks/read", "Microsoft.Network/virtualNetworks/write", "Microsoft.Network/virtualNetworks/delete", @@ -147,8 +150,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.19.5.34762", - "templateHash": "8841300181762250120" + "version": "0.23.1.45101", + "templateHash": "2636051152564020606" } }, "parameters": { diff --git a/deployments/azure/templates/all-in-one.cloudinit.yaml b/deployments/azure/templates/all-in-one.cloudinit.yaml new file mode 100644 index 0000000..9e7a75f --- /dev/null +++ b/deployments/azure/templates/all-in-one.cloudinit.yaml @@ -0,0 +1,26 @@ +#cloud-config +write_files: +- encoding: b64 + content: "{0}" + owner: root:root + path: /usr/lib/systemd/system/workspaces.service + permissions: '0640' +- encoding: b64 + content: "{1}" + owner: root:root + path: /usr/lib/systemd/system/jupyter.service + permissions: '0640' + +runcmd: +- mkdir -p /etc/td +- | + export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}'); + if [ -n "${{PERMDISK}}" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi + /usr/bin/echo "LABEL=WORKSPACES /etc/td ext4 defaults 0 2" >> /etc/fstab + /usr/bin/mount -a +- while [ $(systemctl status docker | grep "active (running)" | wc -l) -lt 1 ]; do sleep 5; done +- sleep 60 +- systemctl enable workspaces.service +- systemctl start workspaces.service +- systemctl enable jupyter.service +- systemctl start jupyter.service diff --git a/deployments/azure/templates/jupyter.cloudinit.yaml b/deployments/azure/templates/jupyter.cloudinit.yaml index 35adbf2..987dc49 100644 --- a/deployments/azure/templates/jupyter.cloudinit.yaml +++ b/deployments/azure/templates/jupyter.cloudinit.yaml @@ -7,7 +7,13 @@ write_files: permissions: '0640' runcmd: +- mkdir -p /etc/td +- | + export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}'); + if [ -n "${{PERMDISK}}" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi + /usr/bin/echo "LABEL=WORKSPACES /etc/td ext4 defaults 0 2" >> /etc/fstab + /usr/bin/mount -a - while [ $(systemctl status docker | grep "active (running)" | wc -l) -lt 1 ]; do sleep 5; done - sleep 60 - systemctl enable jupyter.service -- systemctl start jupyter.service \ No newline at end of file +- systemctl start jupyter.service diff --git a/deployments/azure/templates/jupyter.service b/deployments/azure/templates/jupyter.service index 2310f41..2fe6cd7 100644 --- a/deployments/azure/templates/jupyter.service +++ b/deployments/azure/templates/jupyter.service @@ -9,7 +9,8 @@ StartLimitBurst=10 TimeoutStartSec=0 Restart=always RestartSec=2 -ExecStartPre=-/usr/bin/mkdir -p /etc/td +ExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited +ExecStartPre=-/usr/bin/mkdir -p /etc/td/jupyter/{{userdata,ipython}} ExecStartPre=-/usr/bin/docker exec %n stop || true ExecStartPre=-/usr/bin/docker rm %n || true ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} @@ -17,8 +18,10 @@ ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} ExecStart=/usr/bin/docker run \ -e accept_license=Y \ -e JUPYTER_TOKEN={4} \ - -v /etc/td:/home/jovyan/JupyterLabRoot/userdata \ + -v /etc/td/jupyter/userdata:/home/jovyan/JupyterLabRoot/userdata \ + -v /etc/td/jupyter/ipython:/home/jovyan/.ipython \ -p {3}:8888 \ + --network ai_unlimited \ --rm --name %n {0}/{1}:{2} [Install] diff --git a/deployments/azure/templates/workspaces.cloudinit.yaml b/deployments/azure/templates/workspaces.cloudinit.yaml index fb59a4d..4d93707 100644 --- a/deployments/azure/templates/workspaces.cloudinit.yaml +++ b/deployments/azure/templates/workspaces.cloudinit.yaml @@ -7,7 +7,13 @@ write_files: permissions: '0640' runcmd: +- mkdir -p /etc/td +- | + export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}'); + if [ -n "${{PERMDISK}}" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi + /usr/bin/echo "LABEL=WORKSPACES /etc/td ext4 defaults 0 2" >> /etc/fstab + /usr/bin/mount -a - while [ $(systemctl status docker | grep "active (running)" | wc -l) -lt 1 ]; do sleep 5; done - sleep 60 - systemctl enable workspaces.service -- systemctl start workspaces.service \ No newline at end of file +- systemctl start workspaces.service diff --git a/deployments/azure/templates/workspaces.service b/deployments/azure/templates/workspaces.service index fdf8b0e..b558ecf 100644 --- a/deployments/azure/templates/workspaces.service +++ b/deployments/azure/templates/workspaces.service @@ -9,7 +9,8 @@ StartLimitBurst=10 TimeoutStartSec=0 Restart=always RestartSec=2 -ExecStartPre=-/usr/bin/mkdir -p /etc/td +ExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited +ExecStartPre=-/usr/bin/mkdir -p /etc/td/workspace ExecStartPre=-/usr/bin/docker exec %n stop || true ExecStartPre=-/usr/bin/docker rm %n || true ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} @@ -20,9 +21,10 @@ ExecStart=/usr/bin/docker run \ -e ARM_USE_MSI=true \ -e ARM_SUBSCRIPTION_ID={5} \ -e ARM_TENANT_ID={6} \ - -v /etc/td:/etc/td \ + -v /etc/td/workspace:/etc/td \ -p {3}:3000 \ -p {4}:3282 \ + --network ai_unlimited \ --rm --name %n {0}/{1}:{2} workspaces serve -v [Install]