From e5e8f8a465f84019f7bd9775724dc3f0798f24c2 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Fri, 24 May 2024 13:45:14 +0200 Subject: [PATCH] Add Azure Synapse Workspace and Pools example --- examples/azure-synapse/.gitignore | 10 +++ examples/azure-synapse/Main.scala | 125 +++++++++++++++++++++++++++ examples/azure-synapse/Pulumi.yaml | 3 + examples/azure-synapse/README.md | 43 +++++++++ examples/azure-synapse/project.scala | 6 ++ 5 files changed, 187 insertions(+) create mode 100644 examples/azure-synapse/.gitignore create mode 100644 examples/azure-synapse/Main.scala create mode 100644 examples/azure-synapse/Pulumi.yaml create mode 100644 examples/azure-synapse/README.md create mode 100644 examples/azure-synapse/project.scala diff --git a/examples/azure-synapse/.gitignore b/examples/azure-synapse/.gitignore new file mode 100644 index 00000000..7da74da6 --- /dev/null +++ b/examples/azure-synapse/.gitignore @@ -0,0 +1,10 @@ +### Scala an JVM +*.class +*.log +.bsp +.scala-build + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +kubeconfig.json diff --git a/examples/azure-synapse/Main.scala b/examples/azure-synapse/Main.scala new file mode 100644 index 00000000..88f8b256 --- /dev/null +++ b/examples/azure-synapse/Main.scala @@ -0,0 +1,125 @@ +import besom.* +import besom.api.azurenative +import besom.api.random + +@main def main = Pulumi.run { + + val resourceGroup = azurenative.resources.ResourceGroup("synapse-rg") + + val storageAccount = azurenative.storage.StorageAccount( + name = "synapsesa", + azurenative.storage.StorageAccountArgs( + resourceGroupName = resourceGroup.name, + accessTier = azurenative.storage.enums.AccessTier.Hot, + enableHttpsTrafficOnly = true, + isHnsEnabled = true, + kind = azurenative.storage.enums.Kind.StorageV2, + sku = azurenative.storage.inputs.SkuArgs( + name = azurenative.storage.enums.SkuName.Standard_RAGRS + ) + ) + ) + + val users = azurenative.storage.BlobContainer( + name = "users", + azurenative.storage.BlobContainerArgs( + resourceGroupName = resourceGroup.name, + accountName = storageAccount.name, + publicAccess = None + ) + ) + + val dataLakeStorageAccountUrl = p"https://${storageAccount.name}.dfs.core.windows.net" + + val workspace = azurenative.synapse.Workspace( + name = "my-workspace", + azurenative.synapse.WorkspaceArgs( + resourceGroupName = resourceGroup.name, + defaultDataLakeStorage = azurenative.synapse.inputs.DataLakeStorageAccountDetailsArgs( + accountUrl = dataLakeStorageAccountUrl, + filesystem = "users" + ), + identity = azurenative.synapse.inputs.ManagedIdentityArgs( + `type` = azurenative.synapse.enums.ResourceIdentityType.SystemAssigned + ), + sqlAdministratorLogin = "sqladminuser", + sqlAdministratorLoginPassword = random.RandomPassword("workspacePwd", random.RandomPasswordArgs(length = 12)).result + ) + ) + + val firewallRule = azurenative.synapse.IpFirewallRule( + name = "allowAll", + azurenative.synapse.IpFirewallRuleArgs( + resourceGroupName = resourceGroup.name, + workspaceName = workspace.name, + endIpAddress = "255.255.255.255", + startIpAddress = "0.0.0.0" + ) + ) + + val subscriptionId = resourceGroup.id.map(_.split("/")(2)) + val roleDefinitionId = + p"/subscriptions/$subscriptionId/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe" + + val storageAccess = azurenative.authorization.RoleAssignment( + name = "storageAccess", + azurenative.authorization.RoleAssignmentArgs( + roleAssignmentName = random.RandomUuid("roleName").result, + scope = storageAccount.id, + principalId = workspace.identity.principalId.getOrElse(""), + principalType = azurenative.authorization.enums.PrincipalType.ServicePrincipal, + roleDefinitionId = roleDefinitionId + ) + ) + + val clientConfig = azurenative.authorization.getClientConfig() + + val userAccess = azurenative.authorization.RoleAssignment( + name = "userAccess", + azurenative.authorization.RoleAssignmentArgs( + roleAssignmentName = random.RandomUuid("userRoleName").result, + scope = storageAccount.id, + principalId = clientConfig.objectId, + principalType = azurenative.authorization.enums.PrincipalType.User, + roleDefinitionId = roleDefinitionId + ) + ) + + val sqlPool = azurenative.synapse.SqlPool( + name = "SQLPOOL1", + azurenative.synapse.SqlPoolArgs( + resourceGroupName = resourceGroup.name, + workspaceName = workspace.name, + collation = "SQL_Latin1_General_CP1_CI_AS", + createMode = azurenative.synapse.enums.CreateMode.Default, + sku = azurenative.synapse.inputs.SkuArgs( + name = "DW100c" + ) + ) + ) + + val sparkPool = azurenative.synapse.BigDataPool( + name = "Spark1", + azurenative.synapse.BigDataPoolArgs( + resourceGroupName = resourceGroup.name, + workspaceName = workspace.name, + autoPause = azurenative.synapse.inputs.AutoPausePropertiesArgs( + delayInMinutes = 15, + enabled = true + ), + autoScale = azurenative.synapse.inputs.AutoScalePropertiesArgs( + enabled = true, + maxNodeCount = 3, + minNodeCount = 3 + ), + nodeCount = 3, + nodeSize = azurenative.synapse.enums.NodeSize.Small, + nodeSizeFamily = azurenative.synapse.enums.NodeSizeFamily.MemoryOptimized, + sparkVersion = "3.3" + ) + ) + + Stack(users, firewallRule, storageAccess, userAccess, sqlPool, sparkPool).exports( + sparkPoolName = sparkPool.name + ) +} diff --git a/examples/azure-synapse/Pulumi.yaml b/examples/azure-synapse/Pulumi.yaml new file mode 100644 index 00000000..c82c8dd7 --- /dev/null +++ b/examples/azure-synapse/Pulumi.yaml @@ -0,0 +1,3 @@ +name: azure-synapse +description: Azure Synapse example +runtime: scala diff --git a/examples/azure-synapse/README.md b/examples/azure-synapse/README.md new file mode 100644 index 00000000..a1372128 --- /dev/null +++ b/examples/azure-synapse/README.md @@ -0,0 +1,43 @@ +# Azure Synapse Workspace and Pools + +Starting point for enterprise analytics solutions based on Azure Synapse. + +## Deploying the App + +To deploy your infrastructure, follow the below steps. + +### Prerequisites + +1. [Install Pulumi](https://www.pulumi.com/docs/get-started/install/) +2. [Configure Azure Credentials](https://www.pulumi.com/docs/intro/cloud-providers/azure/setup/) + +## Running the App + +1. Create a new stack: + + ```bash + $ pulumi stack init dev + ``` + +2. Set the Azure region location to use: + + ```bash + $ pulumi config set azure-native:location westus2 + ``` + +3. Stand up the cluster by invoking pulumi + ```bash + $ pulumi up + ``` + +4. Navigate to https://web.azuresynapse.net and sign in to your new workspace. + +5. From there, feel free to experiment. Simply making edits and running `pulumi up` will incrementally update your + stack. + +6. Once you've finished experimenting, tear down your stack's resources by destroying and removing it: + + ```bash + $ pulumi destroy --yes + $ pulumi stack rm --yes + ``` \ No newline at end of file diff --git a/examples/azure-synapse/project.scala b/examples/azure-synapse/project.scala new file mode 100644 index 00000000..6af83894 --- /dev/null +++ b/examples/azure-synapse/project.scala @@ -0,0 +1,6 @@ +//> using scala "3.3.1" +//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using dep "org.virtuslab::besom-core:0.4.0-SNAPSHOT" +//> using dep "org.virtuslab::besom-random:4.16.1-core.0.4-SNAPSHOT" +//> using dep "org.virtuslab::besom-azure-native:2.38.0-core.0.4-SNAPSHOT" +//> using repository sonatype:snapshots