Skip to content

Commit

Permalink
Add Azure Synapse Workspace and Pools example
Browse files Browse the repository at this point in the history
  • Loading branch information
polkx authored and lbialy committed Aug 22, 2024
1 parent e296d7f commit e5e8f8a
Show file tree
Hide file tree
Showing 5 changed files with 187 additions and 0 deletions.
10 changes: 10 additions & 0 deletions examples/azure-synapse/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
### Scala an JVM
*.class
*.log
.bsp
.scala-build

# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*

kubeconfig.json
125 changes: 125 additions & 0 deletions examples/azure-synapse/Main.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import besom.*
import besom.api.azurenative
import besom.api.random

@main def main = Pulumi.run {

val resourceGroup = azurenative.resources.ResourceGroup("synapse-rg")

val storageAccount = azurenative.storage.StorageAccount(
name = "synapsesa",
azurenative.storage.StorageAccountArgs(
resourceGroupName = resourceGroup.name,
accessTier = azurenative.storage.enums.AccessTier.Hot,
enableHttpsTrafficOnly = true,
isHnsEnabled = true,
kind = azurenative.storage.enums.Kind.StorageV2,
sku = azurenative.storage.inputs.SkuArgs(
name = azurenative.storage.enums.SkuName.Standard_RAGRS
)
)
)

val users = azurenative.storage.BlobContainer(
name = "users",
azurenative.storage.BlobContainerArgs(
resourceGroupName = resourceGroup.name,
accountName = storageAccount.name,
publicAccess = None
)
)

val dataLakeStorageAccountUrl = p"https://${storageAccount.name}.dfs.core.windows.net"

val workspace = azurenative.synapse.Workspace(
name = "my-workspace",
azurenative.synapse.WorkspaceArgs(
resourceGroupName = resourceGroup.name,
defaultDataLakeStorage = azurenative.synapse.inputs.DataLakeStorageAccountDetailsArgs(
accountUrl = dataLakeStorageAccountUrl,
filesystem = "users"
),
identity = azurenative.synapse.inputs.ManagedIdentityArgs(
`type` = azurenative.synapse.enums.ResourceIdentityType.SystemAssigned
),
sqlAdministratorLogin = "sqladminuser",
sqlAdministratorLoginPassword = random.RandomPassword("workspacePwd", random.RandomPasswordArgs(length = 12)).result
)
)

val firewallRule = azurenative.synapse.IpFirewallRule(
name = "allowAll",
azurenative.synapse.IpFirewallRuleArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
endIpAddress = "255.255.255.255",
startIpAddress = "0.0.0.0"
)
)

val subscriptionId = resourceGroup.id.map(_.split("/")(2))
val roleDefinitionId =
p"/subscriptions/$subscriptionId/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe"

val storageAccess = azurenative.authorization.RoleAssignment(
name = "storageAccess",
azurenative.authorization.RoleAssignmentArgs(
roleAssignmentName = random.RandomUuid("roleName").result,
scope = storageAccount.id,
principalId = workspace.identity.principalId.getOrElse("<preview>"),
principalType = azurenative.authorization.enums.PrincipalType.ServicePrincipal,
roleDefinitionId = roleDefinitionId
)
)

val clientConfig = azurenative.authorization.getClientConfig()

val userAccess = azurenative.authorization.RoleAssignment(
name = "userAccess",
azurenative.authorization.RoleAssignmentArgs(
roleAssignmentName = random.RandomUuid("userRoleName").result,
scope = storageAccount.id,
principalId = clientConfig.objectId,
principalType = azurenative.authorization.enums.PrincipalType.User,
roleDefinitionId = roleDefinitionId
)
)

val sqlPool = azurenative.synapse.SqlPool(
name = "SQLPOOL1",
azurenative.synapse.SqlPoolArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
collation = "SQL_Latin1_General_CP1_CI_AS",
createMode = azurenative.synapse.enums.CreateMode.Default,
sku = azurenative.synapse.inputs.SkuArgs(
name = "DW100c"
)
)
)

val sparkPool = azurenative.synapse.BigDataPool(
name = "Spark1",
azurenative.synapse.BigDataPoolArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
autoPause = azurenative.synapse.inputs.AutoPausePropertiesArgs(
delayInMinutes = 15,
enabled = true
),
autoScale = azurenative.synapse.inputs.AutoScalePropertiesArgs(
enabled = true,
maxNodeCount = 3,
minNodeCount = 3
),
nodeCount = 3,
nodeSize = azurenative.synapse.enums.NodeSize.Small,
nodeSizeFamily = azurenative.synapse.enums.NodeSizeFamily.MemoryOptimized,
sparkVersion = "3.3"
)
)

Stack(users, firewallRule, storageAccess, userAccess, sqlPool, sparkPool).exports(
sparkPoolName = sparkPool.name
)
}
3 changes: 3 additions & 0 deletions examples/azure-synapse/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name: azure-synapse
description: Azure Synapse example
runtime: scala
43 changes: 43 additions & 0 deletions examples/azure-synapse/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Azure Synapse Workspace and Pools

Starting point for enterprise analytics solutions based on Azure Synapse.

## Deploying the App

To deploy your infrastructure, follow the below steps.

### Prerequisites

1. [Install Pulumi](https://www.pulumi.com/docs/get-started/install/)
2. [Configure Azure Credentials](https://www.pulumi.com/docs/intro/cloud-providers/azure/setup/)

## Running the App

1. Create a new stack:

```bash
$ pulumi stack init dev
```

2. Set the Azure region location to use:

```bash
$ pulumi config set azure-native:location westus2
```

3. Stand up the cluster by invoking pulumi
```bash
$ pulumi up
```

4. Navigate to https://web.azuresynapse.net and sign in to your new workspace.

5. From there, feel free to experiment. Simply making edits and running `pulumi up` will incrementally update your
stack.

6. Once you've finished experimenting, tear down your stack's resources by destroying and removing it:

```bash
$ pulumi destroy --yes
$ pulumi stack rm --yes
```
6 changes: 6 additions & 0 deletions examples/azure-synapse/project.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//> using scala "3.3.1"
//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement
//> using dep "org.virtuslab::besom-core:0.4.0-SNAPSHOT"
//> using dep "org.virtuslab::besom-random:4.16.1-core.0.4-SNAPSHOT"
//> using dep "org.virtuslab::besom-azure-native:2.38.0-core.0.4-SNAPSHOT"
//> using repository sonatype:snapshots

0 comments on commit e5e8f8a

Please sign in to comment.