From 76cd060aad73e411d4b4ac2fcb98035af494b841 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Mon, 9 Dec 2024 13:17:50 -0800
Subject: [PATCH 1/8] Add a script to automatically provision a Prefect Cloud
 environment for the debugging tutorial

---
 .gitignore           |  12 ++++-
 infra/destroy_env.sh |  43 ++++++++++++++++++
 infra/main.tf        |  38 ++++++++++++++++
 infra/variables.tf   |  10 +++++
 setup_env.sh         | 103 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100755 infra/destroy_env.sh
 create mode 100644 infra/main.tf
 create mode 100644 infra/variables.tf
 create mode 100755 setup_env.sh

diff --git a/.gitignore b/.gitignore
index 151121e..415e09a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,12 @@
-env
+# Python
 __pycache__
+env
+venv
+
+# Terraform files
+.terraform
+*.tfstate
+*.tfstate.*
+.terraform.lock.hcl
+.terraformrc
+terraform.rc
diff --git a/infra/destroy_env.sh b/infra/destroy_env.sh
new file mode 100755
index 0000000..4e33049
--- /dev/null
+++ b/infra/destroy_env.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+###############################################################################
+# This script sets up a Prefect Cloud account with the following environment  #
+#                                                                             #
+# 1. Two workspaces: `production` and `staging`                               #
+# 2. A default Docker work pool in each workspace                             #
+# 3. A flow in each workspace                                                 #
+# 4. The flow in each workspace is run multiple times                         #
+# 5. The flow in `staging` has failures to demonstrate debugging              #
+#                                                                             #
+# NOTE: You must have Docker running on your machine to run this script!!!    #
+###############################################################################
+
+# Exit on any error
+set -e
+
+echo "🔑 Reading Prefect API key and account ID..."
+
+# Get active profile from `profiles.toml`
+ACTIVE_PROFILE=$(awk -F ' = ' '/^active/ {gsub(/"/, "", $2); print $2}' ~/.prefect/profiles.toml)
+
+# Get API key for the active profile from `profiles.toml`
+API_KEY=$(awk -v profile="profiles.$ACTIVE_PROFILE" '
+  $0 ~ "\\[" profile "\\]" {in_section=1; next}
+  in_section && /^\[/ {in_section=0}
+  in_section && /PREFECT_API_KEY/ {
+    gsub(/"/, "", $3)
+    print $3
+    exit
+  }
+' ~/.prefect/profiles.toml)
+export TF_VAR_prefect_api_key=$API_KEY
+
+# Extract account ID from `prefect config view`
+ACCOUNT_ID=$(prefect config view | awk -F'/' '/^https:\/\/app.prefect.cloud\/account\// {print $5}')
+export TF_VAR_prefect_account_id=$ACCOUNT_ID
+
+# Get account handle from the active workspace
+ACCOUNT_HANDLE=$(prefect cloud workspace ls | awk '/^│ \*/ {print $3}' | cut -d'/' -f1)
+
+echo "🏗️ Running Terraform to provision infrastructure..."
+terraform destroy -auto-approve
\ No newline at end of file
diff --git a/infra/main.tf b/infra/main.tf
new file mode 100644
index 0000000..f4d5056
--- /dev/null
+++ b/infra/main.tf
@@ -0,0 +1,38 @@
+terraform {
+  required_providers {
+    prefect = {
+      source = "PrefectHQ/prefect"
+    }
+  }
+}
+
+provider "prefect" {
+  api_key = var.prefect_api_key
+  account_id = var.prefect_account_id
+}
+
+# Create staging workspace
+resource "prefect_workspace" "staging" {
+  name        = "Staging"
+  handle      = "staging"
+}
+
+# Create production workspace
+resource "prefect_workspace" "production" {
+  name        = "Production"
+  handle      = "production"
+}
+
+# Create default work pool in staging workspace
+resource "prefect_work_pool" "staging_default" {
+  name        = "default-work-pool"
+  workspace_id = prefect_workspace.staging.id
+  type        = "docker"
+}
+
+# Create default work pool in production workspace
+resource "prefect_work_pool" "production_default" {
+  name        = "default-work-pool"
+  workspace_id = prefect_workspace.production.id
+  type        = "docker"
+} 
\ No newline at end of file
diff --git a/infra/variables.tf b/infra/variables.tf
new file mode 100644
index 0000000..5bcc6e4
--- /dev/null
+++ b/infra/variables.tf
@@ -0,0 +1,10 @@
+variable "prefect_api_key" {
+  description = "Prefect Cloud API key"
+  type        = string
+  sensitive   = true
+}
+
+variable "prefect_account_id" {
+  description = "Prefect Cloud Account ID"
+  type        = string
+} 
\ No newline at end of file
diff --git a/setup_env.sh b/setup_env.sh
new file mode 100755
index 0000000..430b0dd
--- /dev/null
+++ b/setup_env.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+###############################################################################
+# This script sets up a Prefect Cloud account with the following environment  #
+#                                                                             #
+# 1. Two workspaces: `production` and `staging`                               #
+# 2. A default Docker work pool in each workspace                             #
+# 3. A flow in each workspace                                                 #
+# 4. The flow in each workspace is run multiple times                         #
+# 5. The flow in `staging` has failures to demonstrate debugging              #
+#                                                                             #
+# NOTE: You must have Docker running on your machine to run this script!!!    #
+###############################################################################
+
+# Exit on any error
+set -e
+
+# Check if Docker is running
+echo "🐳 Checking if Docker is running..."
+if ! docker info > /dev/null 2>&1; then
+    echo "❌ Error: Docker is not running. Please start Docker and try again."
+    exit 1
+fi
+
+echo "✅ Docker is running"
+
+echo "🔑 Reading Prefect API key and account ID..."
+
+# Get active profile from `profiles.toml`
+ACTIVE_PROFILE=$(awk -F ' = ' '/^active/ {gsub(/"/, "", $2); print $2}' ~/.prefect/profiles.toml)
+
+# Get API key for the active profile from `profiles.toml`
+API_KEY=$(awk -v profile="profiles.$ACTIVE_PROFILE" '
+  $0 ~ "\\[" profile "\\]" {in_section=1; next}
+  in_section && /^\[/ {in_section=0}
+  in_section && /PREFECT_API_KEY/ {
+    gsub(/"/, "", $3)
+    print $3
+    exit
+  }
+' ~/.prefect/profiles.toml)
+export TF_VAR_prefect_api_key=$API_KEY
+
+# Extract account ID from `prefect config view`
+ACCOUNT_ID=$(prefect config view | awk -F'/' '/^https:\/\/app.prefect.cloud\/account\// {print $5}')
+export TF_VAR_prefect_account_id=$ACCOUNT_ID
+
+# Get account handle from the active workspace
+ACCOUNT_HANDLE=$(prefect cloud workspace ls | awk '/^│ \*/ {print $3}' | cut -d'/' -f1)
+
+echo "🏗️ Running Terraform to provision infrastructure..."
+cd infra/
+terraform init
+terraform apply -auto-approve
+cd ..
+
+###############################################################################
+
+echo "🚀 Populate production workspace..."
+
+# Start worker for production workspace
+prefect cloud workspace set --workspace "$ACCOUNT_HANDLE/production"
+prefect worker start --pool "default-work-pool" &
+PROD_WORKER_PID=$!
+
+# Give workers time to start
+sleep 10
+
+# Run in production workspace
+python simulate_failures.py &
+PROD_SIM_PID=$!
+
+# Wait for simulations to complete
+wait $PROD_SIM_PID
+
+# Kill worker process
+kill $PROD_WORKER_PID
+
+###############################################################################
+
+echo "🚀 Populate staging workspace..."
+
+# Start worker for staging workspace
+prefect cloud workspace set --workspace "$ACCOUNT_HANDLE/staging"
+prefect worker start --pool "default-work-pool" &
+STAGING_WORKER_PID=$!
+
+# Give workers time to start
+sleep 10
+
+# Run in staging workspace
+python simulate_failures.py --fail-at-run 3 &
+STAGING_SIM_PID=$!
+
+# Wait for simulations to complete
+wait $STAGING_SIM_PID
+
+# Kill worker process
+kill $STAGING_WORKER_PID
+
+###############################################################################
+
+echo "✅ All done!" 
\ No newline at end of file

From aeb3b637ce9c32bed6047b0e3fd869aa95208b10 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Mon, 9 Dec 2024 13:30:03 -0800
Subject: [PATCH 2/8] Add new lines at the end of files

---
 infra/destroy_env.sh | 2 +-
 infra/main.tf        | 2 +-
 infra/variables.tf   | 2 +-
 setup_env.sh         | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/infra/destroy_env.sh b/infra/destroy_env.sh
index 4e33049..d08421f 100755
--- a/infra/destroy_env.sh
+++ b/infra/destroy_env.sh
@@ -40,4 +40,4 @@ export TF_VAR_prefect_account_id=$ACCOUNT_ID
 ACCOUNT_HANDLE=$(prefect cloud workspace ls | awk '/^│ \*/ {print $3}' | cut -d'/' -f1)
 
 echo "🏗️ Running Terraform to provision infrastructure..."
-terraform destroy -auto-approve
\ No newline at end of file
+terraform destroy -auto-approve
diff --git a/infra/main.tf b/infra/main.tf
index f4d5056..a22f6e5 100644
--- a/infra/main.tf
+++ b/infra/main.tf
@@ -35,4 +35,4 @@ resource "prefect_work_pool" "production_default" {
   name        = "default-work-pool"
   workspace_id = prefect_workspace.production.id
   type        = "docker"
-} 
\ No newline at end of file
+}
diff --git a/infra/variables.tf b/infra/variables.tf
index 5bcc6e4..5c9f471 100644
--- a/infra/variables.tf
+++ b/infra/variables.tf
@@ -7,4 +7,4 @@ variable "prefect_api_key" {
 variable "prefect_account_id" {
   description = "Prefect Cloud Account ID"
   type        = string
-} 
\ No newline at end of file
+}
diff --git a/setup_env.sh b/setup_env.sh
index 430b0dd..b870835 100755
--- a/setup_env.sh
+++ b/setup_env.sh
@@ -100,4 +100,4 @@ kill $STAGING_WORKER_PID
 
 ###############################################################################
 
-echo "✅ All done!" 
\ No newline at end of file
+echo "✅ All done!"

From 647147c0968f43583a7f74468b030a46356fce42 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 08:28:32 -0800
Subject: [PATCH 3/8] Update misleading description in the 'destroy_env.sh'
 script

---
 infra/destroy_env.sh | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/infra/destroy_env.sh b/infra/destroy_env.sh
index d08421f..8d7e1c1 100755
--- a/infra/destroy_env.sh
+++ b/infra/destroy_env.sh
@@ -1,15 +1,7 @@
 #!/bin/bash
 
 ###############################################################################
-# This script sets up a Prefect Cloud account with the following environment  #
-#                                                                             #
-# 1. Two workspaces: `production` and `staging`                               #
-# 2. A default Docker work pool in each workspace                             #
-# 3. A flow in each workspace                                                 #
-# 4. The flow in each workspace is run multiple times                         #
-# 5. The flow in `staging` has failures to demonstrate debugging              #
-#                                                                             #
-# NOTE: You must have Docker running on your machine to run this script!!!    #
+# This script destroys any Prefect Cloud resources created by `setup_env.sh`  #
 ###############################################################################
 
 # Exit on any error

From 12d304cbbd8c32540bc30b7fdb69995542f8e493 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 08:30:13 -0800
Subject: [PATCH 4/8] Update .gitignore to only include the min required
 Terraform files

---
 .gitignore | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 415e09a..56254eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,5 +8,3 @@ venv
 *.tfstate
 *.tfstate.*
 .terraform.lock.hcl
-.terraformrc
-terraform.rc

From 6800f1dae5a570c243143670e3fa8f7a3805f99f Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 09:37:06 -0800
Subject: [PATCH 5/8] Use a more robust way of getting the account handle for
 the active profile

---
 setup_env.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 67 insertions(+), 5 deletions(-)

diff --git a/setup_env.sh b/setup_env.sh
index b870835..7fb3a17 100755
--- a/setup_env.sh
+++ b/setup_env.sh
@@ -8,13 +8,17 @@
 # 3. A flow in each workspace                                                 #
 # 4. The flow in each workspace is run multiple times                         #
 # 5. The flow in `staging` has failures to demonstrate debugging              #
-#                                                                             #
-# NOTE: You must have Docker running on your machine to run this script!!!    #
+#
+# NOTE: You must have Docker and Terraform installed                          #
 ###############################################################################
 
 # Exit on any error
 set -e
 
+###############################################################################
+# Check for dependencies
+###############################################################################
+
 # Check if Docker is running
 echo "🐳 Checking if Docker is running..."
 if ! docker info > /dev/null 2>&1; then
@@ -24,8 +28,53 @@ fi
 
 echo "✅ Docker is running"
 
+# Check if Terraform is installed
+echo "🔧 Checking if Terraform is installed..."
+if ! command -v terraform &> /dev/null; then
+    echo "❌ Error: Terraform is not installed. Please install Terraform and try again."
+    exit 1
+fi
+
+echo "✅ Terraform is installed"
+
+# Check if Python is installed and determine the Python command
+echo "🐍 Checking if Python is installed..."
+if command -v python3 &> /dev/null; then
+    PYTHON_CMD="python3"
+elif command -v python &> /dev/null; then
+    PYTHON_CMD="python"
+else
+    echo "❌ Error: Python is not installed. Please install Python 3.9 or higher and try again."
+    exit 1
+fi
+
+# Verify Python version is 3.9 or higher
+if ! $PYTHON_CMD -c "import sys; assert sys.version_info >= (3, 9), 'Python 3.9 or higher is required'" &> /dev/null; then
+    echo "❌ Error: Python 3.9 or higher is required. Found $($PYTHON_CMD --version)"
+    exit 1
+fi
+
+echo "✅ Python $(${PYTHON_CMD} --version) is installed"
+
+###############################################################################
+# Set up virtual environment
+###############################################################################
+
+# Create and activate virtual environment
+echo "🌟 Setting up Python virtual environment..."
+$PYTHON_CMD -m venv temp_venv
+source temp_venv/bin/activate
+
+# Install requirements
+echo "📦 Installing Python packages..."
+pip install -r requirements.txt
+
 echo "🔑 Reading Prefect API key and account ID..."
 
+###############################################################################
+# Get auth credentials
+###############################################################################
+
 # Get active profile from `profiles.toml`
 ACTIVE_PROFILE=$(awk -F ' = ' '/^active/ {gsub(/"/, "", $2); print $2}' ~/.prefect/profiles.toml)
 
@@ -45,8 +94,12 @@ export TF_VAR_prefect_api_key=$API_KEY
 ACCOUNT_ID=$(prefect config view | awk -F'/' '/^https:\/\/app.prefect.cloud\/account\// {print $5}')
 export TF_VAR_prefect_account_id=$ACCOUNT_ID
 
-# Get account handle from the active workspace
-ACCOUNT_HANDLE=$(prefect cloud workspace ls | awk '/^│ \*/ {print $3}' | cut -d'/' -f1)
+# Get account handle for the account ID given above
+ACCOUNT_HANDLE=$(curl -s "https://api.prefect.cloud/api/accounts/$ACCOUNT_ID" -H "Authorization: Bearer $API_KEY" | awk -F'"handle":"' '{print $2}' | awk -F'"' '{print $1}')
+
+###############################################################################
+# Provision Prefect Cloud resources
+###############################################################################
 
 echo "🏗️ Running Terraform to provision infrastructure..."
 cd infra/
@@ -54,6 +107,8 @@ terraform init
 terraform apply -auto-approve
 cd ..
 
+###############################################################################
+# Run flows in production
 ###############################################################################
 
 echo "🚀 Populate production workspace..."
@@ -76,6 +131,8 @@ wait $PROD_SIM_PID
 # Kill worker process
 kill $PROD_WORKER_PID
 
+###############################################################################
+# Run flows in production
 ###############################################################################
 
 echo "🚀 Populate staging workspace..."
@@ -99,5 +156,10 @@ wait $STAGING_SIM_PID
 kill $STAGING_WORKER_PID
 
 ###############################################################################
+# Cleanup virtual environment
+###############################################################################
+
+deactivate
+rm -rf temp_venv
 
-echo "✅ All done!"
+echo "✅ All done!"
\ No newline at end of file

From d5cb89f8f3fe2379940be4313e928855340f5a5e Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 09:43:57 -0800
Subject: [PATCH 6/8] Suppress Docker worker log output

---
 setup_env.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/setup_env.sh b/setup_env.sh
index 7fb3a17..0e367bc 100755
--- a/setup_env.sh
+++ b/setup_env.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ###############################################################################
-# This script sets up a Prefect Cloud account with the following environment  #
+# This script sets up a _paid_ Prefect Cloud account with resources:          #
 #                                                                             #
 # 1. Two workspaces: `production` and `staging`                               #
 # 2. A default Docker work pool in each workspace                             #
@@ -113,13 +113,13 @@ cd ..
 
 echo "🚀 Populate production workspace..."
 
-# Start worker for production workspace
+# Start worker for production workspace with suppressed output
 prefect cloud workspace set --workspace "$ACCOUNT_HANDLE/production"
-prefect worker start --pool "default-work-pool" &
+prefect worker start --pool "default-work-pool" > /dev/null 2>&1 &
 PROD_WORKER_PID=$!
 
 # Give workers time to start
-sleep 10
+sleep 5
 
 # Run in production workspace
 python simulate_failures.py &
@@ -132,18 +132,18 @@ wait $PROD_SIM_PID
 kill $PROD_WORKER_PID
 
 ###############################################################################
-# Run flows in production
+# Run flows in staging
 ###############################################################################
 
 echo "🚀 Populate staging workspace..."
 
-# Start worker for staging workspace
+# Start worker for staging workspace with suppressed output
 prefect cloud workspace set --workspace "$ACCOUNT_HANDLE/staging"
-prefect worker start --pool "default-work-pool" &
+prefect worker start --pool "default-work-pool" > /dev/null 2>&1 &
 STAGING_WORKER_PID=$!
 
 # Give workers time to start
-sleep 10
+sleep 5
 
 # Run in staging workspace
 python simulate_failures.py --fail-at-run 3 &

From 8c1e94de01da4275419c25d03499690d5a5c1a52 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 09:45:17 -0800
Subject: [PATCH 7/8] Remove the account handle extraction from the
 destroy_env.sh script since it's not needed

---
 infra/destroy_env.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/infra/destroy_env.sh b/infra/destroy_env.sh
index 8d7e1c1..cd94f92 100755
--- a/infra/destroy_env.sh
+++ b/infra/destroy_env.sh
@@ -28,8 +28,5 @@ export TF_VAR_prefect_api_key=$API_KEY
 ACCOUNT_ID=$(prefect config view | awk -F'/' '/^https:\/\/app.prefect.cloud\/account\// {print $5}')
 export TF_VAR_prefect_account_id=$ACCOUNT_ID
 
-# Get account handle from the active workspace
-ACCOUNT_HANDLE=$(prefect cloud workspace ls | awk '/^│ \*/ {print $3}' | cut -d'/' -f1)
-
 echo "🏗️ Running Terraform to provision infrastructure..."
 terraform destroy -auto-approve

From 8ad77a02aaaffad0077be337de6cd300b63653f5 Mon Sep 17 00:00:00 2001
From: Daniel Sauble <djsauble@gmail.com>
Date: Tue, 10 Dec 2024 10:14:56 -0800
Subject: [PATCH 8/8] Confirm that this is a Prefect account that supports
 multiple workspaces

---
 setup_env.sh | 45 ++++++++++++++++++++++++++++-----------------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/setup_env.sh b/setup_env.sh
index 0e367bc..4ffad86 100755
--- a/setup_env.sh
+++ b/setup_env.sh
@@ -56,21 +56,6 @@ fi
 
 echo "✅ Python $(${PYTHON_CMD} --version) is installed"
 
-###############################################################################
-# Set up virtual environment
-###############################################################################
-
-# Create and activate virtual environment
-echo "🌟 Setting up Python virtual environment..."
-$PYTHON_CMD -m venv temp_venv
-source temp_venv/bin/activate
-
-# Install requirements
-echo "📦 Installing Python packages..."
-pip install -r requirements.txt
-
-echo "🔑 Reading Prefect API key and account ID..."
-
 ###############################################################################
 # Get auth credentials
 ###############################################################################
@@ -94,8 +79,34 @@ export TF_VAR_prefect_api_key=$API_KEY
 ACCOUNT_ID=$(prefect config view | awk -F'/' '/^https:\/\/app.prefect.cloud\/account\// {print $5}')
 export TF_VAR_prefect_account_id=$ACCOUNT_ID
 
-# Get account handle for the account ID given above
-ACCOUNT_HANDLE=$(curl -s "https://api.prefect.cloud/api/accounts/$ACCOUNT_ID" -H "Authorization: Bearer $API_KEY" | awk -F'"handle":"' '{print $2}' | awk -F'"' '{print $1}')
+# Account details
+ACCOUNT_DETAILS=$(curl -s "https://api.prefect.cloud/api/accounts/$ACCOUNT_ID" -H "Authorization: Bearer $API_KEY")
+
+# Get account handle
+ACCOUNT_HANDLE=$(echo $ACCOUNT_DETAILS | awk -F'"handle":"' '{print $2}' | awk -F'"' '{print $1}')
+
+# Get plan type (fail if a personal account, since these do not support multiple workspaces)
+PLAN_TYPE=$(echo $ACCOUNT_DETAILS | awk -F'"plan_type":"' '{print $2}' | awk -F'"' '{print $1}')
+
+if [[ $PLAN_TYPE == "PERSONAL" ]]; then
+    echo "❌ Error: This script requires a paid Prefect Cloud account with support for multiple workspaces."
+    exit 1
+fi
+
+###############################################################################
+# Set up virtual environment
+###############################################################################
+
+# Create and activate virtual environment
+echo "🌟 Setting up Python virtual environment..."
+$PYTHON_CMD -m venv temp_venv
+source temp_venv/bin/activate
+
+# Install requirements
+echo "📦 Installing Python packages..."
+pip install -r requirements.txt
+
+echo "🔑 Reading Prefect API key and account ID..."
 
 ###############################################################################
 # Provision Prefect Cloud resources