From 87389ef5daade79ce098fa394355a3530af58d99 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 26 Aug 2024 18:26:05 +0530 Subject: [PATCH] feat: Scaffolding for using OpenTofu Deploys a small DigitalOcean project using a thin abstraction layer --- pilot/pilot/design-wip/provision.md | 53 ++++++ pilot/provision/README.wtf.md | 40 +++++ pilot/provision/doctype/region/region.json | 87 +++++++++- pilot/provision/doctype/region/region.py | 18 ++- pilot/provision/opentofu.py | 179 +++++++++++++++++++++ pilot/provision/providers/digitalocean.py | 66 ++++++++ 6 files changed, 438 insertions(+), 5 deletions(-) create mode 100644 pilot/pilot/design-wip/provision.md create mode 100644 pilot/provision/README.wtf.md create mode 100644 pilot/provision/opentofu.py create mode 100644 pilot/provision/providers/digitalocean.py diff --git a/pilot/pilot/design-wip/provision.md b/pilot/pilot/design-wip/provision.md new file mode 100644 index 0000000..5234b3e --- /dev/null +++ b/pilot/pilot/design-wip/provision.md @@ -0,0 +1,53 @@ +# Provision +There's no way to do something like tofu.deploy() right now. + +This goes through two levels of abstractions?. JSII + +### CDKTF +We write TerraformStack subclass and define our configration in init. It should something like this +```python +from cdktf import App, Fn, TerraformStack, TerraformVariable +from cdktf_cdktf_provider_digitalocean.provider import DigitaloceanProvider +from cdktf_cdktf_provider_digitalocean.vpc import Vpc +from constructs import Construct + + +class MyStack(TerraformStack): + def __init__(self, scope: Construct, id: str): + super().__init__(scope, id) + do_token_variable = TerraformVariable(self,"do_token", type="string") + DigitaloceanProvider(self, "digitalocean", token=do_token_variable.string_value) + vpc = Vpc(self, "example_vpc", name="vpc-1", region="blr-1", ip_range="ip_range") + +``` + +Unfortunately, Pilot config isn't static. But good news is, this is actually implemented as following + +1. Define TerraformStack subclass, like we did before +This is equivalent of writing a HCL file + +2. Define an app and call `synth` on it + +```python +from cdktf import App, Fn, TerraformStack, TerraformVariable + + +app = App() +MyStack(app, "cdktf-demo") +app.synth() +``` + + +3. Apply generated plan +`cdktf deploy` +We can open up the implemntation and see what happens underneath. + +1. If the implementation is complicated then we can run `cdktf deploy` ourselves + + +--- + +We need to put some dynamic logic on Step 1. +We can't write a class everytime. +Generating Python code is basically the same as writing HCL +What we can do is build a class implementation and app implementation on the fly diff --git a/pilot/provision/README.wtf.md b/pilot/provision/README.wtf.md new file mode 100644 index 0000000..dbd28fd --- /dev/null +++ b/pilot/provision/README.wtf.md @@ -0,0 +1,40 @@ +We want to do as much work in Python / JSON as possible + +Luckily Tofu helps out. + +Every plan / state can be stored as JSON or converted to JSON. + +Here's the rough idea + +```mermaid +stateDiagram + code: Python Declaration + plan: Plan + json: Synthesized JSON + infra: Infrastructure + code --> json: app.synth() + json --> plan: tofu plan + plan --> infra: tofu deploy +``` + + +Same thing in words + +1. Write Python code in `TerraformStack.__init__()` that describes the infra we need +2. "Synthesize" this TerraformStack object `app.synth()` + +The synth actually executes the `__init__` so we can do whatever we want in Python (loops, conditionals etc) + +`synth` generates a `cdktf.out/stacks//cdktf.json` file in the working directory (this is `frappe-bench/sites`) + +We will use `sites//stacks` for now. So the state moves with the site without any special handling. + +TODO: Include this directory in the file backups. + +3. Store this synthesized JSON in some DocType Provision + + +Note: Our Stack can have bugs or the code that defines what we need can have bugs. Have a way to prevent catastrophies at this stage. We need sanity checks in Production to guard against +- Don't trigger anything that can cause data loss +- Don't trigger massive changes ( More than n resources at a time) +- Cross stack changes ?! (Don't delete someone other region?!) diff --git a/pilot/provision/doctype/region/region.json b/pilot/provision/doctype/region/region.json index 630d6b4..c08f57a 100644 --- a/pilot/provision/doctype/region/region.json +++ b/pilot/provision/doctype/region/region.json @@ -7,7 +7,17 @@ "engine": "InnoDB", "field_order": [ "title", - "cloud_provider" + "cloud_provider", + "status", + "column_break_quxt", + "region_slug", + "provision", + "destroy", + "credentials_section", + "access_token", + "networking_section", + "vpc_cidr_block", + "vpc_id" ], "fields": [ { @@ -30,11 +40,82 @@ "options": "Cloud Provider", "reqd": 1, "set_only_once": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "label": "Status", + "options": "Draft\nPending\nActive\nArchived" + }, + { + "fieldname": "networking_section", + "fieldtype": "Section Break", + "label": "Networking" + }, + { + "fieldname": "vpc_cidr_block", + "fieldtype": "Data", + "label": "VPC CIDR Block" + }, + { + "fieldname": "vpc_id", + "fieldtype": "Data", + "label": "VPC ID" + }, + { + "fieldname": "credentials_section", + "fieldtype": "Section Break", + "label": "Credentials" + }, + { + "fieldname": "access_token", + "fieldtype": "Password", + "label": "Access Token" + }, + { + "fieldname": "column_break_quxt", + "fieldtype": "Column Break" + }, + { + "fieldname": "provision", + "fieldtype": "Button", + "label": "Provision", + "options": "provision" + }, + { + "fieldname": "region_slug", + "fieldtype": "Data", + "label": "Region Slug", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "destroy", + "fieldtype": "Button", + "label": "Destroy", + "options": "destroy" } ], "index_web_pages_for_search": 1, - "links": [], - "modified": "2024-07-18 15:42:03.445128", + "links": [ + { + "link_doctype": "Provision Action", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision Declaration", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision Plan", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision State", + "link_fieldname": "region" + } + ], + "modified": "2024-08-09 11:54:46.764057", "modified_by": "Administrator", "module": "Provision", "name": "Region", diff --git a/pilot/provision/doctype/region/region.py b/pilot/provision/doctype/region/region.py index e094d96..e17ccf5 100644 --- a/pilot/provision/doctype/region/region.py +++ b/pilot/provision/doctype/region/region.py @@ -1,9 +1,11 @@ # Copyright (c) 2024, Frappe and contributors # For license information, please see license.txt -# import frappe +import frappe from frappe.model.document import Document +from pilot.provision.opentofu import OpenTofu + class Region(Document): # begin: auto-generated types @@ -16,10 +18,22 @@ class Region(Document): access_token: DF.Password | None cloud_provider: DF.Link + region_slug: DF.Data status: DF.Literal["Draft", "Pending", "Active", "Archived"] title: DF.Data vpc_cidr_block: DF.Data | None vpc_id: DF.Data | None # end: auto-generated types - pass + @frappe.whitelist() + def provision(self) -> None: + OpenTofu(self).provision() + + @frappe.whitelist() + def destroy(self) -> None: + OpenTofu(self).destroy() + + def on_trash(self) -> None: + zones = frappe.get_all("Zone", filters={"region": self.name}) + for zone in zones: + frappe.delete_doc("Zone", zone.name) diff --git a/pilot/provision/opentofu.py b/pilot/provision/opentofu.py new file mode 100644 index 0000000..36286d3 --- /dev/null +++ b/pilot/provision/opentofu.py @@ -0,0 +1,179 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +import json +import os +import subprocess +from typing import TYPE_CHECKING + +import frappe +from cdktf import App, LocalBackend, TerraformStack +from constructs import Construct + +from pilot.provision.doctype.provision_declaration.provision_declaration import ProvisionDeclaration +from pilot.provision.doctype.provision_plan.provision_plan import ProvisionPlan +from pilot.provision.doctype.provision_state.provision_state import ProvisionState +from pilot.provision.providers.digitalocean import DigitalOcean + +if TYPE_CHECKING: + from pilot.provision.doctype.region.region import Region + + +STACKS_DIRECTORY = "stacks" +OPENTOFU_BINARY = "tofu" + + +class PilotStack(TerraformStack): + def __init__(self, scope: Construct, name: str, region: "Region"): + super().__init__(scope, name) + + # Backend file by default is placed at + # Which happens to be `sites/terraform..tfstate` + # This moves it to the stack directory + directory = os.path.abspath(frappe.get_site_path(STACKS_DIRECTORY)) + stack_directory = os.path.join(directory, "stacks", region.name) + backend_file = os.path.join(stack_directory, "terraform.tfstate") + LocalBackend(self, path=backend_file) + + DigitalOcean().provision(self, scope, name, region) + + +class OpenTofu: + def __init__(self, region: "Region") -> None: + self.region = region + self.directory = os.path.abspath(frappe.get_site_path(STACKS_DIRECTORY)) + self.stack_directory = os.path.join(self.directory, "stacks", self.region.name) + self.tofu = TofuCLI(self.stack_directory, region) + + def provision(self) -> None: + self.synth() + self.sync() + self.init() + self.plan() + self.deploy() + self.sync() + + def destroy(self) -> None: + self.sync() + self._destroy() + self.sync() + + def synth(self) -> ProvisionDeclaration: + # Creates sites//stacks directory on first run + app = App(outdir=self.directory) + PilotStack(app, self.region.name, self.region) + app.synth() + synth_file = os.path.join(self.directory, "stacks", self.region.name, "cdk.tf.json") + with open(synth_file) as f: + declaration = f.read() + return self.create_declaration(declaration) + + def sync(self) -> ProvisionState: + # We might not have any state to show in the beginning + state, pretty_state = "{}", "" + try: + state = self.tofu.show() + pretty_state = self.tofu.pretty_show() + except Exception: + pass + return self.create_state(state, pretty_state) + + def init(self) -> str: + # Creates the .terraform directory + return self.tofu.init() + + def plan(self) -> ProvisionPlan: + self.tofu.plan("tf.plan") + plan = self.tofu.show("tf.plan") + pretty_plan = self.tofu.pretty_show("tf.plan") + return self.create_plan(plan, pretty_plan) + + def deploy(self) -> str: + return self.tofu.apply("tf.plan") + + def _destroy(self) -> str: + return self.tofu.destroy() + + def create_declaration(self, declaration: str) -> ProvisionDeclaration: + return frappe.new_doc( + "Provision Declaration", region=self.region.name, stack=self.region.name, declaration=declaration + ).insert() + + def create_plan(self, plan: str, pretty_plan: str) -> ProvisionPlan: + return frappe.new_doc( + "Provision Plan", + region=self.region.name, + stack=self.region.name, + plan=json.dumps(json.loads(plan), indent=2), + pretty_plan=pretty_plan, + ).insert() + + def create_state(self, state: str, pretty_state: str) -> ProvisionState: + return frappe.new_doc( + "Provision State", + region=self.region.name, + stack=self.region.name, + state=json.dumps(json.loads(state), indent=2), + pretty_state=pretty_state, + ).insert() + + +class TofuCLI: + def __init__(self, path, region) -> None: + self.path = path + self.region = region + + def run(self, command) -> str: + command = [OPENTOFU_BINARY, *command] + process = subprocess.run(command, cwd=self.path, capture_output=True) + output = process.stdout.decode() + error = process.stderr.decode() + + parsed_output = self.parse_output(output) + frappe.new_doc( + "Provision Action", + region=self.region.name, + stack=self.region.name, + action=command[1], + output=output, + error=error, + parsed_output=parsed_output, + ).insert() + return output + + def init(self) -> str: + return self.run(["init", "-json"]) + + def plan(self, file) -> str: + return self.run(["plan", "-json", "-out", file]) + + def apply(self, file) -> str: + return self.run(["apply", "-json", file]) + + def show(self, file: str | None = None) -> str: + if file is None: + return self.run(["show", "-json"]) + else: + return self.run(["show", "-json", file]) + + def pretty_show(self, file: str | None = None) -> str: + if file is None: + return self.run(["show", "-no-color"]) + else: + return self.run(["show", file, "-no-color"]) + + def destroy(self) -> str: + return self.run(["destroy", "-json", "-auto-approve"]) + + def parse_output(self, output: str) -> str: + # Parse the output of the tofu command + # Return the @message field on all lines + parsed_lines = [] + for line in output.split("\n"): + if not line: + continue + try: + parsed_lines.append(json.loads(line)["@message"]) + except Exception: + pass + return "\n".join(parsed_lines) diff --git a/pilot/provision/providers/digitalocean.py b/pilot/provision/providers/digitalocean.py new file mode 100644 index 0000000..07f9c28 --- /dev/null +++ b/pilot/provision/providers/digitalocean.py @@ -0,0 +1,66 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt +from cdktf import Fn +from cdktf_cdktf_provider_digitalocean.droplet import Droplet +from cdktf_cdktf_provider_digitalocean.project import Project +from cdktf_cdktf_provider_digitalocean.provider import DigitaloceanProvider +from cdktf_cdktf_provider_digitalocean.volume import Volume +from cdktf_cdktf_provider_digitalocean.volume_attachment import VolumeAttachment +from cdktf_cdktf_provider_digitalocean.vpc import Vpc +from constructs import Construct + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pilot.provision.doctype.region.region import Region + from pilot.provision.opentofu import PilotStack + + +class DigitalOcean: + def provision(self, stack: "PilotStack", scope: Construct, name: str, region: "Region") -> None: + DigitaloceanProvider(stack, "digitalocean", token=region.get_password("access_token")) + vpc = Vpc( + stack, + f"{name}_vpc", + name=f"{name}-vpc-1", + region=region.region_slug, + ip_range=region.vpc_cidr_block, + ) + + droplet = Droplet( + stack, + f"{name}_droplet", + image="ubuntu-24-04-x64", + name=f"{name}-droplet-1", + region=region.region_slug, + size="s-1vcpu-1gb", + vpc_uuid=vpc.id, + ssh_keys=["39020628"], + ) + + volume = Volume( + stack, + f"{name}_volume", + region=region.region_slug, + name=f"{name}-volume-1", + size=10, + initial_filesystem_type="ext4", + description="an example volume", + ) + + VolumeAttachment( + stack, + f"{name}_volume_attachment", + droplet_id=Fn.tonumber(droplet.id), + volume_id=volume.id, + ) + + Project( + stack, + f"{name}_project", + name="Pilot Tofu Playground", + description="Project for playing around with Tofu", + purpose="Web Application", + environment="Development", + resources=[droplet.urn], + )