diff --git a/pilot/bootstrap/__init__.py b/pilot/bootstrap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pilot/pilot/design/concepts.md b/pilot/pilot/design/concepts.md new file mode 100644 index 0000000..3522caf --- /dev/null +++ b/pilot/pilot/design/concepts.md @@ -0,0 +1,27 @@ +# Concepts +--- +## Global + +### Region +Inter Region Latencies +10 - 100 +Maybe 300ms + + +### Availability Zone +< 10 ms +~3 ms + +Some components are only available in the same Zone +- AWS EBS volume +- Subnet + +### Node + + +--- + +## Orchestration + +### Allocation +An Allocation is a mapping between a task group in a job and a client node. A single job may have hundreds or thousands of task groups, meaning an equivalent number of allocations must exist to map the work to client machines. Allocations are created by the Nomad servers as part of scheduling decisions made during an evaluation. \ No newline at end of file diff --git a/pilot/pilot/design/general.md b/pilot/pilot/design/general.md new file mode 100644 index 0000000..4d75bfe --- /dev/null +++ b/pilot/pilot/design/general.md @@ -0,0 +1,24 @@ +Single Global Control Plane + +e.g. +- AWS Dashboard +- DO Dashboard + +For handling Auth / Teams / Billing Centrally + +### Regions +#### Authority Region +There's still single Primary +Maybe Control Plane resides here + +#### Isolated +No shared fault domain +Failures don't impact each other + +#### Autonomous ?!? + + +#### Minimal Communication +Data usually doesn't leave the Region. +Public Internet is slow and Expensive + diff --git a/pilot/pilot/design/influences.md b/pilot/pilot/design/influences.md new file mode 100644 index 0000000..8f65d8c --- /dev/null +++ b/pilot/pilot/design/influences.md @@ -0,0 +1,19 @@ +# Nomad +Single Binary +Node +- Drivers +- VMs + - KVM + - FireCracker +- Containers + - Podman + - Docker + - Rkt? +- Metadata + +Self Registration + +Federation + +# + diff --git a/pilot/pilot/design/objects.md b/pilot/pilot/design/objects.md new file mode 100644 index 0000000..b35ce83 --- /dev/null +++ b/pilot/pilot/design/objects.md @@ -0,0 +1,39 @@ +Region + +Cluster?!?! VPC ?!!?! Smaller Blast Redius + +Zone +- Name +- Subnet + +Node +- Name / UUID +- Version +- Status +- Eligible? +- Drained? +- Address + + +- Pool? +- Zone +- Drivers (KVM, Firecracker, Doker, Podman) + - Status + - Network (Bridge) + - Version + - Runtime? + +- Resources (RAM, CPU) + +Attributes +- OS +- Arch +- CPU / Cores / Freq + + +Node Events + + +Client??!?! +Server?!?! + diff --git a/pilot/pilot/design/provision.md b/pilot/pilot/design/provision.md new file mode 100644 index 0000000..5234b3e --- /dev/null +++ b/pilot/pilot/design/provision.md @@ -0,0 +1,53 @@ +# Provision +There's no way to do something like tofu.deploy() right now. + +This goes through two levels of abstractions?. JSII + +### CDKTF +We write TerraformStack subclass and define our configration in init. It should something like this +```python +from cdktf import App, Fn, TerraformStack, TerraformVariable +from cdktf_cdktf_provider_digitalocean.provider import DigitaloceanProvider +from cdktf_cdktf_provider_digitalocean.vpc import Vpc +from constructs import Construct + + +class MyStack(TerraformStack): + def __init__(self, scope: Construct, id: str): + super().__init__(scope, id) + do_token_variable = TerraformVariable(self,"do_token", type="string") + DigitaloceanProvider(self, "digitalocean", token=do_token_variable.string_value) + vpc = Vpc(self, "example_vpc", name="vpc-1", region="blr-1", ip_range="ip_range") + +``` + +Unfortunately, Pilot config isn't static. But good news is, this is actually implemented as following + +1. Define TerraformStack subclass, like we did before +This is equivalent of writing a HCL file + +2. Define an app and call `synth` on it + +```python +from cdktf import App, Fn, TerraformStack, TerraformVariable + + +app = App() +MyStack(app, "cdktf-demo") +app.synth() +``` + + +3. Apply generated plan +`cdktf deploy` +We can open up the implemntation and see what happens underneath. + +1. If the implementation is complicated then we can run `cdktf deploy` ourselves + + +--- + +We need to put some dynamic logic on Step 1. +We can't write a class everytime. +Generating Python code is basically the same as writing HCL +What we can do is build a class implementation and app implementation on the fly diff --git a/pilot/pilot/design/workloads.md b/pilot/pilot/design/workloads.md new file mode 100644 index 0000000..8205f60 --- /dev/null +++ b/pilot/pilot/design/workloads.md @@ -0,0 +1,7 @@ +# System +DaemonSet + +Run on all (or matching nodes at all times) + +# Service +Typical Container \ No newline at end of file diff --git a/pilot/provision/README.wtf.md b/pilot/provision/README.wtf.md new file mode 100644 index 0000000..dbd28fd --- /dev/null +++ b/pilot/provision/README.wtf.md @@ -0,0 +1,40 @@ +We want to do as much work in Python / JSON as possible + +Luckily Tofu helps out. + +Every plan / state can be stored as JSON or converted to JSON. + +Here's the rough idea + +```mermaid +stateDiagram + code: Python Declaration + plan: Plan + json: Synthesized JSON + infra: Infrastructure + code --> json: app.synth() + json --> plan: tofu plan + plan --> infra: tofu deploy +``` + + +Same thing in words + +1. Write Python code in `TerraformStack.__init__()` that describes the infra we need +2. "Synthesize" this TerraformStack object `app.synth()` + +The synth actually executes the `__init__` so we can do whatever we want in Python (loops, conditionals etc) + +`synth` generates a `cdktf.out/stacks//cdktf.json` file in the working directory (this is `frappe-bench/sites`) + +We will use `sites//stacks` for now. So the state moves with the site without any special handling. + +TODO: Include this directory in the file backups. + +3. Store this synthesized JSON in some DocType Provision + + +Note: Our Stack can have bugs or the code that defines what we need can have bugs. Have a way to prevent catastrophies at this stage. We need sanity checks in Production to guard against +- Don't trigger anything that can cause data loss +- Don't trigger massive changes ( More than n resources at a time) +- Cross stack changes ?! (Don't delete someone other region?!) diff --git a/pilot/provision/bootstrap.py b/pilot/provision/bootstrap.py new file mode 100644 index 0000000..9d5c4d2 --- /dev/null +++ b/pilot/provision/bootstrap.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +import frappe + +PROVIDER = "do" +DO_ACCESS_TOKEN = "" +CIDR = "10.10.0.0/24" +REGION = "blr1" +TITLE = "Bangalore" +NAME = "do-bangalore-blr1" + + +def create(): + if frappe.db.exists("Region", NAME): + region = frappe.get_doc("Region", NAME) + else: + region = frappe.new_doc( + "Region", + **{ + "name": NAME, + "access_token": DO_ACCESS_TOKEN, + "cloud_provider": PROVIDER, + "region_slug": REGION, + "title": TITLE, + "vpc_cidr_block": CIDR, + }, + ).insert() + + region.provision() + + +def destroy(): + if frappe.db.exists("Region", NAME): + region = frappe.get_doc("Region", NAME) + region.destroy() + + +def clear(): + doctypes = ["Provision Declaration", "Provision Plan", "Provision State", "Provision Action"] + for doctype in doctypes: + frappe.db.delete(doctype) diff --git a/pilot/provision/doctype/provision_action/__init__.py b/pilot/provision/doctype/provision_action/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pilot/provision/doctype/provision_action/provision_action.js b/pilot/provision/doctype/provision_action/provision_action.js new file mode 100644 index 0000000..c868618 --- /dev/null +++ b/pilot/provision/doctype/provision_action/provision_action.js @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Provision Action", { +// refresh(frm) { + +// }, +// }); diff --git a/pilot/provision/doctype/provision_action/provision_action.json b/pilot/provision/doctype/provision_action/provision_action.json new file mode 100644 index 0000000..75321b9 --- /dev/null +++ b/pilot/provision/doctype/provision_action/provision_action.json @@ -0,0 +1,105 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2024-07-29 11:26:21.416387", + "doctype": "DocType", + "engine": "MyISAM", + "field_order": [ + "region", + "stack", + "column_break_payx", + "action", + "section_break_tioa", + "parsed_output", + "output", + "error" + ], + "fields": [ + { + "fieldname": "stack", + "fieldtype": "Data", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Stack", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "action", + "fieldtype": "Data", + "in_list_view": 1, + "in_preview": 1, + "in_standard_filter": 1, + "label": "Action", + "read_only": 1, + "reqd": 1 + }, + { + "default": "{}", + "fieldname": "output", + "fieldtype": "Code", + "label": "Output", + "read_only": 1 + }, + { + "fieldname": "error", + "fieldtype": "Code", + "label": "Error", + "read_only": 1 + }, + { + "fieldname": "region", + "fieldtype": "Link", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Region", + "options": "Region", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "column_break_payx", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_tioa", + "fieldtype": "Section Break" + }, + { + "default": "{}", + "fieldname": "parsed_output", + "fieldtype": "Code", + "label": "Parsed Output", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-08-09 16:33:22.536559", + "modified_by": "Administrator", + "module": "Provision", + "name": "Provision Action", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "show_title_field_in_link": 1, + "sort_field": "creation", + "sort_order": "DESC", + "states": [], + "title_field": "stack" +} \ No newline at end of file diff --git a/pilot/provision/doctype/provision_action/provision_action.py b/pilot/provision/doctype/provision_action/provision_action.py new file mode 100644 index 0000000..825e872 --- /dev/null +++ b/pilot/provision/doctype/provision_action/provision_action.py @@ -0,0 +1,26 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class ProvisionAction(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + action: DF.Data + error: DF.Code | None + name: DF.Int | None + output: DF.Code | None + parsed_output: DF.Code | None + region: DF.Link + stack: DF.Data + # end: auto-generated types + + pass diff --git a/pilot/provision/doctype/provision_action/test_provision_action.py b/pilot/provision/doctype/provision_action/test_provision_action.py new file mode 100644 index 0000000..cdd7aa0 --- /dev/null +++ b/pilot/provision/doctype/provision_action/test_provision_action.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestProvisionAction(FrappeTestCase): + pass diff --git a/pilot/provision/doctype/provision_declaration/__init__.py b/pilot/provision/doctype/provision_declaration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pilot/provision/doctype/provision_declaration/provision_declaration.js b/pilot/provision/doctype/provision_declaration/provision_declaration.js new file mode 100644 index 0000000..a528543 --- /dev/null +++ b/pilot/provision/doctype/provision_declaration/provision_declaration.js @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Provision Declaration", { +// refresh(frm) { + +// }, +// }); diff --git a/pilot/provision/doctype/provision_declaration/provision_declaration.json b/pilot/provision/doctype/provision_declaration/provision_declaration.json new file mode 100644 index 0000000..1696d0e --- /dev/null +++ b/pilot/provision/doctype/provision_declaration/provision_declaration.json @@ -0,0 +1,72 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2024-07-24 13:15:13.965380", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "region", + "stack", + "declaration" + ], + "fields": [ + { + "default": "{}", + "fieldname": "declaration", + "fieldtype": "Code", + "in_list_view": 1, + "label": "Declaration", + "options": "JSON", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "stack", + "fieldtype": "Data", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Stack", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "region", + "fieldtype": "Link", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Region", + "options": "Region", + "read_only": 1, + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-08-09 11:54:20.677181", + "modified_by": "Administrator", + "module": "Provision", + "name": "Provision Declaration", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "show_title_field_in_link": 1, + "sort_field": "creation", + "sort_order": "DESC", + "states": [], + "title_field": "stack" +} \ No newline at end of file diff --git a/pilot/provision/doctype/provision_declaration/provision_declaration.py b/pilot/provision/doctype/provision_declaration/provision_declaration.py new file mode 100644 index 0000000..6dba486 --- /dev/null +++ b/pilot/provision/doctype/provision_declaration/provision_declaration.py @@ -0,0 +1,23 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class ProvisionDeclaration(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + declaration: DF.Code + name: DF.Int | None + region: DF.Link + stack: DF.Data + # end: auto-generated types + + pass diff --git a/pilot/provision/doctype/provision_declaration/test_provision_declaration.py b/pilot/provision/doctype/provision_declaration/test_provision_declaration.py new file mode 100644 index 0000000..d873034 --- /dev/null +++ b/pilot/provision/doctype/provision_declaration/test_provision_declaration.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestProvisionDeclaration(FrappeTestCase): + pass diff --git a/pilot/provision/doctype/provision_plan/__init__.py b/pilot/provision/doctype/provision_plan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pilot/provision/doctype/provision_plan/provision_plan.js b/pilot/provision/doctype/provision_plan/provision_plan.js new file mode 100644 index 0000000..6a5cfff --- /dev/null +++ b/pilot/provision/doctype/provision_plan/provision_plan.js @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Provision Plan", { +// refresh(frm) { + +// }, +// }); diff --git a/pilot/provision/doctype/provision_plan/provision_plan.json b/pilot/provision/doctype/provision_plan/provision_plan.json new file mode 100644 index 0000000..d743608 --- /dev/null +++ b/pilot/provision/doctype/provision_plan/provision_plan.json @@ -0,0 +1,79 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2024-07-24 00:23:10.407887", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "region", + "stack", + "pretty_plan", + "plan" + ], + "fields": [ + { + "default": "{}", + "fieldname": "plan", + "fieldtype": "Code", + "label": "Plan", + "options": "JSON", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "stack", + "fieldtype": "Data", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Stack", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "region", + "fieldtype": "Link", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Region", + "options": "Region", + "read_only": 1, + "reqd": 1, + "search_index": 1 + }, + { + "fieldname": "pretty_plan", + "fieldtype": "Code", + "label": "Pretty Plan", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-08-09 16:33:31.128936", + "modified_by": "Administrator", + "module": "Provision", + "name": "Provision Plan", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "show_title_field_in_link": 1, + "sort_field": "creation", + "sort_order": "DESC", + "states": [], + "title_field": "stack" +} \ No newline at end of file diff --git a/pilot/provision/doctype/provision_plan/provision_plan.py b/pilot/provision/doctype/provision_plan/provision_plan.py new file mode 100644 index 0000000..c1118d3 --- /dev/null +++ b/pilot/provision/doctype/provision_plan/provision_plan.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class ProvisionPlan(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + name: DF.Int | None + plan: DF.Code + pretty_plan: DF.Code | None + region: DF.Link + stack: DF.Data + # end: auto-generated types + + pass diff --git a/pilot/provision/doctype/provision_plan/test_provision_plan.py b/pilot/provision/doctype/provision_plan/test_provision_plan.py new file mode 100644 index 0000000..b8b3af7 --- /dev/null +++ b/pilot/provision/doctype/provision_plan/test_provision_plan.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestProvisionPlan(FrappeTestCase): + pass diff --git a/pilot/provision/doctype/provision_state/__init__.py b/pilot/provision/doctype/provision_state/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pilot/provision/doctype/provision_state/provision_state.js b/pilot/provision/doctype/provision_state/provision_state.js new file mode 100644 index 0000000..38865b8 --- /dev/null +++ b/pilot/provision/doctype/provision_state/provision_state.js @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Provision State", { +// refresh(frm) { + +// }, +// }); diff --git a/pilot/provision/doctype/provision_state/provision_state.json b/pilot/provision/doctype/provision_state/provision_state.json new file mode 100644 index 0000000..6ab9b95 --- /dev/null +++ b/pilot/provision/doctype/provision_state/provision_state.json @@ -0,0 +1,78 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-07-24 22:52:36.927492", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "region", + "stack", + "pretty_state", + "state" + ], + "fields": [ + { + "fieldname": "stack", + "fieldtype": "Data", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Stack", + "read_only": 1, + "reqd": 1 + }, + { + "default": "{}", + "fieldname": "state", + "fieldtype": "Code", + "label": "State", + "options": "JSON", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "region", + "fieldtype": "Link", + "in_filter": 1, + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Region", + "options": "Region", + "read_only": 1, + "reqd": 1, + "search_index": 1 + }, + { + "fieldname": "pretty_state", + "fieldtype": "Code", + "label": "Pretty State", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-08-09 16:20:02.612683", + "modified_by": "Administrator", + "module": "Provision", + "name": "Provision State", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "show_title_field_in_link": 1, + "sort_field": "creation", + "sort_order": "DESC", + "states": [], + "title_field": "stack" +} \ No newline at end of file diff --git a/pilot/provision/doctype/provision_state/provision_state.py b/pilot/provision/doctype/provision_state/provision_state.py new file mode 100644 index 0000000..2f632b4 --- /dev/null +++ b/pilot/provision/doctype/provision_state/provision_state.py @@ -0,0 +1,23 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class ProvisionState(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + pretty_state: DF.Code | None + region: DF.Link + stack: DF.Data + state: DF.Code + # end: auto-generated types + + pass diff --git a/pilot/provision/doctype/provision_state/test_provision_state.py b/pilot/provision/doctype/provision_state/test_provision_state.py new file mode 100644 index 0000000..f92cd7f --- /dev/null +++ b/pilot/provision/doctype/provision_state/test_provision_state.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestProvisionState(FrappeTestCase): + pass diff --git a/pilot/provision/doctype/region/region.json b/pilot/provision/doctype/region/region.json index 630d6b4..c08f57a 100644 --- a/pilot/provision/doctype/region/region.json +++ b/pilot/provision/doctype/region/region.json @@ -7,7 +7,17 @@ "engine": "InnoDB", "field_order": [ "title", - "cloud_provider" + "cloud_provider", + "status", + "column_break_quxt", + "region_slug", + "provision", + "destroy", + "credentials_section", + "access_token", + "networking_section", + "vpc_cidr_block", + "vpc_id" ], "fields": [ { @@ -30,11 +40,82 @@ "options": "Cloud Provider", "reqd": 1, "set_only_once": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "label": "Status", + "options": "Draft\nPending\nActive\nArchived" + }, + { + "fieldname": "networking_section", + "fieldtype": "Section Break", + "label": "Networking" + }, + { + "fieldname": "vpc_cidr_block", + "fieldtype": "Data", + "label": "VPC CIDR Block" + }, + { + "fieldname": "vpc_id", + "fieldtype": "Data", + "label": "VPC ID" + }, + { + "fieldname": "credentials_section", + "fieldtype": "Section Break", + "label": "Credentials" + }, + { + "fieldname": "access_token", + "fieldtype": "Password", + "label": "Access Token" + }, + { + "fieldname": "column_break_quxt", + "fieldtype": "Column Break" + }, + { + "fieldname": "provision", + "fieldtype": "Button", + "label": "Provision", + "options": "provision" + }, + { + "fieldname": "region_slug", + "fieldtype": "Data", + "label": "Region Slug", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "destroy", + "fieldtype": "Button", + "label": "Destroy", + "options": "destroy" } ], "index_web_pages_for_search": 1, - "links": [], - "modified": "2024-07-18 15:42:03.445128", + "links": [ + { + "link_doctype": "Provision Action", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision Declaration", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision Plan", + "link_fieldname": "region" + }, + { + "link_doctype": "Provision State", + "link_fieldname": "region" + } + ], + "modified": "2024-08-09 11:54:46.764057", "modified_by": "Administrator", "module": "Provision", "name": "Region", diff --git a/pilot/provision/doctype/region/region.py b/pilot/provision/doctype/region/region.py index e094d96..e17ccf5 100644 --- a/pilot/provision/doctype/region/region.py +++ b/pilot/provision/doctype/region/region.py @@ -1,9 +1,11 @@ # Copyright (c) 2024, Frappe and contributors # For license information, please see license.txt -# import frappe +import frappe from frappe.model.document import Document +from pilot.provision.opentofu import OpenTofu + class Region(Document): # begin: auto-generated types @@ -16,10 +18,22 @@ class Region(Document): access_token: DF.Password | None cloud_provider: DF.Link + region_slug: DF.Data status: DF.Literal["Draft", "Pending", "Active", "Archived"] title: DF.Data vpc_cidr_block: DF.Data | None vpc_id: DF.Data | None # end: auto-generated types - pass + @frappe.whitelist() + def provision(self) -> None: + OpenTofu(self).provision() + + @frappe.whitelist() + def destroy(self) -> None: + OpenTofu(self).destroy() + + def on_trash(self) -> None: + zones = frappe.get_all("Zone", filters={"region": self.name}) + for zone in zones: + frappe.delete_doc("Zone", zone.name) diff --git a/pilot/provision/opentofu.py b/pilot/provision/opentofu.py new file mode 100644 index 0000000..e108a68 --- /dev/null +++ b/pilot/provision/opentofu.py @@ -0,0 +1,178 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +import json +import os +import subprocess + +import frappe +from cdktf import App, LocalBackend, TerraformStack +from constructs import Construct + +from pilot.provision.doctype.provision_declaration.provision_declaration import ProvisionDeclaration +from pilot.provision.doctype.provision_plan.provision_plan import ProvisionPlan +from pilot.provision.doctype.provision_state.provision_state import ProvisionState +from pilot.provision.providers.digitalocean import DigitalOcean + +STACKS_DIRECTORY = "stacks" +OPENTOFU_BINARY = "tofu" + + +class PilotStack(TerraformStack): + from pilot.provision.doctype.region.region import Region + + def __init__(self, scope: Construct, name: str, region: "Region"): + super().__init__(scope, name) + + # Backend file by default is placed at + # Which happens to be `sites/terraform..tfstate` + # This moves it to the stack directory + directory = os.path.abspath(frappe.get_site_path(STACKS_DIRECTORY)) + stack_directory = os.path.join(directory, "stacks", region.name) + backend_file = os.path.join(stack_directory, "terraform.tfstate") + LocalBackend(self, path=backend_file) + + DigitalOcean().provision(self, scope, name, region) + + +class OpenTofu: + from pilot.provision.doctype.region.region import Region + + def __init__(self, region: "Region") -> None: + self.region = region + self.directory = os.path.abspath(frappe.get_site_path(STACKS_DIRECTORY)) + self.stack_directory = os.path.join(self.directory, "stacks", self.region.name) + self.tofu = TofuCLI(self.stack_directory, region) + + def provision(self) -> None: + self.synth() + self.sync() + self.init() + self.plan() + self.deploy() + self.sync() + + def destroy(self) -> None: + self.sync() + self._destroy() + self.sync() + + def synth(self) -> ProvisionDeclaration: + # Creates sites//stacks directory on first run + app = App(outdir=self.directory) + PilotStack(app, self.region.name, self.region) + app.synth() + synth_file = os.path.join(self.directory, "stacks", self.region.name, "cdk.tf.json") + with open(synth_file) as f: + declaration = f.read() + return self.create_declaration(declaration) + + def sync(self) -> ProvisionState: + # We might not have any state to show in the beginning + state, pretty_state = "{}", "" + try: + state = self.tofu.show() + pretty_state = self.tofu.pretty_show() + except Exception: + pass + return self.create_state(state, pretty_state) + + def init(self) -> str: + # Creates the .terraform directory + return self.tofu.init() + + def plan(self) -> ProvisionPlan: + self.tofu.plan("tf.plan") + plan = self.tofu.show("tf.plan") + pretty_plan = self.tofu.pretty_show("tf.plan") + return self.create_plan(plan, pretty_plan) + + def deploy(self) -> str: + return self.tofu.apply("tf.plan") + + def _destroy(self) -> str: + return self.tofu.destroy() + + def create_declaration(self, declaration: str) -> ProvisionDeclaration: + return frappe.new_doc( + "Provision Declaration", region=self.region.name, stack=self.region.name, declaration=declaration + ).insert() + + def create_plan(self, plan: str, pretty_plan: str) -> ProvisionPlan: + return frappe.new_doc( + "Provision Plan", + region=self.region.name, + stack=self.region.name, + plan=json.dumps(json.loads(plan), indent=2), + pretty_plan=pretty_plan, + ).insert() + + def create_state(self, state: str, pretty_state: str) -> ProvisionState: + return frappe.new_doc( + "Provision State", + region=self.region.name, + stack=self.region.name, + state=json.dumps(json.loads(state), indent=2), + pretty_state=pretty_state, + ).insert() + + +class TofuCLI: + def __init__(self, path, region) -> None: + self.path = path + self.region = region + + def run(self, command) -> str: + command = [OPENTOFU_BINARY, *command] + process = subprocess.run(command, cwd=self.path, capture_output=True) + output = process.stdout.decode() + error = process.stderr.decode() + + parsed_output = self.parse_output(output) + frappe.new_doc( + "Provision Action", + region=self.region.name, + stack=self.region.name, + action=command[1], + output=output, + error=error, + parsed_output=parsed_output, + ).insert() + return output + + def init(self) -> str: + return self.run(["init", "-json"]) + + def plan(self, file) -> str: + return self.run(["plan", "-json", "-out", file]) + + def apply(self, file) -> str: + return self.run(["apply", "-json", file]) + + def show(self, file: str | None = None) -> str: + if file is None: + return self.run(["show", "-json"]) + else: + return self.run(["show", "-json", file]) + + def pretty_show(self, file: str | None = None) -> str: + if file is None: + return self.run(["show", "-no-color"]) + else: + return self.run(["show", file, "-no-color"]) + + def destroy(self) -> str: + return self.run(["destroy", "-json", "-auto-approve"]) + + def parse_output(self, output: str) -> str: + # Parse the output of the tofu command + # Return the @message field on all lines + parsed_lines = [] + for line in output.split("\n"): + if not line: + continue + try: + parsed_lines.append(json.loads(line)["@message"]) + except Exception: + pass + return "\n".join(parsed_lines) diff --git a/pilot/provision/providers/digitalocean.py b/pilot/provision/providers/digitalocean.py new file mode 100644 index 0000000..64b1ae3 --- /dev/null +++ b/pilot/provision/providers/digitalocean.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt +from cdktf import Fn +from cdktf_cdktf_provider_digitalocean.droplet import Droplet +from cdktf_cdktf_provider_digitalocean.project import Project +from cdktf_cdktf_provider_digitalocean.provider import DigitaloceanProvider +from cdktf_cdktf_provider_digitalocean.volume import Volume +from cdktf_cdktf_provider_digitalocean.volume_attachment import VolumeAttachment +from cdktf_cdktf_provider_digitalocean.vpc import Vpc +from constructs import Construct + + +class DigitalOcean: + from pilot.provision.doctype.region.region import Region + from pilot.provision.opentofu import PilotStack + + def provision(self, stack: "PilotStack", scope: Construct, name: str, region: "Region") -> None: + DigitaloceanProvider(stack, "digitalocean", token=region.get_password("access_token")) + vpc = Vpc( + stack, + f"{name}_vpc", + name=f"{name}-vpc-1", + region=region.region_slug, + ip_range=region.vpc_cidr_block, + ) + + droplet = Droplet( + stack, + f"{name}_droplet", + image="ubuntu-24-04-x64", + name=f"{name}-droplet-1", + region=region.region_slug, + size="s-1vcpu-1gb", + vpc_uuid=vpc.id, + ssh_keys=["39020628"], + ) + + volume = Volume( + stack, + f"{name}_volume", + region=region.region_slug, + name=f"{name}-volume-1", + size=10, + initial_filesystem_type="ext4", + description="an example volume", + ) + + VolumeAttachment( + stack, + f"{name}_volume_attachment", + droplet_id=Fn.tonumber(droplet.id), + volume_id=volume.id, + ) + + Project( + stack, + f"{name}_project", + name="Pilot Tofu Playground", + description="Project for playing around with Tofu", + purpose="Web Application", + environment="Development", + resources=[droplet.urn], + )