DONOTMERGE: sketching PM and PM Pre-work

asdf.md

@@ -0,0 +1,43 @@
+
+
+
+- [x] Tidy migration 
+
+- [ ] Fully kill the idea of user types, consts, fns
+      Everything should be in package space.
+
+- [ ] Split up "handlers" into separate tables
+
+- [ ] stop referring to package thigns by name. refer to them by ID instead.
+  - [ ] how do we do that for
+
+
+- [ ] Review Apiserver endpoints
+- [ ] Add endpoints for package stuff
+- [ ] Write the dark editor canvas as if we have everything (sketch all endpoints, dbs, etc), comment a lot of it out 
+- [ ] Sketch out new built-ins
+- [ ] reconsider if we need ExternalTypesToPrgoramTypes and Types... or if we should just use ProgramTypes
+
+
+tables
+	Types
+	Values
+	Functions
+
+	Even a type and a function are values, though...
+
+	Types are values of the type type
+	Functions are values of the function type
+	Http handlers are values of the HttpHandler type
+	Crons?
+	Workers?
+	How do traces fit in?
+	Hmmm
+
+
+
+
+
+Design matter darklang com
+	packages.darklang.com/Stachu
+	packages.darklang.com/

backend/migrations/20230709_000000_migration.sql

@@ -1,61 +1,22 @@
 CREATE TABLE IF NOT EXISTS
-accounts_v0
-( id UUID PRIMARY KEY
-, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+system_migrations_v0
+( name TEXT PRIMARY KEY
+, execution_date TIMESTAMPTZ NOT NULL
+, sql TEXT NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS
-canvases_v0
-( id UUID PRIMARY KEY
-, account_id UUID REFERENCES accounts_v0(id) NOT NULL
-, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-);
 
 CREATE TABLE IF NOT EXISTS
-cron_records_v0
-( id UUID PRIMARY KEY
-, tlid BIGINT NOT NULL
-, canvas_id UUID NOT NULL
-, ran_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -- default as it's cheap
-);
-
-CREATE INDEX IF NOT EXISTS
-idx_cron_records_tlid_canvas_id_id
-ON cron_records_v0
-(tlid, canvas_id, id DESC);
-
-
-CREATE TABLE IF NOT EXISTS
-domains_v0
-( domain TEXT PRIMARY KEY
-, canvas_id UUID NOT NULL
-, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW());
-
-
-CREATE TABLE IF NOT EXISTS
-queue_events_v0
+accounts_v0
 ( id UUID PRIMARY KEY
-, canvas_id UUID NOT NULL
-, module TEXT NOT NULL
-, name TEXT NOT NULL
-, modifier TEXT NOT NULL
-, locked_at TIMESTAMPTZ -- nullable
-, enqueued_at TIMESTAMPTZ NOT NULL
-, value TEXT NOT NULL
+, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );
 
--- We want to use this index to:
--- 1) count the number of items in this queue, so it's important that the entire
--- search term is in the index or it will need to hit disk. This is true even though
--- the module rarely changes
--- 2) fetch the indexes for all items we're unpausing. This is rare so it's fine to
-CREATE INDEX IF NOT EXISTS
-idx_queue_events_count
-ON queue_events_v0 (canvas_id, module, name);
-
-
+--------------------
+-- Stuff that belongs in "package space"
+--------------------
 CREATE TABLE IF NOT EXISTS
-package_functions_v0
+package_types_v0
 -- IDs
 ( id UUID PRIMARY KEY
 , tlid BIGINT NOT NULL -- includes TLID for tracing
@@ -66,7 +27,7 @@ package_functions_v0
 -- allow search by name
 , owner TEXT NOT NULL
 , modules TEXT NOT NULL /* eg Twitter.Other; includes package name, but not owner name */
-, fnname TEXT NOT NULL /* eg sendText */
+, typename TEXT NOT NULL /* eg sendText */
 , version INTEGER NOT NULL /* eg 0 */
 -- the actual definition
 , definition BYTEA NOT NULL /* the whole thing serialized as binary */
@@ -75,34 +36,33 @@ package_functions_v0
 );
 
 CREATE TABLE IF NOT EXISTS
-package_types_v0
+package_constants_v0
 -- IDs
 ( id UUID PRIMARY KEY
-, tlid BIGINT NOT NULL -- includes TLID for tracing
-  /* owner/namespace part of the string, eg dark.
-   * CLEANUP This isn't a good way to store this because the username should be
-   * stored in the editor canvas. But we haven't got all the details worked out so
-   * for now store the owner */
--- allow search by name
+, tlid BIGINT NOT NULL
 , owner TEXT NOT NULL
 , modules TEXT NOT NULL /* eg Twitter.Other; includes package name, but not owner name */
-, typename TEXT NOT NULL /* eg sendText */
+, name TEXT NOT NULL /* eg pi */
 , version INTEGER NOT NULL /* eg 0 */
 -- the actual definition
 , definition BYTEA NOT NULL /* the whole thing serialized as binary */
 -- bonus
 , created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );
 
-
 CREATE TABLE IF NOT EXISTS
-package_constants_v0
+package_functions_v0
 -- IDs
 ( id UUID PRIMARY KEY
-, tlid BIGINT NOT NULL
+, tlid BIGINT NOT NULL -- includes TLID for tracing
+  /* owner/namespace part of the string, eg dark.
+   * CLEANUP This isn't a good way to store this because the username should be
+   * stored in the editor canvas. But we haven't got all the details worked out so
+   * for now store the owner */
+-- allow search by name
 , owner TEXT NOT NULL
 , modules TEXT NOT NULL /* eg Twitter.Other; includes package name, but not owner name */
-, name TEXT NOT NULL /* eg pi */
+, fnname TEXT NOT NULL /* eg sendText */
 , version INTEGER NOT NULL /* eg 0 */
 -- the actual definition
 , definition BYTEA NOT NULL /* the whole thing serialized as binary */
@@ -111,6 +71,44 @@ package_constants_v0
 );
 
 
+--------------------
+-- Stuff that belongs in "user space"
+--------------------
+CREATE TABLE IF NOT EXISTS
+canvases_v0
+( id UUID PRIMARY KEY
+, account_id UUID REFERENCES accounts_v0(id) NOT NULL
+, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- HttpHandlers
+CREATE TABLE IF NOT EXISTS
+domains_v0
+( domain TEXT PRIMARY KEY
+, canvas_id UUID NOT NULL
+, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW());
+
+-- TODO: HttpHandlers
+
+
+-- Crons
+CREATE TABLE IF NOT EXISTS
+cron_records_v0
+( id UUID PRIMARY KEY
+, tlid BIGINT NOT NULL
+, canvas_id UUID NOT NULL
+, ran_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -- default as it's cheap
+);
+
+CREATE INDEX IF NOT EXISTS
+  idx_cron_records_tlid_canvas_id_id
+ON cron_records_v0
+  (tlid, canvas_id, id DESC);
+
+
+
+
+-- Queues / Workers
 CREATE TYPE scheduling_rule_type AS ENUM ('pause', 'block');
 
 CREATE TABLE IF NOT EXISTS
@@ -123,6 +121,31 @@ scheduling_rules_v0
 , created_at TIMESTAMPTZ NOT NULL DEFAULT now()
 );
 
+CREATE TABLE IF NOT EXISTS
+queue_events_v0
+( id UUID PRIMARY KEY
+, canvas_id UUID NOT NULL
+, module TEXT NOT NULL
+, name TEXT NOT NULL
+, modifier TEXT NOT NULL
+, locked_at TIMESTAMPTZ -- nullable
+, enqueued_at TIMESTAMPTZ NOT NULL
+, value TEXT NOT NULL
+);
+
+-- We want to use this index to:
+-- 1) count the number of items in this queue, so it's important that the entire
+-- search term is in the index or it will need to hit disk. This is true even though
+-- the module rarely changes
+-- 2) fetch the indexes for all items we're unpausing. This is rare so it's fine to
+CREATE INDEX IF NOT EXISTS
+  idx_queue_events_count
+ON
+  queue_events_v0 (canvas_id, module, name);
+
+
+
+
 
 CREATE TABLE IF NOT EXISTS
 secrets_v0
@@ -137,14 +160,6 @@ secrets_v0
 
 
 
-CREATE TABLE IF NOT EXISTS
-system_migrations_v0
-( name TEXT PRIMARY KEY
-, execution_date TIMESTAMPTZ NOT NULL
-, sql TEXT NOT NULL
-);
-
-
 
 CREATE TYPE toplevel_type AS
 ENUM ('handler', 'db', 'user_function', 'user_type', 'user_constant');
@@ -179,6 +194,7 @@ traces_v0
 );
 
 
+-- "User DB" data
 CREATE TABLE IF NOT EXISTS
 user_data_v0
 ( id UUID PRIMARY KEY
@@ -194,17 +210,100 @@ user_data_v0
 );
 
 CREATE INDEX IF NOT EXISTS
-idx_user_data_fetch
+  idx_user_data_fetch
 ON user_data_v0
-(canvas_id, table_tlid, user_version, dark_version);
+  (canvas_id, table_tlid, user_version, dark_version);
 
 CREATE INDEX IF NOT EXISTS
 idx_user_data_current_data_for_tlid
 ON user_data_v0
 (user_version, dark_version, canvas_id, table_tlid);
 
 CREATE INDEX IF NOT EXISTS
-idx_user_data_gin_data
+  idx_user_data_gin_data
 ON user_data_v0
 USING GIN
 (data jsonb_path_ops)
+
+
+
+
+
+-- the "Idea" of some "Pizza" that I'm modeling
+-- many specific implementations of the "Pizza" idea
+-- _the_ "current" Pizza
+
+
+
+
+CREATE TABLE IF NOT EXISTS
+package_type_declaration_v0
+-- IDs
+( id UUID PRIMARY KEY
+, tlid BIGINT NOT NULL
+, definition BYTEA NOT NULL 
+, hash BYTEA NOT NULL --?
+, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS
+package_type_declaration_v0
+( id UUID PRIMARY KEY
+, tlid BIGINT NOT NULL
+, owner TEXT NOT NULL
+, modules TEXT NOT NULL 
+, typename TEXT NOT NULL 
+--, version INTEGER NOT NULL 
+, definition BYTEA NOT NULL 
+, hash BYTEA NOT NULL
+, dependencies_types BYTEA NOT NULL -- other type. also: not sure if we store only _immediate_ references, or also transitive refs
+, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- type FQPackageTypeName =
+-- { owner : string
+--     // TODO: consider whether modules should be a NonEmptyList
+--     modules : List<string>
+--     name : string
+--     version : int }
+
+
+type PackageTypeDeclaration =
+    {
+        declaration : TypeDeclaration.T
+        hash: ...
+    }
+
+type NamedPackageType =
+    {
+        name : FQTypeName.Package;
+        hash: ...
+    }
+
+
+
+
+
+
+
+-- CREATE TABLE IF NOT EXISTS
+-- named_package_type
+-- ( id UUID PRIMARY KEY
+-- , owner TEXT NOT NULL
+-- , typename TEXT NOT NULL 
+
+-- , current_modulespace
+-- , current_hash
+-- , is_current BOOL
+-- , created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+-- );
+
+-- CREATE TABLE IF NOT EXISTS
+-- package_type_declaration_v0
+-- ( id UUID PRIMARY KEY
+-- , tlid BIGINT NOT NULL
+-- , definition BYTEA NOT NULL 
+-- , hash BYTEA NOT NULL
+-- , dependencies_types BYTEA NOT NULL -- other type. also: not sure if we store only _immediate_ references, or also transitive refs
+-- , created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+-- );