fix: Reduce open handles (#57)

- Prevent the open handle on the tests due to the db connection and
makes failure on db migrations "fail fast".
- Introduce `AbortController` to clients to abort polling requests when
agents shut down.

control-plane/package.json

@@ -3,12 +3,12 @@
   "version": "0.0.16",
   "description": "",
   "scripts": {
-    "test": "jest src --setupFiles dotenv/config --forceExit --detectOpenHandles",
-    "test:dev": "jest --watch src --setupFiles dotenv/config --forceExit --onlyChanged",
-    "dev": "nodemon src/index.ts",
+    "test": "tsx src/utilities/migrate.ts && jest src --runInBand --setupFiles dotenv/config --forceExit --detectOpenHandles",
+    "test:dev": "tsx src/utilities/migrate.ts && jest --runInBand --watch src --setupFiles dotenv/config --forceExit --onlyChanged",
+    "dev": "tsx src/utilities/migrate.ts && nodemon src/index.ts",
     "migrations": "npx drizzle-kit generate:pg --schema src/modules/data.ts",
     "build": "tsc",
-    "start": "node dist/index.js",
+    "start": "node dist/utilities/migrate.js && node dist/index.js",
     "deploy": "git add . && git commit -m 'update' && git push origin main && fly deploy"
   },
   "author": "",
@@ -40,7 +40,8 @@
   "devDependencies": {
     "@types/jsonwebtoken": "^9.0.2",
     "@types/pg": "^8.10.2",
-    "@types/ws": "^8.5.5"
+    "@types/ws": "^8.5.5",
+    "tsx": "^4.7.0"
   },
   "private": true
 }

control-plane/src/index.ts

@@ -1,4 +1,4 @@
-require("dotenv").config();
+import "./utilities/env";
 
 import { initServer } from "@ts-rest/fastify";
 import fastify from "fastify";

control-plane/src/modules/auth.ts

@@ -1,7 +1,7 @@
 import { eq } from "drizzle-orm";
 import * as data from "./data";
 import jwt from "jsonwebtoken";
-import { invariant } from "../utils";
+import { invariant } from "../utilities/invariant";
 
 const jwtSecret = invariant(
   process.env.JWT_SECRET,

control-plane/src/modules/cron.ts

@@ -11,6 +11,7 @@ export const registerCron = async (
 };
 
 process.on("beforeExit", () => {
+  console.log("%%%%%%% beforeExit");
   intervals.forEach((intervalId) => {
     clearInterval(intervalId);
   });

control-plane/src/modules/data.ts

@@ -1,5 +1,4 @@
 import { drizzle } from "drizzle-orm/node-postgres";
-import { migrate } from "drizzle-orm/node-postgres/migrator";
 import {
   integer,
   json,
@@ -10,7 +9,7 @@ import {
   varchar,
 } from "drizzle-orm/pg-core";
 import { Pool } from "pg";
-import { invariant } from "../utils";
+import { invariant } from "../utilities/invariant";
 import { sql } from "drizzle-orm";
 
 const connectionString = invariant(
@@ -24,7 +23,7 @@ console.log("Attempting to connect to database", {
   sslDisabled,
 });
 
-const pool = new Pool({
+export const pool = new Pool({
   connectionString,
   ssl: sslDisabled
     ? false
@@ -130,13 +129,3 @@ export const db = drizzle(pool);
 export const isAlive = async () => {
   await db.execute(sql`select 1`);
 };
-
-(async function () {
-  console.log("Migrating database...");
-  try {
-    await migrate(db, { migrationsFolder: "./drizzle" });
-  } catch (e) {
-    console.error("Error migrating database", e);
-    process.exit(1);
-  }
-})();

control-plane/src/modules/influx.ts

@@ -1,5 +1,5 @@
-import { InfluxDB } from '@influxdata/influxdb-client'
-import { invariant } from '../utils';
+import { InfluxDB } from "@influxdata/influxdb-client";
+import { invariant } from "../utilities/invariant";
 
 // Temporary feature flag for influxdb
 let client: InfluxDB | undefined;
@@ -9,21 +9,26 @@ if (process.env.INFLUXDB_ENABLED) {
     "INFLUXDB_TOKEN must be set"
   );
 
-  const url = invariant(
-    process.env.INFLUXDB_URL,
-    "INFLUXDB_URL must be set"
-  );
+  const url = invariant(process.env.INFLUXDB_URL, "INFLUXDB_URL must be set");
 
-  client = new InfluxDB({url, token})
+  client = new InfluxDB({ url, token });
 } else {
-  console.warn("INFLUXDB_ENABLED is not set, metrics will not be sent to influxdb");
+  console.warn(
+    "INFLUXDB_ENABLED is not set, metrics will not be sent to influxdb"
+  );
 }
 
-export const INFLUXDB_ORG = 'differential';
-export const INFLUXDB_BUCKET = 'differential';
-
-export const queryClient = client?.getQueryApi(INFLUXDB_ORG)
-export const writeClient = client?.getWriteApi(INFLUXDB_ORG, INFLUXDB_BUCKET , 'ms', {
-  ...(process.env['NODE_ENV'] !== 'production' ? { flushInterval: 1000 } : {})
-})
+export const INFLUXDB_ORG = "differential";
+export const INFLUXDB_BUCKET = "differential";
 
+export const queryClient = client?.getQueryApi(INFLUXDB_ORG);
+export const writeClient = client?.getWriteApi(
+  INFLUXDB_ORG,
+  INFLUXDB_BUCKET,
+  "ms",
+  {
+    ...(process.env["NODE_ENV"] !== "production"
+      ? { flushInterval: 1000 }
+      : {}),
+  }
+);

control-plane/src/utilities/env.ts

@@ -0,0 +1,5 @@
+// separating this and importing it allows
+// tsx to run this before the rest of the app
+// for dev purposes
+
+require("dotenv").config();

control-plane/src/utilities/migrate.ts

@@ -0,0 +1,17 @@
+import "./env";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import * as data from "../modules/data";
+
+(async function runMigrations() {
+  console.log("Migrating database...");
+
+  try {
+    await migrate(data.db, { migrationsFolder: "./drizzle" });
+    console.log("Database migrated successfully");
+  } catch (e) {
+    console.error("Error migrating database", e);
+    process.exit(1);
+  }
+
+  await data.pool.end();
+})();