From 36b6c35c428fc42815bdefd0da7265bfe97e5135 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Thu, 16 May 2024 21:59:22 +0200 Subject: [PATCH 01/21] WIP support for Node and Bun --- .gitignore | 2 - .npmrc | 1 + application.meta.ts | 4 +- deno.json | 9 +- docs/src/changelog.md | 5 ++ docs/src/contributing/packaging.md | 3 +- docs/src/examples/telemetry/.npmrc | 1 + docs/src/examples/telemetry/package.json | 6 ++ lib/cli/main.ts | 21 +++-- lib/cli/status.ts | 6 +- lib/cli/upgrade.ts | 3 - lib/common/sysinfo.ts | 79 +++++++++++++++++ lib/core/configuration.ts | 2 +- lib/core/loadbalancer.ts | 9 +- lib/core/logger.ts | 105 ++++++++++------------- lib/core/process.ts | 3 +- lib/core/pup.ts | 30 +++++-- lib/core/status.ts | 57 +++++++----- lib/core/worker.ts | 3 +- package.json | 32 +++++++ 20 files changed, 259 insertions(+), 122 deletions(-) create mode 100644 .npmrc create mode 100644 docs/src/examples/telemetry/.npmrc create mode 100644 docs/src/examples/telemetry/package.json create mode 100644 lib/common/sysinfo.ts create mode 100644 package.json diff --git a/.gitignore b/.gitignore index b4d5943..32354fd 100644 --- a/.gitignore +++ b/.gitignore @@ -46,9 +46,7 @@ coverage _site # Node files -.npmrc package-lock.json -package.json node_modules # VSCode files diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..691d217 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +@jsr:registry=https://npm.jsr.io \ No newline at end of file diff --git a/application.meta.ts b/application.meta.ts index 69c70ca..9de9049 100644 --- a/application.meta.ts +++ b/application.meta.ts @@ -21,10 +21,10 @@ const Application = { name: "pup", - version: "1.0.0-rc.39", + version: "1.0.0-rc.40", url: "jsr:@pup/pup@$VERSION", canary_url: "https://raw.githubusercontent.com/Hexagon/pup/main/pup.ts", - deno: null, /* Minimum stable version of Deno required to run Pup (without --unstable-* flags) */ + deno: "1.43.0", /* Minimum stable version of Deno required to run Pup (without --unstable-* flags) */ deno_unstable: "1.43.0", /* Minimum version of Deno required to run Pup (with --unstable-* flags) */ repository: "https://github.com/hexagon/pup", changelog: "https://hexagon.github.io/pup/changelog.html", diff --git a/deno.json b/deno.json index 6e272bf..1cb9658 100644 --- a/deno.json +++ b/deno.json @@ -7,10 +7,6 @@ "./lib": "./mod.ts" }, - "unstable": [ - "kv" - ], - "fmt": { "lineWidth": 200, "semiColons": false, @@ -30,7 +26,7 @@ "tasks": { "update-deps": "deno run --allow-read=. --allow-net=jsr.io,registry.npmjs.org jsr:@check/deps", - "check": "deno fmt --check && deno lint && deno check --unstable-kv pup.ts && deno test --allow-read --allow-write --allow-env --allow-net --allow-sys --allow-run --unstable-kv --coverage=cov_profile && echo \"Generating coverage\" && deno coverage cov_profile --exclude=pup/test --lcov --output=cov_profile.lcov", + "check": "deno fmt --check && deno lint && deno check pup.ts && deno test --allow-read --allow-write --allow-env --allow-net --allow-sys --allow-run --coverage=cov_profile && echo \"Generating coverage\" && deno coverage cov_profile --exclude=pup/test --lcov --output=cov_profile.lcov", "check-coverage": "deno task check && genhtml cov_profile.lcov --output-directory cov_profile/html && lcov --list cov_profile.lcov && deno run --allow-net --allow-read https://deno.land/std/http/file_server.ts cov_profile/html", "build-schema": "deno run --allow-write --allow-read --allow-env=XDG_DATA_HOME,HOME tools/build-schema.ts && deno fmt", "build-versions": "deno run --allow-read --allow-write --allow-env tools/release.ts && deno fmt", @@ -42,12 +38,13 @@ "@cross/env": "jsr:@cross/env@^1.0.2", "@cross/fs": "jsr:@cross/fs@^0.1.11", "@cross/jwt": "jsr:@cross/jwt@^0.4.7", + "@cross/kv": "jsr:@cross/kv@^0.0.13", "@cross/runtime": "jsr:@cross/runtime@^1.0.0", "@cross/service": "jsr:@cross/service@^1.0.3", "@cross/test": "jsr:@cross/test@^0.0.9", "@cross/utils": "jsr:@cross/utils@^0.12.0", "@hexagon/croner": "jsr:@hexagon/croner@^8.0.2", - "@oak/oak": "jsr:@oak/oak@^15.0.0", + "@oak/oak": "jsr:@oak/oak@^16.0.0", "@pup/api-client": "jsr:@pup/api-client@^1.0.6", "@pup/api-definitions": "jsr:@pup/api-definitions@^1.0.2", "@pup/common": "jsr:@pup/common@^1.0.3", diff --git a/docs/src/changelog.md b/docs/src/changelog.md index f18657d..77d204d 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -9,6 +9,11 @@ nav_order: 13 All notable changes to this project will be documented in this section. +## [1.0.0-rc.40] - Unreleased + +- fix(core): Replace `Deno.Kv` with `@cross/kv` for cross-runtime compatibility, more compact logs and avoiding `--unstable` +- fix(core): Make Pup work in Node and Bun + ## [1.0.0-rc.39] - 2024-05-04 - fix(core): Clustered processes were duplicated in API and `pup status` diff --git a/docs/src/contributing/packaging.md b/docs/src/contributing/packaging.md index 1e560bf..bf35513 100644 --- a/docs/src/contributing/packaging.md +++ b/docs/src/contributing/packaging.md @@ -14,8 +14,7 @@ If you have experience with software packaging, your contribution can greatly en - Pup can be compiled into an executable prior to packaging using `deno compile`. The procedure is described at [https://deno.com/manual@v1.34.3/tools/compiler](https://deno.com/manual@v1.34.3/tools/compiler). The command should be similar to - `deno compile --allow-all --reload --unstable-kv --output pup pup.ts --external-installer`. The `--unstable-kv` flag should be included if the version you are packaging (mostly pre-releases) - requires unstable features according to `versions.json`. + `deno compile --allow-all --reload --output pup pup.ts --external-installer`. - The `--external-installer` argument to the Pup script disables the built-in installer, hiding `setup` and `upgrade` options from `--help`. diff --git a/docs/src/examples/telemetry/.npmrc b/docs/src/examples/telemetry/.npmrc new file mode 100644 index 0000000..41583e3 --- /dev/null +++ b/docs/src/examples/telemetry/.npmrc @@ -0,0 +1 @@ +@jsr:registry=https://npm.jsr.io diff --git a/docs/src/examples/telemetry/package.json b/docs/src/examples/telemetry/package.json new file mode 100644 index 0000000..b49fd43 --- /dev/null +++ b/docs/src/examples/telemetry/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "dependencies": { + "@pup/telemetry": "npm:@jsr/pup__telemetry@^1.0.5" + } +} diff --git a/lib/cli/main.ts b/lib/cli/main.ts index 096e951..a10a729 100644 --- a/lib/cli/main.ts +++ b/lib/cli/main.ts @@ -431,6 +431,7 @@ async function main() { if (baseArgument === "logs") { const logStore = `${await toPersistentPath(configFile as string)}/.main.log` const logger = new Logger(configuration!.logger || {}, logStore) + await logger.init() const startTimestamp = checkedArgs.get("start") ? new Date(Date.parse(checkedArgs.get("start")!)).getTime() : undefined const endTimestamp = checkedArgs.get("end") ? new Date(Date.parse(checkedArgs.get("end")!)).getTime() : undefined const numberOfRows = checkedArgs.get("n") ? parseInt(checkedArgs.get("n")!, 10) : undefined @@ -496,7 +497,7 @@ async function main() { exit(1) } } catch (_e) { - console.error("Action failed: Could not contact the Pup instance.") + console.error("Action failed: Could not contact the Pup instance.", _e) exit(1) } } @@ -587,14 +588,16 @@ async function main() { pup.init() // Register for running pup.terminate() if not already run on clean exit - let hasRunShutdownCode = false - globalThis.addEventListener("beforeunload", (evt) => { - if (!hasRunShutdownCode) { - evt.preventDefault() - hasRunShutdownCode = true - ;(async () => await pup.terminate(30000))() - } - }) + if (globalThis.addEventListener) { + let hasRunShutdownCode = false + globalThis.addEventListener("beforeunload", (evt) => { + if (!hasRunShutdownCode) { + evt.preventDefault() + hasRunShutdownCode = true + ;(async () => await pup.terminate(30000))() + } + }) + } if (CurrentRuntime === Runtime.Deno) { // This is needed to trigger termination in Deno diff --git a/lib/cli/status.ts b/lib/cli/status.ts index c456f3e..ceeb874 100644 --- a/lib/cli/status.ts +++ b/lib/cli/status.ts @@ -14,8 +14,8 @@ import { filesize } from "filesize" import { blockedFormatter, codeFormatter, naFormatter, restartsFormatter, statusFormatter } from "./formatters/strings.ts" import { timeagoFormatter } from "./formatters/times.ts" import { Configuration, DEFAULT_REST_API_HOSTNAME } from "../core/configuration.ts" -import { resolve } from "@std/path" import { ApiApplicationState } from "@pup/api-definitions" +import { toResolvedAbsolutePath } from "@pup/common/path" /** * Helper which print the status of all running processes, @@ -29,8 +29,8 @@ import { ApiApplicationState } from "@pup/api-definitions" export function printStatus(configFile: string, configuration: Configuration, cwd: string | undefined, status: ApiApplicationState) { // Print configuration console.log("") - console.log(Colors.bold("Configuration:") + "\t" + resolve(configFile)) - console.log(Colors.bold("Working dir:") + "\t" + cwd || "Not set (default: pup)") + console.log(Colors.bold("Configuration:") + "\t" + toResolvedAbsolutePath(configFile)) + console.log(Colors.bold("Working dir:") + "\t" + (cwd ? toResolvedAbsolutePath(cwd as string) : "Not set (default: pup)")) console.log(Colors.bold("Instance name:") + "\t" + (configuration.name || "Not set")) console.log(Colors.bold("Rest API URL:") + "\thttp://" + (configuration.api?.hostname || DEFAULT_REST_API_HOSTNAME) + ":" + status.port) diff --git a/lib/cli/upgrade.ts b/lib/cli/upgrade.ts index 14f8f31..6a20df7 100644 --- a/lib/cli/upgrade.ts +++ b/lib/cli/upgrade.ts @@ -176,9 +176,6 @@ export async function upgrade( if (ignoreCertificateErrorsString && ignoreCertificateErrorsString !== "") { installCmd.push(ignoreCertificateErrorsString) } - if (unstableInstall) { - installCmd.push("--unstable-kv") - } installCmd.push("-n", "pup") // Installed command name = pup installCmd.push(canaryInstall ? versions.canary_url : (requestedVersion as Version).url) diff --git a/lib/common/sysinfo.ts b/lib/common/sysinfo.ts new file mode 100644 index 0000000..f4b4968 --- /dev/null +++ b/lib/common/sysinfo.ts @@ -0,0 +1,79 @@ +import { CurrentRuntime, Runtime } from "@cross/runtime" +import { ApiMemoryUsage, ApiSystemMemory } from "@pup/api-definitions" +import { freemem, loadavg, totalmem, uptime as nodeUptime } from "node:os" + +export function memoryUsage(): ApiMemoryUsage { + let memoryUsageResult: ApiMemoryUsage + if (CurrentRuntime === Runtime.Deno) { + //@ts-ignore cross-runtime + const { external, heapTotal, heapUsed, rss } = Deno.memoryUsage() + memoryUsageResult = { external, heapTotal, heapUsed, rss } + } else if ( + CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun + ) { + //@ts-ignore cross-runtime + const { external = 0, heapTotal, heapUsed, rss } = process.memoryUsage() + memoryUsageResult = { external, heapTotal, heapUsed, rss } + } else { + memoryUsageResult = { external: 0, heapTotal: 0, heapUsed: 0, rss: 0 } + } + return memoryUsageResult +} + +export function loadAvg(): number[] { + let loadAvgResult: number[] + if (CurrentRuntime === Runtime.Deno) { + loadAvgResult = Deno.loadavg() + } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { + // Node.js and Bun provide os module for loadAvg + loadAvgResult = loadavg() + } else { + // Unsupported runtime + loadAvgResult = [] + } + return loadAvgResult +} + +export function uptime(): number { + let uptimeResult: number + if (CurrentRuntime === Runtime.Deno) { + uptimeResult = Deno.osUptime() + } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { + // Node.js and Bun provide os module for uptime + uptimeResult = nodeUptime() + } else { + uptimeResult = -1 + } + return uptimeResult +} + +export function systemMemoryInfo(): ApiSystemMemory { + let memoryInfoResult: ApiSystemMemory + if (CurrentRuntime === Runtime.Deno) { + memoryInfoResult = Deno.systemMemoryInfo() + } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { + // Node.js and Bun don't have a direct equivalent to Deno.systemMemoryInfo + // We can try to approximate values using os module (limited information) + memoryInfoResult = { + total: totalmem(), + free: freemem(), + available: -1, // Not directly available + buffers: -1, // Not directly available + cached: -1, // Not directly available + swapTotal: -1, // Approximate swap total + swapFree: -1, // Not directly available + } + } else { + // Unsupported runtime + memoryInfoResult = { + total: -1, + free: -1, + available: -1, + buffers: -1, + cached: -1, + swapTotal: -1, + swapFree: -1, + } + } + return memoryInfoResult +} diff --git a/lib/core/configuration.ts b/lib/core/configuration.ts index c67c4ad..31b70c0 100644 --- a/lib/core/configuration.ts +++ b/lib/core/configuration.ts @@ -10,7 +10,7 @@ import { PluginConfiguration } from "@pup/plugin" // Logger constants export const DEFAULT_INTERNAL_LOG_HOURS = 48 -export const KV_SIZE_LIMIT_BYTES = 65_536 +export const KV_LIMIT_STRING_LENGTH_BYTES = 12_000 // Core constants export const MAINTENANCE_INTERVAL_MS = 900_000 diff --git a/lib/core/loadbalancer.ts b/lib/core/loadbalancer.ts index 054ec87..7d3be50 100644 --- a/lib/core/loadbalancer.ts +++ b/lib/core/loadbalancer.ts @@ -5,6 +5,7 @@ * @license MIT */ +import { CurrentRuntime, Runtime } from "@cross/runtime" import { LOAD_BALANCER_DEFAULT_VALIDATION_INTERVAL_S } from "./configuration.ts" export enum BalancingStrategy { @@ -77,7 +78,13 @@ export class LoadBalancer { private setupValidationTimer(): number { const timer = setInterval(() => this.validateBackends(), this.validationInterval * 1000) // Make the timer non-blocking - Deno.unrefTimer(timer) + if (CurrentRuntime === Runtime.Deno) { + Deno.unrefTimer(timer) + // @ts-ignore unref exists in node and bun + } else if (timer.unref) { + // @ts-ignore unref exists in node and bun + timer.unref() + } return timer } diff --git a/lib/core/logger.ts b/lib/core/logger.ts index 1cdcdfd..073133c 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -7,7 +7,9 @@ */ import { stripAnsi } from "@cross/utils" -import { type GlobalLoggerConfiguration, KV_SIZE_LIMIT_BYTES, type ProcessConfiguration } from "./configuration.ts" +import { type GlobalLoggerConfiguration, KV_LIMIT_STRING_LENGTH_BYTES, type ProcessConfiguration } from "./configuration.ts" +import { KV, KVKeyRange, KVQuery } from "@cross/kv" +import { writeFile } from "@cross/fs" export interface LogEvent { severity: string @@ -29,11 +31,17 @@ type AttachedLogger = (severity: string, category: string, text: string, process class Logger { private config: GlobalLoggerConfiguration = {} private attachedLogger?: AttachedLogger - private storeName?: string + private storeName: string + private kv: KV - constructor(globalConfiguration: GlobalLoggerConfiguration, storeName?: string) { + constructor(globalConfiguration: GlobalLoggerConfiguration, storeName: string) { this.config = globalConfiguration this.storeName = storeName + this.kv = new KV({ autoSync: false }) + } + + public async init(): Promise { + await this.kv.open(this.storeName) } // Used for attaching the logger hook @@ -42,27 +50,32 @@ class Logger { } // Prepare log event selector - private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): { prefix: Deno.KvKey } | { start: Deno.KvKey; end: Deno.KvKey } { - const key = processId ? ["logs_by_process", processId] : ["logs_by_time"] + private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): KVQuery { + const key: KVQuery = processId ? ["logs_by_time", {}, processId] : ["logs_by_time"] if (startTimeStamp || endTimeStamp) { - const startKey: (string | number)[] = [...key, startTimeStamp || 0] - const endKey: (string | number)[] = [...key, endTimeStamp || Infinity] - return { start: startKey, end: endKey } + const rangeSelector: KVKeyRange = {} + if (startTimeStamp) { + rangeSelector.from = startTimeStamp + } + if (endTimeStamp) { + rangeSelector.to = endTimeStamp + } + key.push(rangeSelector) } - return { prefix: key } + return key } // Fetch logs from store - private async fetchLogsFromStore(selector: { prefix: Deno.KvKey } | { start: Deno.KvKey; end: Deno.KvKey }, nRows?: number): Promise { - const store = await Deno.openKv(this.storeName) - const result = await store.list(selector) + private async fetchLogsFromStore(selector: KVQuery, nRows?: number): Promise { + const result = await this.kv.listAll(selector) const resultArray: LogEventData[] = [] - for await (const res of result) resultArray.push(res.value as LogEventData) + for await (const res of result) { + resultArray.push(res.data as LogEventData) + } if (nRows) { const spliceNumber = Math.max(0, resultArray.length - nRows) resultArray.splice(0, spliceNumber) } - store.close() return resultArray } @@ -84,7 +97,7 @@ class Logger { } private async internalLog(severity: string, category: string, text: string, process?: ProcessConfiguration, timeStamp?: number) { - // Default initiator to + // Default initiator to core const initiator = process?.id || "core" timeStamp = timeStamp || Date.now() @@ -92,31 +105,19 @@ class Logger { // Write to persistent log store (if a name is supplied and internal logging is enabled) const logHours = this.config.internalLogHours === undefined ? 72 : this.config.internalLogHours if (this.storeName && logHours > 0) { - const textBytes = new TextEncoder().encode(text) - const store = await Deno.openKv(this.storeName) - let i = 0, offset = 0 - while (offset < textBytes.length) { - // Give 6000 bytes margin to make it less likely that the full serialized object exceeds the KV limit - const slice = textBytes.subarray(offset, offset + KV_SIZE_LIMIT_BYTES - 6000) - // Ignore errors when writing to log store - try { - const logObj: LogEventData = { - severity, - category, - text: new TextDecoder().decode(slice), - processId: initiator, - timeStamp: timeStamp + i, - } - await store.set(["logs_by_time", timeStamp + i], logObj) - await store.set(["logs_by_process", initiator, timeStamp + i], logObj) - await store.set(["logs_by_process_lookup", timeStamp + i], ["logs_by_process", initiator, timeStamp + i]) - i++ - } catch (error) { - console.error(`Failed to write log to store '${this.storeName}' due to '${error.message}'. The following message was not logged: ${text}.`) + // Ignore errors when writing to log store + try { + const logObj: LogEventData = { + severity, + category, + text: text.length > KV_LIMIT_STRING_LENGTH_BYTES ? text.substring(0, KV_LIMIT_STRING_LENGTH_BYTES) + "..." : text, + processId: initiator, + timeStamp: timeStamp, } - offset += KV_SIZE_LIMIT_BYTES - 6000 + await this.kv.set(["logs_by_time", timeStamp, initiator], logObj) + } catch (e) { + console.error("Error while writing to log store", e) } - store.close() } // Delegate to attached logger if there is one @@ -199,7 +200,7 @@ class Logger { // Strip colors text = stripAnsi(text) try { - await Deno.writeTextFile(fileName, `${text}\n`, { append: true }) + await writeFile(fileName, `${text}\n`, { append: true }) } catch (_e) { if (!quiet) console.error(`Failed to write log '${fileName}'. The following message were not logged: ${text}.`) } @@ -229,32 +230,18 @@ class Logger { if (!this.storeName) { return 0 } + try { - const store = await Deno.openKv(this.storeName) const now = Date.now() const startTime = now - keepHours * 60 * 60 * 1000 - const logsByTimeSelector = { - prefix: ["logs_by_time"], - end: ["logs_by_time", startTime], - } + const logsByTimeSelector: KVQuery = ["logs_by_time", { to: startTime }] let rowsDeleted = 0 - for await (const entry of store.list(logsByTimeSelector)) { - await store.delete(entry.key) + for await (const entry of this.kv.iterate(logsByTimeSelector)) { + await this.kv.delete(entry.key) rowsDeleted++ } - const logsByProcessSelector = { - prefix: ["logs_by_process_lookup"], - end: ["logs_by_process_lookup", startTime], - } - let rowsDeletedProcess = 0 - for await (const entry of store.list(logsByProcessSelector)) { - // Delete both the lookup key and the actual key - await store.delete(entry.value as Deno.KvKey) - await store.delete(entry.key) - rowsDeletedProcess++ - } - store.close() - return rowsDeleted + rowsDeletedProcess + await this.kv.vacuum() + return rowsDeleted } catch (error) { this.log("error", `Failed to purge logs from store '${this.storeName}': ${error.message}`) return 0 diff --git a/lib/core/process.ts b/lib/core/process.ts index 89d9578..e9f9bd4 100644 --- a/lib/core/process.ts +++ b/lib/core/process.ts @@ -16,6 +16,7 @@ import { Cron } from "@hexagon/croner" import { delay } from "@std/async" import { ApiProcessState } from "@pup/api-definitions" +import { CurrentOS, OperatingSystem } from "@cross/runtime" interface ProcessStateChangedEvent { old?: ApiProcessState @@ -274,7 +275,7 @@ class Process { persistent: true, } delay((this.config.terminateGracePeriod ?? this.pup.configuration.terminateGracePeriod ?? 0) * 1000, graceDelayOptions).then(() => { - if (Deno.build.os == "windows") { + if (CurrentOS == OperatingSystem.Windows) { // On Windows, SIGTERM kills the process because Windows can't handle signals without cumbersome workarounds: https://stackoverflow.com/questions/35772001/how-to-handle-a-signal-sigint-on-a-windows-os-machine#35792192 this.pup.logger.log("stopping", `Killing process, reason: ${reason}`, this.config) } else { diff --git a/lib/core/pup.ts b/lib/core/pup.ts index eea171e..7a393f6 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -31,6 +31,7 @@ import { rm } from "@cross/fs" import { findFreePort } from "./port.ts" import { Plugin } from "./plugin.ts" import { GenerateToken, SecondsToExpiry } from "../common/token.ts" +import { CurrentRuntime, Runtime } from "@cross/runtime" interface InstructionResponse { success: boolean action?: string @@ -90,7 +91,7 @@ class Pup { this.configuration = validateConfiguration(unvalidatedConfiguration) // Initialise core logger - this.logger = new Logger(this.configuration.logger ?? {}, logStore) + this.logger = new Logger(this.configuration.logger ?? {}, logStore || "./main.log") // Global error handler this.registerGlobalErrorHandler() @@ -129,6 +130,9 @@ class Pup { } public init = async () => { + // Intialize logging + await this.logger.init() + // Initialize api await this.api() @@ -398,7 +402,13 @@ class Pup { this.maintenanceTimer = setTimeout(() => { this.maintenance() }, MAINTENANCE_INTERVAL_MS) - Deno.unrefTimer(this.maintenanceTimer) + if (CurrentRuntime === Runtime.Deno) { + Deno.unrefTimer(this.maintenanceTimer) + // @ts-ignore unref exists in node and bun + } else if (this.maintenanceTimer?.unref) { + // @ts-ignore unref exists in node and bun + this.maintenanceTimer.unref() + } } public restart(id: string, requestor: string): boolean { @@ -513,13 +523,15 @@ class Pup { } private registerGlobalErrorHandler() { - addEventListener("error", (event) => { - this.logger.error( - "fatal", - `Unhandled error caught by core: ${event.error.message}`, - ) - event.preventDefault() - }) + if (globalThis.addEventListener) { + addEventListener("error", (event) => { + this.logger.error( + "fatal", + `Unhandled error caught by core: ${event.error.message}`, + ) + event.preventDefault() + }) + } } } diff --git a/lib/core/status.ts b/lib/core/status.ts index c807ece..0dd8a42 100644 --- a/lib/core/status.ts +++ b/lib/core/status.ts @@ -8,9 +8,13 @@ import { Application } from "../../application.meta.ts" import { APPLICATION_STATE_WRITE_LIMIT_MS } from "./configuration.ts" import { type Process, type ProcessInformation } from "./process.ts" -import { ApiProcessState } from "@pup/api-definitions" +import { ApiMemoryUsage, ApiProcessState, ApiSystemMemory } from "@pup/api-definitions" +import { KV } from "@cross/kv" import { Prop } from "../common/prop.ts" +import { loadAvg, memoryUsage, systemMemoryInfo, uptime } from "../common/sysinfo.ts" +import { getCurrentOS, getCurrentRuntime, getCurrentVersion } from "@cross/runtime" +import { pid } from "@cross/utils" const started = new Date() @@ -24,12 +28,12 @@ export interface ApplicationState { updated: string started: string port: number - memory: Deno.MemoryUsage - systemMemory: Deno.SystemMemoryInfo + memory: ApiMemoryUsage + systemMemory: ApiSystemMemory loadAvg: number[] osUptime: number osRelease: string - denoVersion: { deno: string; v8: string; typescript: string } + runtime: string type: string processes: ProcessInformation[] } @@ -58,7 +62,8 @@ class Status { */ public async writeToStore(applicationState: ApplicationState) { try { - const kv = await Deno.openKv(this.storeName) + const kv = new KV({ autoSync: false }) + await kv.open(this.storeName!) // Initialize lastWrite if it's not set if (!this.lastWrite) { @@ -73,7 +78,7 @@ class Status { // Always write last_application_state await kv.set(["last_application_state"], applicationState) - kv.close() + await kv.close() } catch (e) { console.error("Error while writing status to kv store: " + e.message) } @@ -85,9 +90,10 @@ class Status { */ public async cleanup() { try { - const kv = await Deno.openKv(this.storeName) + const kv = new KV({ autoSync: false }) + await kv.open(this.storeName!) await kv.delete(["last_application_state"]) - kv.close() + await kv.close() } catch (e) { console.error("Error while writing status to kv store: " + e.message) } @@ -104,19 +110,18 @@ class Status { return 0 } try { - const store = await Deno.openKv(this.storeName) + const kv = new KV({ autoSync: false }) + await kv.open(this.storeName) const now = Date.now() const startTime = now - keepHours * 60 * 60 * 1000 - const logsByTimeSelector = { - prefix: ["application_state"], - end: ["application_state", startTime], - } + const logsByTimeSelector = ["application_state", { to: startTime }] let rowsDeleted = 0 - for await (const entry of store.list(logsByTimeSelector)) { + for await (const entry of kv.iterate(logsByTimeSelector)) { rowsDeleted++ - await store.delete(entry.key) + await kv.delete(entry.key) } - store.close() + await kv.vacuum() + await kv.close() return rowsDeleted } catch (error) { console.error(`Failed to purge logs from store '${this.storeName}': ${error.message}`) @@ -134,19 +139,25 @@ class Status { for (const p of processes) { processStates.push(p.getStatus()) } + const memory = memoryUsage() + const systemMemory = systemMemoryInfo() + const loadAverage = loadAvg() + const osUptime = uptime() + const osRelease = getCurrentOS().toString() + const runtime = getCurrentRuntime().toString() + " " + getCurrentVersion() return { - pid: Deno.pid, + pid: pid(), version: Application.version, status: ApiProcessState[ApiProcessState.RUNNING], updated: new Date().toISOString(), started: started.toISOString(), - memory: Deno.memoryUsage(), + memory, port: port ? parseInt(port.fromCache()!, 10) : 0, - systemMemory: Deno.systemMemoryInfo(), - loadAvg: Deno.loadavg(), - osUptime: Deno.osUptime(), - osRelease: Deno.osRelease(), - denoVersion: Deno.version, + systemMemory, + loadAvg: loadAverage, + osUptime, + osRelease, + runtime, type: "main", processes: processStates, } diff --git a/lib/core/worker.ts b/lib/core/worker.ts index 9f23abb..b324ebd 100644 --- a/lib/core/worker.ts +++ b/lib/core/worker.ts @@ -9,6 +9,7 @@ import type { ProcessConfiguration, Pup } from "./pup.ts" import { readLines, StringReader } from "@std/io" import { resolve } from "@std/path" import { BaseRunner, type RunnerCallback, type RunnerResult } from "../types/runner.ts" +import { cwd } from "@cross/fs" class WorkerRunner extends BaseRunner { private worker?: Worker @@ -55,7 +56,7 @@ class WorkerRunner extends BaseRunner { if (this.pup.temporaryStoragePath) env.PUP_TEMP_STORAGE = this.pup.temporaryStoragePath if (this.pup.persistentStoragePath) env.PUP_DATA_STORAGE = this.pup.persistentStoragePath - const workingDir = this.processConfig.cwd || Deno.cwd() + const workingDir = this.processConfig.cwd || cwd() const workingDirUrl = new URL(`file://${resolve(workingDir)}/`).href try { diff --git a/package.json b/package.json new file mode 100644 index 0000000..8a9490d --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "type": "module", + "dependencies": { + "@cross/deepmerge": "npm:@jsr/cross__deepmerge@^1.0.0", + "@cross/env": "npm:@jsr/cross__env@^1.0.2", + "@cross/fs": "npm:@jsr/cross__fs@^0.1.11", + "@cross/jwt": "npm:@jsr/cross__jwt@^0.4.7", + "@cross/kv": "npm:@jsr/cross__kv@^0.9.0", + "@cross/runtime": "npm:@jsr/cross__runtime@^1.0.0", + "@cross/service": "npm:@jsr/cross__service@^1.0.3", + "@cross/test": "npm:@jsr/cross__test@^0.0.9", + "@cross/utils": "npm:@jsr/cross__utils@^0.12.0", + "@hexagon/croner": "npm:@jsr/hexagon__croner@^8.0.2", + "@oak/oak": "npm:@jsr/oak__oak@^16.0.0", + "@pup/api-client": "npm:@jsr/pup__api-client@^1.0.6", + "@pup/api-definitions": "npm:@jsr/pup__api-definitions@^1.0.2", + "@pup/common": "npm:@jsr/pup__common@^1.0.3", + "@pup/plugin": "npm:@jsr/pup__plugin@^1.0.1", + "@std/assert": "npm:@jsr/std__assert@^0.224.0", + "@std/async": "npm:@jsr/std__async@^0.224.0", + "@std/encoding": "npm:@jsr/std__encoding@^0.224.0", + "@std/io": "npm:@jsr/std__io@^0.224.0", + "@std/path": "npm:@jsr/std__path@^0.224.0", + "@std/semver": "npm:@jsr/std__semver@^0.224.0", + "dax-sh": "^0.41.0", + "filesize": "^10.1.2", + "json5": "^2.2.3", + "timeago.js": "^4.0.2", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.0" + } +} From 8be9e4b83bf3348f9fcc5efce7c043e2dacd049c Mon Sep 17 00:00:00 2001 From: Hexagon Date: Mon, 20 May 2024 21:51:48 +0200 Subject: [PATCH 02/21] Update @cross/kv --- deno.json | 2 +- lib/core/logger.ts | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deno.json b/deno.json index 1cb9658..ea83845 100644 --- a/deno.json +++ b/deno.json @@ -38,7 +38,7 @@ "@cross/env": "jsr:@cross/env@^1.0.2", "@cross/fs": "jsr:@cross/fs@^0.1.11", "@cross/jwt": "jsr:@cross/jwt@^0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.0.13", + "@cross/kv": "jsr:@cross/kv@^0.11.0", "@cross/runtime": "jsr:@cross/runtime@^1.0.0", "@cross/service": "jsr:@cross/service@^1.0.3", "@cross/test": "jsr:@cross/test@^0.0.9", diff --git a/lib/core/logger.ts b/lib/core/logger.ts index 073133c..0165db0 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -8,7 +8,7 @@ import { stripAnsi } from "@cross/utils" import { type GlobalLoggerConfiguration, KV_LIMIT_STRING_LENGTH_BYTES, type ProcessConfiguration } from "./configuration.ts" -import { KV, KVKeyRange, KVQuery } from "@cross/kv" +import { KV, KVQuery, KVQueryRange } from "@cross/kv" import { writeFile } from "@cross/fs" export interface LogEvent { @@ -53,7 +53,7 @@ class Logger { private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): KVQuery { const key: KVQuery = processId ? ["logs_by_time", {}, processId] : ["logs_by_time"] if (startTimeStamp || endTimeStamp) { - const rangeSelector: KVKeyRange = {} + const rangeSelector: KVQueryRange = {} if (startTimeStamp) { rangeSelector.from = startTimeStamp } diff --git a/package.json b/package.json index 8a9490d..ba5054a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "@cross/env": "npm:@jsr/cross__env@^1.0.2", "@cross/fs": "npm:@jsr/cross__fs@^0.1.11", "@cross/jwt": "npm:@jsr/cross__jwt@^0.4.7", - "@cross/kv": "npm:@jsr/cross__kv@^0.9.0", + "@cross/kv": "npm:@jsr/cross__kv@^0.11.0", "@cross/runtime": "npm:@jsr/cross__runtime@^1.0.0", "@cross/service": "npm:@jsr/cross__service@^1.0.3", "@cross/test": "npm:@jsr/cross__test@^0.0.9", From 232cb4434cf8bffa864878f494b2dc2e1649b978 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Mon, 20 May 2024 22:10:08 +0200 Subject: [PATCH 03/21] Use tighter version ranges in deno.json. Fix type errors. --- deno.json | 54 +++++++++++++++++++++++----------------------- lib/core/logger.ts | 2 +- lib/core/pup.ts | 1 + lib/core/status.ts | 26 +++------------------- 4 files changed, 32 insertions(+), 51 deletions(-) diff --git a/deno.json b/deno.json index ea83845..0e27cbd 100644 --- a/deno.json +++ b/deno.json @@ -34,32 +34,32 @@ }, "imports": { - "@cross/deepmerge": "jsr:@cross/deepmerge@^1.0.0", - "@cross/env": "jsr:@cross/env@^1.0.2", - "@cross/fs": "jsr:@cross/fs@^0.1.11", - "@cross/jwt": "jsr:@cross/jwt@^0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.11.0", - "@cross/runtime": "jsr:@cross/runtime@^1.0.0", - "@cross/service": "jsr:@cross/service@^1.0.3", - "@cross/test": "jsr:@cross/test@^0.0.9", - "@cross/utils": "jsr:@cross/utils@^0.12.0", - "@hexagon/croner": "jsr:@hexagon/croner@^8.0.2", - "@oak/oak": "jsr:@oak/oak@^16.0.0", - "@pup/api-client": "jsr:@pup/api-client@^1.0.6", - "@pup/api-definitions": "jsr:@pup/api-definitions@^1.0.2", - "@pup/common": "jsr:@pup/common@^1.0.3", - "@pup/plugin": "jsr:@pup/plugin@^1.0.1", - "@std/assert": "jsr:@std/assert@^0.224.0", - "@std/async": "jsr:@std/async@^0.224.0", - "@std/encoding": "jsr:@std/encoding@^0.224.0", - "@std/io": "jsr:@std/io@^0.224.0", - "@std/path": "jsr:@std/path@^0.224.0", - "@std/semver": "jsr:@std/semver@^0.224.0", - "dax-sh": "npm:dax-sh@^0.41.0", - "filesize": "npm:filesize@^10.1.1", - "json5": "npm:json5@^2.2.3", - "timeago.js": "npm:timeago.js@^4.0.2", - "zod": "npm:zod@^3.23.6", - "zod-to-json-schema": "npm:zod-to-json-schema@^3.23.0" + "@cross/deepmerge": "jsr:@cross/deepmerge@~1.0.0", + "@cross/env": "jsr:@cross/env@~1.0.2", + "@cross/fs": "jsr:@cross/fs@~0.1.11", + "@cross/jwt": "jsr:@cross/jwt@~0.4.7", + "@cross/kv": "jsr:@cross/kv@~0.11.0", + "@cross/runtime": "jsr:@cross/runtime@~1.0.0", + "@cross/service": "jsr:@cross/service@~1.0.3", + "@cross/test": "jsr:@cross/test@~0.0.9", + "@cross/utils": "jsr:@cross/utils@~0.12.0", + "@hexagon/croner": "jsr:@hexagon/croner@~8.0.2", + "@oak/oak": "jsr:@oak/oak@~16.0.0", + "@pup/api-client": "jsr:@pup/api-client@~1.0.6", + "@pup/api-definitions": "jsr:@pup/api-definitions@~1.0.2", + "@pup/common": "jsr:@pup/common@~1.0.3", + "@pup/plugin": "jsr:@pup/plugin@~1.0.1", + "@std/assert": "jsr:@std/assert@^0.225.2", + "@std/async": "jsr:@std/async@~0.224.0", + "@std/encoding": "jsr:@std/encoding@~0.224.0", + "@std/io": "jsr:@std/io@~0.224.0", + "@std/path": "jsr:@std/path@^0.225.1", + "@std/semver": "jsr:@std/semver@~0.224.0", + "dax-sh": "npm:dax-sh@~0.41.0", + "filesize": "npm:filesize@~10.1.1", + "json5": "npm:json5@~2.2.3", + "timeago.js": "npm:timeago.js@~4.0.2", + "zod": "npm:zod@~3.23.6", + "zod-to-json-schema": "npm:zod-to-json-schema@~3.23.0" } } diff --git a/lib/core/logger.ts b/lib/core/logger.ts index 0165db0..a6c187a 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -200,7 +200,7 @@ class Logger { // Strip colors text = stripAnsi(text) try { - await writeFile(fileName, `${text}\n`, { append: true }) + await writeFile(fileName, `${text}\n`, { mode: "a+" }) } catch (_e) { if (!quiet) console.error(`Failed to write log '${fileName}'. The following message were not logged: ${text}.`) } diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 7a393f6..2edb028 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -523,6 +523,7 @@ class Pup { } private registerGlobalErrorHandler() { + // @ts-ignore Cross Runtime if (globalThis.addEventListener) { addEventListener("error", (event) => { this.logger.error( diff --git a/lib/core/status.ts b/lib/core/status.ts index 0dd8a42..827995c 100644 --- a/lib/core/status.ts +++ b/lib/core/status.ts @@ -8,7 +8,7 @@ import { Application } from "../../application.meta.ts" import { APPLICATION_STATE_WRITE_LIMIT_MS } from "./configuration.ts" import { type Process, type ProcessInformation } from "./process.ts" -import { ApiMemoryUsage, ApiProcessState, ApiSystemMemory } from "@pup/api-definitions" +import { ApiApplicationState, ApiProcessState } from "@pup/api-definitions" import { KV } from "@cross/kv" import { Prop } from "../common/prop.ts" @@ -18,26 +18,6 @@ import { pid } from "@cross/utils" const started = new Date() -/** - * Represents the current status of the application. - */ -export interface ApplicationState { - pid: number - version: string - status: string - updated: string - started: string - port: number - memory: ApiMemoryUsage - systemMemory: ApiSystemMemory - loadAvg: number[] - osUptime: number - osRelease: string - runtime: string - type: string - processes: ProcessInformation[] -} - /** * Represents the status of the application and provides methods to write the status to disk or store. */ @@ -60,7 +40,7 @@ class Status { * Key ["application_state", ] is written at most once per 20 seconds. * @param applicationState The application state to be stored. */ - public async writeToStore(applicationState: ApplicationState) { + public async writeToStore(applicationState: ApiApplicationState) { try { const kv = new KV({ autoSync: false }) await kv.open(this.storeName!) @@ -134,7 +114,7 @@ class Status { * @param processes The list of processes to retrieve the statuses from. * @returns The application state object. */ - public applicationState(processes: Process[], port?: Prop): ApplicationState { + public applicationState(processes: Process[], port?: Prop): ApiApplicationState { const processStates: ProcessInformation[] = [] for (const p of processes) { processStates.push(p.getStatus()) From a477d3e0f8e1c5ba48594ade0ad3b171100ba9fe Mon Sep 17 00:00:00 2001 From: Hexagon Date: Tue, 21 May 2024 21:26:00 +0200 Subject: [PATCH 04/21] Update deps. Test fixes. --- deno.json | 6 +++--- test/core/logger.test.ts | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/deno.json b/deno.json index 0e27cbd..3933e8e 100644 --- a/deno.json +++ b/deno.json @@ -38,15 +38,15 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@~0.11.0", + "@cross/kv": "jsr:@cross/kv@^0.12.0", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", "@cross/utils": "jsr:@cross/utils@~0.12.0", "@hexagon/croner": "jsr:@hexagon/croner@~8.0.2", "@oak/oak": "jsr:@oak/oak@~16.0.0", - "@pup/api-client": "jsr:@pup/api-client@~1.0.6", - "@pup/api-definitions": "jsr:@pup/api-definitions@~1.0.2", + "@pup/api-client": "jsr:@pup/api-client@~2.0.0", + "@pup/api-definitions": "jsr:@pup/api-definitions@~2.0.0", "@pup/common": "jsr:@pup/common@~1.0.3", "@pup/plugin": "jsr:@pup/plugin@~1.0.1", "@std/assert": "jsr:@std/assert@^0.225.2", diff --git a/test/core/logger.test.ts b/test/core/logger.test.ts index 3d02ceb..8488c7e 100644 --- a/test/core/logger.test.ts +++ b/test/core/logger.test.ts @@ -2,8 +2,9 @@ import { assertEquals, assertGreater } from "@std/assert" import { type AttachedLogger, type LogEventData, Logger } from "../../lib/core/logger.ts" import type { ProcessConfiguration } from "../../mod.ts" import { test } from "@cross/test" +import { tempfile } from "@cross/fs" -test("Logger - Creation with Global Configuration", () => { +test("Logger - Creation with Global Configuration", async () => { const globalConfig = { console: false, colors: true, @@ -12,12 +13,12 @@ test("Logger - Creation with Global Configuration", () => { stderr: "test_stderr.log", } - const logger = new Logger(globalConfig) + const logger = new Logger(globalConfig, await tempfile()) assertEquals(logger instanceof Logger, true) }) -test("Logger - Attachment of External Logger", () => { +test("Logger - Attachment of External Logger", async () => { let externalLoggerCalled = false let externalLoggerText = "" const expectedExteralLoggerText = "Testing attached logger" @@ -32,7 +33,7 @@ test("Logger - Attachment of External Logger", () => { return false } - const logger = new Logger({}) + const logger = new Logger({}, await tempfile()) logger.attach(externalLogger) logger.log("test", expectedExteralLoggerText) @@ -40,8 +41,8 @@ test("Logger - Attachment of External Logger", () => { assertEquals(externalLoggerText, expectedExteralLoggerText) }) -test("Logger - Logging with Different Methods", () => { - const logger = new Logger({ console: false }) +test("Logger - Logging with Different Methods", async () => { + const logger = new Logger({ console: false }, await tempfile()) logger.log("test", "Testing log method") logger.info("test", "Testing info method") @@ -52,8 +53,9 @@ test("Logger - Logging with Different Methods", () => { }) test("Logger - Logging Line Larger than KV Limit", async () => { - const tempStore = Deno.makeTempFileSync() + const tempStore = await tempfile() const logger = new Logger({ console: false }, tempStore) + await logger.init() let chars = 60000 let data = "" @@ -83,7 +85,7 @@ test("Logger - Logging Line Larger than KV Limit", async () => { }) test("Logger - File Writing with writeFile Method", async () => { - const logger = new Logger({ console: false }) + const logger = new Logger({ console: false }, await tempfile()) const testFileName = "test_writeFile.log" const testText = "Testing writeFile" await logger["writeFile"](testFileName, testText) @@ -95,8 +97,8 @@ test("Logger - File Writing with writeFile Method", async () => { }) test("Logger - getLogContents: Fetch all logs", async () => { - const tempStore = await Deno.makeTempDir() + "/.store" - const logger = new Logger({}, tempStore) + const logger = new Logger({}, await tempfile()) + await logger.init() const expectedLogs: LogEventData[] = [ { severity: "info", category: "test1", text: "Log 1", processId: "process-1", timeStamp: 1623626400000 }, @@ -116,6 +118,7 @@ test("Logger - getLogContents: Fetch all logs", async () => { test("Logger - getLogContents: Fetch logs by process ID", async () => { const tempStore = await Deno.makeTempDir() + "/.store" const logger = new Logger({}, tempStore) + await logger.init() const processId = "process-1" @@ -136,6 +139,7 @@ test("Logger - getLogContents: Fetch logs by process ID", async () => { test("Logger - getLogContents: Fetch logs by time range", async () => { const tempStore = await Deno.makeTempDir() + "/.store" const logger = new Logger({}, tempStore) + await logger.init() const startTimeStamp = 1623626400000 // 2023-06-13T12:00:00.000Z const endTimeStamp = 1623626500000 // 2023-06-13T12:01:40.000Z @@ -155,8 +159,8 @@ test("Logger - getLogContents: Fetch logs by time range", async () => { }) test("Logger - getLogContents: Fetch logs by process ID and time range", async () => { - const tempStore = await Deno.makeTempDir() + "/.store" - const logger = new Logger({}, tempStore) + const logger = new Logger({}, await tempfile()) + await logger.init() const processId = "process-1" const startTimeStamp = 1623626400000 // 2023-06-13T12:00:00.000Z From 9fd32633fb0a48ae946482101099a28206ec7299 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Tue, 21 May 2024 22:04:23 +0200 Subject: [PATCH 05/21] Fix tests --- lib/core/logger.ts | 15 ++++++--- package.json | 54 ++++++++++++++++---------------- test/core/logger.test.ts | 67 ++++++++++------------------------------ 3 files changed, 54 insertions(+), 82 deletions(-) diff --git a/lib/core/logger.ts b/lib/core/logger.ts index a6c187a..c07e22e 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -31,17 +31,17 @@ type AttachedLogger = (severity: string, category: string, text: string, process class Logger { private config: GlobalLoggerConfiguration = {} private attachedLogger?: AttachedLogger - private storeName: string + private storeName?: string private kv: KV - constructor(globalConfiguration: GlobalLoggerConfiguration, storeName: string) { + constructor(globalConfiguration: GlobalLoggerConfiguration, storeName?: string) { this.config = globalConfiguration this.storeName = storeName this.kv = new KV({ autoSync: false }) } public async init(): Promise { - await this.kv.open(this.storeName) + await this.kv.open(this.storeName!) } // Used for attaching the logger hook @@ -51,7 +51,7 @@ class Logger { // Prepare log event selector private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): KVQuery { - const key: KVQuery = processId ? ["logs_by_time", {}, processId] : ["logs_by_time"] + const key: KVQuery = ["logs_by_time"] if (startTimeStamp || endTimeStamp) { const rangeSelector: KVQueryRange = {} if (startTimeStamp) { @@ -61,6 +61,11 @@ class Logger { rangeSelector.to = endTimeStamp } key.push(rangeSelector) + } else if (processId) { + key.push({}) + } + if (processId) { + key.push(processId) } return key } @@ -200,7 +205,7 @@ class Logger { // Strip colors text = stripAnsi(text) try { - await writeFile(fileName, `${text}\n`, { mode: "a+" }) + await writeFile(fileName, `${text}\n`, { flag: "a+" }) } catch (_e) { if (!quiet) console.error(`Failed to write log '${fileName}'. The following message were not logged: ${text}.`) } diff --git a/package.json b/package.json index ba5054a..3ba647e 100644 --- a/package.json +++ b/package.json @@ -1,32 +1,32 @@ { "type": "module", "dependencies": { - "@cross/deepmerge": "npm:@jsr/cross__deepmerge@^1.0.0", - "@cross/env": "npm:@jsr/cross__env@^1.0.2", - "@cross/fs": "npm:@jsr/cross__fs@^0.1.11", - "@cross/jwt": "npm:@jsr/cross__jwt@^0.4.7", - "@cross/kv": "npm:@jsr/cross__kv@^0.11.0", - "@cross/runtime": "npm:@jsr/cross__runtime@^1.0.0", - "@cross/service": "npm:@jsr/cross__service@^1.0.3", - "@cross/test": "npm:@jsr/cross__test@^0.0.9", - "@cross/utils": "npm:@jsr/cross__utils@^0.12.0", - "@hexagon/croner": "npm:@jsr/hexagon__croner@^8.0.2", - "@oak/oak": "npm:@jsr/oak__oak@^16.0.0", - "@pup/api-client": "npm:@jsr/pup__api-client@^1.0.6", - "@pup/api-definitions": "npm:@jsr/pup__api-definitions@^1.0.2", - "@pup/common": "npm:@jsr/pup__common@^1.0.3", - "@pup/plugin": "npm:@jsr/pup__plugin@^1.0.1", - "@std/assert": "npm:@jsr/std__assert@^0.224.0", - "@std/async": "npm:@jsr/std__async@^0.224.0", - "@std/encoding": "npm:@jsr/std__encoding@^0.224.0", - "@std/io": "npm:@jsr/std__io@^0.224.0", - "@std/path": "npm:@jsr/std__path@^0.224.0", - "@std/semver": "npm:@jsr/std__semver@^0.224.0", - "dax-sh": "^0.41.0", - "filesize": "^10.1.2", - "json5": "^2.2.3", - "timeago.js": "^4.0.2", - "zod": "^3.23.8", - "zod-to-json-schema": "^3.23.0" + "@cross/deepmerge": "npm:@jsr/cross__deepmerge@~1.0.0", + "@cross/env": "npm:@jsr/cross__env@~1.0.2", + "@cross/fs": "npm:@jsr/cross__fs@~0.1.11", + "@cross/jwt": "npm:@jsr/cross__jwt@~0.4.7", + "@cross/kv": "npm:@jsr/cross__kv@~0.12.0", + "@cross/runtime": "npm:@jsr/cross__runtime@~1.0.0", + "@cross/service": "npm:@jsr/cross__service@~1.0.3", + "@cross/test": "npm:@jsr/cross__test@~0.0.9", + "@cross/utils": "npm:@jsr/cross__utils@~0.12.0", + "@hexagon/croner": "npm:@jsr/hexagon__croner@~8.0.2", + "@oak/oak": "npm:@jsr/oak__oak@~16.0.0", + "@pup/api-client": "npm:@jsr/pup__api-client@~2.0.0", + "@pup/api-definitions": "npm:@jsr/pup__api-definitions@~2.0.0", + "@pup/common": "npm:@jsr/pup__common@~1.0.3", + "@pup/plugin": "npm:@jsr/pup__plugin@~1.0.1", + "@std/assert": "npm:@jsr/std__assert@~0.224.0", + "@std/async": "npm:@jsr/std__async@~0.224.0", + "@std/encoding": "npm:@jsr/std__encoding@~0.224.0", + "@std/io": "npm:@jsr/std__io@~0.224.0", + "@std/path": "npm:@jsr/std__path@~0.224.0", + "@std/semver": "npm:@jsr/std__semver@~0.224.0", + "dax-sh": "~0.41.0", + "filesize": "~10.1.2", + "json5": "~2.2.3", + "timeago.js": "~4.0.2", + "zod": "~3.23.8", + "zod-to-json-schema": "~3.23.0" } } diff --git a/test/core/logger.test.ts b/test/core/logger.test.ts index 8488c7e..a949dad 100644 --- a/test/core/logger.test.ts +++ b/test/core/logger.test.ts @@ -1,10 +1,10 @@ -import { assertEquals, assertGreater } from "@std/assert" +import { assertEquals } from "@std/assert" import { type AttachedLogger, type LogEventData, Logger } from "../../lib/core/logger.ts" import type { ProcessConfiguration } from "../../mod.ts" import { test } from "@cross/test" -import { tempfile } from "@cross/fs" +import { readFile, tempfile, unlink } from "@cross/fs" -test("Logger - Creation with Global Configuration", async () => { +test("Logger - Creation with Global Configuration", () => { const globalConfig = { console: false, colors: true, @@ -13,7 +13,7 @@ test("Logger - Creation with Global Configuration", async () => { stderr: "test_stderr.log", } - const logger = new Logger(globalConfig, await tempfile()) + const logger = new Logger(globalConfig) assertEquals(logger instanceof Logger, true) }) @@ -33,67 +33,36 @@ test("Logger - Attachment of External Logger", async () => { return false } - const logger = new Logger({}, await tempfile()) + const logger = new Logger({}) logger.attach(externalLogger) - logger.log("test", expectedExteralLoggerText) + await logger.log("test", expectedExteralLoggerText) assertEquals(externalLoggerCalled, true) assertEquals(externalLoggerText, expectedExteralLoggerText) }) test("Logger - Logging with Different Methods", async () => { - const logger = new Logger({ console: false }, await tempfile()) + const logger = new Logger({ console: false }) - logger.log("test", "Testing log method") - logger.info("test", "Testing info method") - logger.warn("test", "Testing warn method") - logger.error("test", "Testing error method") + await logger.log("test", "Testing log method") + await logger.info("test", "Testing info method") + await logger.warn("test", "Testing warn method") + await logger.error("test", "Testing error method") assertEquals(true, true) // This is just to assert that the test passed if no errors are thrown }) -test("Logger - Logging Line Larger than KV Limit", async () => { - const tempStore = await tempfile() - const logger = new Logger({ console: false }, tempStore) - await logger.init() - - let chars = 60000 - let data = "" - while (chars--) { - data += "ๅœ‹" - } - await logger.log("test", data) - assertGreater(Deno.statSync(tempStore).size, 200_000) - - chars = 70000 - data = "" - while (chars--) { - data += "a" - } - await logger.log("test", data) - assertGreater(Deno.statSync(tempStore).size, 400_000) - - chars = 50000 - data = "" - while (chars--) { - data += "๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ" - } - await logger.log("test", data) - assertGreater(Deno.statSync(tempStore).size, 2_000_000) - - await Deno.remove(tempStore) -}) - test("Logger - File Writing with writeFile Method", async () => { - const logger = new Logger({ console: false }, await tempfile()) + const logger = new Logger({ console: false }) const testFileName = "test_writeFile.log" const testText = "Testing writeFile" await logger["writeFile"](testFileName, testText) - const fileContent = await Deno.readTextFile(testFileName) + const fileContentData = await readFile(testFileName) + const fileContent = new TextDecoder().decode(fileContentData) assertEquals(fileContent, `${testText}\n`) - await Deno.remove(testFileName) + await unlink(testFileName) }) test("Logger - getLogContents: Fetch all logs", async () => { @@ -116,8 +85,7 @@ test("Logger - getLogContents: Fetch all logs", async () => { }) test("Logger - getLogContents: Fetch logs by process ID", async () => { - const tempStore = await Deno.makeTempDir() + "/.store" - const logger = new Logger({}, tempStore) + const logger = new Logger({}, await tempfile()) await logger.init() const processId = "process-1" @@ -137,8 +105,7 @@ test("Logger - getLogContents: Fetch logs by process ID", async () => { }) test("Logger - getLogContents: Fetch logs by time range", async () => { - const tempStore = await Deno.makeTempDir() + "/.store" - const logger = new Logger({}, tempStore) + const logger = new Logger({}, await tempfile()) await logger.init() const startTimeStamp = 1623626400000 // 2023-06-13T12:00:00.000Z From 09f147752be85233dd785c960eeb408abc366cda Mon Sep 17 00:00:00 2001 From: Hexagon Date: Wed, 22 May 2024 23:56:00 +0200 Subject: [PATCH 06/21] Dependency update. Fix tests. --- deno.json | 2 +- lib/core/logger.ts | 14 ++++++++++++-- lib/core/pup.ts | 7 +++++-- package.json | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/deno.json b/deno.json index 3933e8e..1f2c387 100644 --- a/deno.json +++ b/deno.json @@ -38,7 +38,7 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.12.0", + "@cross/kv": "jsr:@cross/kv@^0.13.2", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", diff --git a/lib/core/logger.ts b/lib/core/logger.ts index c07e22e..16b0570 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -109,7 +109,7 @@ class Logger { // Write to persistent log store (if a name is supplied and internal logging is enabled) const logHours = this.config.internalLogHours === undefined ? 72 : this.config.internalLogHours - if (this.storeName && logHours > 0) { + if (this.kv.isOpen() && logHours > 0) { // Ignore errors when writing to log store try { const logObj: LogEventData = { @@ -232,7 +232,7 @@ class Logger { await this.internalLog("error", category, text, process, timestamp) } public async purge(keepHours: number): Promise { - if (!this.storeName) { + if (!this.kv?.isOpen()) { return 0 } @@ -252,6 +252,16 @@ class Logger { return 0 } } + /** + * Gracefully shut down the logger + */ + public async cleanup() { + try { + await this.kv?.close() + } catch (e) { + console.error("Error while closing kv store: " + e.message) + } + } } export { Logger } diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 2edb028..d58dabf 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -127,6 +127,9 @@ class Pup { // Unset last application state await this.status.cleanup() + + // Close logger + await this.logger.cleanup() } public init = async () => { @@ -514,11 +517,11 @@ class Pup { // Terminate api if (this.restApi) this.restApi.terminate() + await Promise.allSettled(stoppingProcesses) + // Cleanup await this.cleanup() - await Promise.allSettled(stoppingProcesses) - // Deno should exit gracefully now } diff --git a/package.json b/package.json index 3ba647e..46cc854 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "@cross/env": "npm:@jsr/cross__env@~1.0.2", "@cross/fs": "npm:@jsr/cross__fs@~0.1.11", "@cross/jwt": "npm:@jsr/cross__jwt@~0.4.7", - "@cross/kv": "npm:@jsr/cross__kv@~0.12.0", + "@cross/kv": "npm:@jsr/cross__kv@^0.13.2", "@cross/runtime": "npm:@jsr/cross__runtime@~1.0.0", "@cross/service": "npm:@jsr/cross__service@~1.0.3", "@cross/test": "npm:@jsr/cross__test@~0.0.9", From 98bca4746122c11d62d1ee1e1fafaa7660c28d84 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Wed, 22 May 2024 23:59:14 +0200 Subject: [PATCH 07/21] Update minimum version --- .github/workflows/deno.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deno.yaml b/.github/workflows/deno.yaml index 5566f77..06932de 100644 --- a/.github/workflows/deno.yaml +++ b/.github/workflows/deno.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - deno-version: [1.42.0, "v1.x"] + deno-version: [1.43.3, "v1.x"] steps: - name: Git Checkout From ab58541c0892d95035836cf7a4b7d7438359f65e Mon Sep 17 00:00:00 2001 From: Hexagon Date: Thu, 23 May 2024 00:01:34 +0200 Subject: [PATCH 08/21] Graceful shutdown in tests --- test/core/pup.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/core/pup.test.ts b/test/core/pup.test.ts index aae4eed..5d694ac 100644 --- a/test/core/pup.test.ts +++ b/test/core/pup.test.ts @@ -60,8 +60,8 @@ test("Create test process. Test start, block, stop, start, unblock, start in seq assertEquals(startResult3, true) assertEquals(testProcess?.getStatus().status, ApiProcessState.STARTING) - // Terminate pup instantly - await pup.terminate(0) + // Terminate pup, allow 2.5 seconds for graceful shutdown + await pup.terminate(2500) }) test("Create test cluster. Test start, block, stop, start, unblock, start in sequence.", async () => { @@ -117,6 +117,6 @@ test("Create test cluster. Test start, block, stop, start, unblock, start in seq assertEquals(startResult3, true) assertEquals(testProcess?.getStatus().status, ApiProcessState.STARTING) - // Terminate pup instantly - await pup.terminate(0) + // Terminate pup, allow 2.5 seconds for graceful shutdown + await pup.terminate(2500) }) From 6355e6597709798ffd22576818780b8b25db95f0 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Thu, 23 May 2024 00:08:35 +0200 Subject: [PATCH 09/21] Rename databases not to collide with existing databases --- lib/core/pup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/pup.ts b/lib/core/pup.ts index d58dabf..7f9f13f 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -81,10 +81,10 @@ class Pup { this.persistentStoragePath = persistentStoragePath - statusFile = `${this.persistentStoragePath}/.main.status` // Deno KV store + statusFile = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store secretFile = `${this.persistentStoragePath}/.main.secret` // Plain text file containing the JWT secret for the rest api portFile = `${this.temporaryStoragePath}/.main.port` // Plain text file containing the port number for the API - logStore = `${this.persistentStoragePath}/.main.log` // Deno KV store + logStore = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store } // Throw on invalid configuration From 75414a38be26dad8ff2f54f24cb4181a6a20882a Mon Sep 17 00:00:00 2001 From: Hexagon Date: Thu, 23 May 2024 00:19:04 +0200 Subject: [PATCH 10/21] Extra grace --- lib/core/pup.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 7f9f13f..6b791ab 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -32,6 +32,7 @@ import { findFreePort } from "./port.ts" import { Plugin } from "./plugin.ts" import { GenerateToken, SecondsToExpiry } from "../common/token.ts" import { CurrentRuntime, Runtime } from "@cross/runtime" +import { delay } from "@std/async" interface InstructionResponse { success: boolean action?: string @@ -522,6 +523,11 @@ class Pup { // Cleanup await this.cleanup() + // Allow some extra time to pass to allow untracked async tasks + // (such as logs about closing down) to finish + // - But only if at least 500ms were used as grace period + if (forceQuitMs >= 500) await delay(500) + // Deno should exit gracefully now } From c54db0babf9b549e1820159bed21fc5914fa8686 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Sat, 1 Jun 2024 21:54:51 +0200 Subject: [PATCH 11/21] Various fixes --- README.md | 2 +- deno.json | 4 +- docs/src/_data.json | 2 +- docs/src/changelog.md | 2 +- .../src/examples/basic-webinterface/pup.jsonc | 2 +- lib/cli/main.ts | 19 +++---- lib/core/api.ts | 2 +- lib/core/logger.ts | 33 ++++++------ lib/core/pup.ts | 2 +- lib/core/status.ts | 52 ++++++++++--------- package.json | 32 ------------ 11 files changed, 62 insertions(+), 90 deletions(-) delete mode 100644 package.json diff --git a/README.md b/README.md index 7debbe0..172dffe 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ _For detailed documentation, visit [pup.56k.guru](https://pup.56k.guru)._ To install Pup, make sure you run the latest version of Deno (`deno upgrade`), then open your terminal and execute the following command: ```bash -deno run -Ar jsr:@pup/pup@1.0.0-rc.39 setup --channel prerelease +deno run -Ar jsr:@pup/pup@1.0.0-rc.40 setup --channel prerelease ``` This command downloads the latest version of Pup and installs it on your system. The `--channel prerelease` option is included as there is no stable version of Pup yet. Read more abour release diff --git a/deno.json b/deno.json index 1f2c387..6d06eb9 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@pup/pup", - "version": "1.0.0-rc.39", + "version": "1.0.0-rc.40", "exports": { ".": "./pup.ts", @@ -38,7 +38,7 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.13.2", + "@cross/kv": "jsr:@cross/kv@^0.15.6", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", diff --git a/docs/src/_data.json b/docs/src/_data.json index 4cf688d..b792588 100644 --- a/docs/src/_data.json +++ b/docs/src/_data.json @@ -6,7 +6,7 @@ "description": "Universal Process Manager" }, "substitute": { - "$PUP_VERSION": "1.0.0-rc.39" + "$PUP_VERSION": "1.0.0-rc.40" }, "top_links": [ { diff --git a/docs/src/changelog.md b/docs/src/changelog.md index 77d204d..253a4a2 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -12,7 +12,7 @@ All notable changes to this project will be documented in this section. ## [1.0.0-rc.40] - Unreleased - fix(core): Replace `Deno.Kv` with `@cross/kv` for cross-runtime compatibility, more compact logs and avoiding `--unstable` -- fix(core): Make Pup work in Node and Bun +- fix(core): Internal changes to support Node and Bun ## [1.0.0-rc.39] - 2024-05-04 diff --git a/docs/src/examples/basic-webinterface/pup.jsonc b/docs/src/examples/basic-webinterface/pup.jsonc index 6fab97a..a77e925 100644 --- a/docs/src/examples/basic-webinterface/pup.jsonc +++ b/docs/src/examples/basic-webinterface/pup.jsonc @@ -50,7 +50,7 @@ "plugins": [ { // Use full uri to plugin, e.g. jsr:@pup/plugin-web-interface - "url": "jsr:@pup/plugin-web-interface", + "url": "jsr:@pup/plugin-web-interface@^2.0.2", "options": { "port": 8002 } diff --git a/lib/cli/main.ts b/lib/cli/main.ts index a10a729..16cd1e4 100644 --- a/lib/cli/main.ts +++ b/lib/cli/main.ts @@ -429,22 +429,19 @@ async function main() { * Base argument: logs */ if (baseArgument === "logs") { - const logStore = `${await toPersistentPath(configFile as string)}/.main.log` + const logStore = `${await toPersistentPath(configFile as string)}/.main.log.ckvdb` const logger = new Logger(configuration!.logger || {}, logStore) await logger.init() const startTimestamp = checkedArgs.get("start") ? new Date(Date.parse(checkedArgs.get("start")!)).getTime() : undefined const endTimestamp = checkedArgs.get("end") ? new Date(Date.parse(checkedArgs.get("end")!)).getTime() : undefined const numberOfRows = checkedArgs.get("n") ? parseInt(checkedArgs.get("n")!, 10) : undefined - let logs = await logger.getLogContents(checkedArgs.get("id"), startTimestamp, endTimestamp) - logs = logs.filter((log) => { - const { processId, severity } = log - const severityFilter = !checkedArgs.get("severity") || checkedArgs.get("severity") === "" || checkedArgs.get("severity")!.toLowerCase() === severity.toLowerCase() - const processFilter = !checkedArgs.get("id") || checkedArgs.get("id") === "" || checkedArgs.get("id")!.toLowerCase() === processId.toLowerCase() - return severityFilter && processFilter - }) - if (numberOfRows) { - logs = logs.slice(-numberOfRows) - } + const logs = await logger.getLogContents( + (!checkedArgs.get("id") || checkedArgs.get("id") === "") ? undefined : checkedArgs.get("id")!.toLowerCase(), + startTimestamp, + endTimestamp, + (!checkedArgs.get("severity") || checkedArgs.get("severity") === "") ? undefined : checkedArgs.get("severity")!.toLowerCase(), + numberOfRows, + ) if (logs && logs.length > 0) { const logWithColors = configuration!.logger?.colors ?? true for (const log of logs) { diff --git a/lib/core/api.ts b/lib/core/api.ts index 094fbe2..013dc05 100644 --- a/lib/core/api.ts +++ b/lib/core/api.ts @@ -100,6 +100,6 @@ export class PupApi { this._pup.logger[severity](`api-${consumer}`, message) } public async getLogs(processId?: string, startTimeStamp?: number, endTimeStamp?: number, nRows?: number): Promise { - return await this._pup.logger.getLogContents(processId, startTimeStamp, endTimeStamp, nRows) + return await this._pup.logger.getLogContents(processId, startTimeStamp, endTimeStamp, undefined, nRows) } } diff --git a/lib/core/logger.ts b/lib/core/logger.ts index 16b0570..f304224 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -50,7 +50,7 @@ class Logger { } // Prepare log event selector - private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): KVQuery { + private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number, severity?: string): KVQuery { const key: KVQuery = ["logs_by_time"] if (startTimeStamp || endTimeStamp) { const rangeSelector: KVQueryRange = {} @@ -61,44 +61,46 @@ class Logger { rangeSelector.to = endTimeStamp } key.push(rangeSelector) - } else if (processId) { + } else if (processId || severity) { key.push({}) } if (processId) { key.push(processId) + } else { + key.push({}) + } + if (severity) { + key.push(severity) } return key } // Fetch logs from store private async fetchLogsFromStore(selector: KVQuery, nRows?: number): Promise { - const result = await this.kv.listAll(selector) const resultArray: LogEventData[] = [] - for await (const res of result) { - resultArray.push(res.data as LogEventData) - } - if (nRows) { - const spliceNumber = Math.max(0, resultArray.length - nRows) - resultArray.splice(0, spliceNumber) + + // Use the generator for efficient iteration + for await (const { data } of this.kv.iterate(selector, nRows, true)) { + resultArray.unshift(data) } return resultArray } - public async getLogContents(processId?: string, startTimeStamp?: number, endTimeStamp?: number, nRows?: number): Promise { - const selector = this.prepareSelector(processId, startTimeStamp, endTimeStamp) + public async getLogContents(processId?: string, startTimeStamp?: number, endTimeStamp?: number, severity?: string, nRows?: number): Promise { + const selector = this.prepareSelector(processId, startTimeStamp, endTimeStamp, severity) return await this.fetchLogsFromStore(selector, nRows) } public async getLogsByProcess(processId: string, nRows?: number): Promise { - return await this.getLogContents(processId, undefined, undefined, nRows) + return await this.getLogContents(processId, undefined, undefined, undefined, nRows) } public async getLogsByTime(startTimeStamp: number, endTimeStamp: number, nRows?: number): Promise { - return await this.getLogContents(undefined, startTimeStamp, endTimeStamp, nRows) + return await this.getLogContents(undefined, startTimeStamp, endTimeStamp, undefined, nRows) } public async getLogsByProcessAndTime(processId: string, startTimeStamp: number, endTimeStamp: number, nRows?: number): Promise { - return await this.getLogContents(processId, startTimeStamp, endTimeStamp, nRows) + return await this.getLogContents(processId, startTimeStamp, endTimeStamp, undefined, nRows) } private async internalLog(severity: string, category: string, text: string, process?: ProcessConfiguration, timeStamp?: number) { @@ -119,7 +121,8 @@ class Logger { processId: initiator, timeStamp: timeStamp, } - await this.kv.set(["logs_by_time", timeStamp, initiator], logObj) + // Append a random uuid to the key, in case two logs should arrive at the same time + await this.kv.set(["logs_by_time", timeStamp, initiator, severity, crypto.randomUUID()], logObj) } catch (e) { console.error("Error while writing to log store", e) } diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 6b791ab..d8c7fe8 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -82,7 +82,7 @@ class Pup { this.persistentStoragePath = persistentStoragePath - statusFile = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store + statusFile = `${this.persistentStoragePath}/.main.status.ckvdb` // Cross/KV store secretFile = `${this.persistentStoragePath}/.main.secret` // Plain text file containing the JWT secret for the rest api portFile = `${this.temporaryStoragePath}/.main.port` // Plain text file containing the port number for the API logStore = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store diff --git a/lib/core/status.ts b/lib/core/status.ts index 827995c..a8c7b89 100644 --- a/lib/core/status.ts +++ b/lib/core/status.ts @@ -41,26 +41,28 @@ class Status { * @param applicationState The application state to be stored. */ public async writeToStore(applicationState: ApiApplicationState) { - try { - const kv = new KV({ autoSync: false }) - await kv.open(this.storeName!) + if (this.storeName) { + try { + const kv = new KV({ autoSync: false, disableIndex: true }) + await kv.open(this.storeName) - // Initialize lastWrite if it's not set - if (!this.lastWrite) { - this.lastWrite = 0 - } + // Initialize lastWrite if it's not set + if (!this.lastWrite) { + this.lastWrite = 0 + } - // Write application_state at most once per APPLICATION_STATE_WRITE_LIMIT_MS - if (Date.now() - this.lastWrite > APPLICATION_STATE_WRITE_LIMIT_MS) { - this.lastWrite = Date.now() - await kv.set(["application_state", Date.now()], applicationState) - } + // Write application_state at most once per APPLICATION_STATE_WRITE_LIMIT_MS + if (Date.now() - this.lastWrite > APPLICATION_STATE_WRITE_LIMIT_MS) { + this.lastWrite = Date.now() + await kv.set(["application_state", Date.now()], applicationState) + } - // Always write last_application_state - await kv.set(["last_application_state"], applicationState) - await kv.close() - } catch (e) { - console.error("Error while writing status to kv store: " + e.message) + // Always write last_application_state + await kv.set(["last_application_state"], applicationState) + await kv.close() + } catch (e) { + console.error("Error while writing status to kv store: " + e.message) + } } } @@ -69,13 +71,15 @@ class Status { * unsetting last_application_state in the kv store. */ public async cleanup() { - try { - const kv = new KV({ autoSync: false }) - await kv.open(this.storeName!) - await kv.delete(["last_application_state"]) - await kv.close() - } catch (e) { - console.error("Error while writing status to kv store: " + e.message) + if (this.storeName) { + try { + const kv = new KV({ autoSync: false, disableIndex: true }) + await kv.open(this.storeName) + await kv.delete(["last_application_state"]) + await kv.close() + } catch (e) { + console.error("Error while writing status to kv store: " + e.message) + } } } diff --git a/package.json b/package.json deleted file mode 100644 index 46cc854..0000000 --- a/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "type": "module", - "dependencies": { - "@cross/deepmerge": "npm:@jsr/cross__deepmerge@~1.0.0", - "@cross/env": "npm:@jsr/cross__env@~1.0.2", - "@cross/fs": "npm:@jsr/cross__fs@~0.1.11", - "@cross/jwt": "npm:@jsr/cross__jwt@~0.4.7", - "@cross/kv": "npm:@jsr/cross__kv@^0.13.2", - "@cross/runtime": "npm:@jsr/cross__runtime@~1.0.0", - "@cross/service": "npm:@jsr/cross__service@~1.0.3", - "@cross/test": "npm:@jsr/cross__test@~0.0.9", - "@cross/utils": "npm:@jsr/cross__utils@~0.12.0", - "@hexagon/croner": "npm:@jsr/hexagon__croner@~8.0.2", - "@oak/oak": "npm:@jsr/oak__oak@~16.0.0", - "@pup/api-client": "npm:@jsr/pup__api-client@~2.0.0", - "@pup/api-definitions": "npm:@jsr/pup__api-definitions@~2.0.0", - "@pup/common": "npm:@jsr/pup__common@~1.0.3", - "@pup/plugin": "npm:@jsr/pup__plugin@~1.0.1", - "@std/assert": "npm:@jsr/std__assert@~0.224.0", - "@std/async": "npm:@jsr/std__async@~0.224.0", - "@std/encoding": "npm:@jsr/std__encoding@~0.224.0", - "@std/io": "npm:@jsr/std__io@~0.224.0", - "@std/path": "npm:@jsr/std__path@~0.224.0", - "@std/semver": "npm:@jsr/std__semver@~0.224.0", - "dax-sh": "~0.41.0", - "filesize": "~10.1.2", - "json5": "~2.2.3", - "timeago.js": "~4.0.2", - "zod": "~3.23.8", - "zod-to-json-schema": "~3.23.0" - } -} From 8ae63f5129dbb42aa64c37b967e1a163307cd8e5 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Sat, 1 Jun 2024 22:09:46 +0200 Subject: [PATCH 12/21] Lint: Adding explicit return types to public api --- lib/core/cluster.ts | 12 ++++++------ lib/core/process.ts | 6 +++--- lib/core/pup.ts | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/core/cluster.ts b/lib/core/cluster.ts index 8bdf86b..e7002e8 100644 --- a/lib/core/cluster.ts +++ b/lib/core/cluster.ts @@ -25,7 +25,7 @@ class Cluster extends Process { this.setInstances(this.config.cluster?.instances || 1) } - public override start = async (reason?: string, restart?: boolean) => { + public override start = async (reason?: string, restart?: boolean): Promise => { await Promise.all( this.processes.map((process) => process.start(reason, restart)), ) @@ -35,7 +35,7 @@ class Cluster extends Process { return true } - public setInstances = (nInstances: number) => { + public setInstances = (nInstances: number): void => { const backends = [] // ToDo: If there already are processes, reuse, stop or add @@ -125,15 +125,15 @@ class Cluster extends Process { return Promise.allSettled(results).then((results) => results.every((result) => result)) } - public override restart = (reason: string) => { + public override restart = (reason: string): void => { this.processes.forEach((process) => process.restart(reason)) } - public override block = (reason: string) => { + public override block = (reason: string): void => { this.processes.forEach((process) => process.block(reason)) } - public override unblock = (reason: string) => { + public override unblock = (reason: string): void => { this.processes.forEach((process) => process.unblock(reason)) } @@ -175,7 +175,7 @@ class Cluster extends Process { return clusterStatus } - public cleanup = () => { + public cleanup = (): void => { this.loadBalancerWorker?.terminate() this.loadBalancerWorker = null } diff --git a/lib/core/process.ts b/lib/core/process.ts index e9f9bd4..3327389 100644 --- a/lib/core/process.ts +++ b/lib/core/process.ts @@ -333,7 +333,7 @@ class Process { * Stops the process. * @param {string} reason - The reason for stopping the process. */ - public restart = (reason: string) => { + public restart = (reason: string): void => { this.stop(reason) this.pendingRestartReason = reason } @@ -342,7 +342,7 @@ class Process { * Blocks the process. * @param {string} reason - The reason for blocking the process. */ - public block = (reason: string) => { + public block = (reason: string): void => { this.blocked = true this.pup.logger.log("block", `Process blocked, reason: ${reason}`, this.config) } @@ -351,7 +351,7 @@ class Process { * Unblocks the process. * @param {string} reason - The reason for unblocking the process. */ - public unblock = (reason: string) => { + public unblock = (reason: string): void => { this.blocked = false this.pup.logger.log("unblock", `Process unblocked, reason: ${reason}`, this.config) } diff --git a/lib/core/pup.ts b/lib/core/pup.ts index d8c7fe8..0545918 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -116,7 +116,7 @@ class Pup { * This is intended to be called by global unload event * and clears any stray files */ - public cleanup = async () => { + public cleanup = async (): Promise => { for (const cleanupFilePath of this.cleanupQueue) { try { await rm(cleanupFilePath, { recursive: true }) @@ -133,7 +133,7 @@ class Pup { await this.logger.cleanup() } - public init = async () => { + public init = async (): Promise => { // Intialize logging await this.logger.init() From c02a232ca65ea822ca4c656e49e7f4b919dd2b26 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Tue, 4 Jun 2024 21:50:42 +0200 Subject: [PATCH 13/21] Dependency update. Remove sysinfo-module. --- deno.json | 4 +- docs/src/changelog.md | 1 + .../src/examples/basic-webinterface/pup.jsonc | 2 +- lib/common/sysinfo.ts | 79 ------------------- lib/core/status.ts | 2 +- 5 files changed, 5 insertions(+), 83 deletions(-) delete mode 100644 lib/common/sysinfo.ts diff --git a/deno.json b/deno.json index 6d06eb9..febf147 100644 --- a/deno.json +++ b/deno.json @@ -38,11 +38,11 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.15.6", + "@cross/kv": "jsr:@cross/kv@^0.15.8", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", - "@cross/utils": "jsr:@cross/utils@~0.12.0", + "@cross/utils": "jsr:@cross/utils@~0.13.0", "@hexagon/croner": "jsr:@hexagon/croner@~8.0.2", "@oak/oak": "jsr:@oak/oak@~16.0.0", "@pup/api-client": "jsr:@pup/api-client@~2.0.0", diff --git a/docs/src/changelog.md b/docs/src/changelog.md index 253a4a2..d9a6b28 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this section. - fix(core): Replace `Deno.Kv` with `@cross/kv` for cross-runtime compatibility, more compact logs and avoiding `--unstable` - fix(core): Internal changes to support Node and Bun +- fix(core): Removes the `sysinfo.ts`-module in favor of `@cross/utils/sysinfo` ## [1.0.0-rc.39] - 2024-05-04 diff --git a/docs/src/examples/basic-webinterface/pup.jsonc b/docs/src/examples/basic-webinterface/pup.jsonc index a77e925..6fab97a 100644 --- a/docs/src/examples/basic-webinterface/pup.jsonc +++ b/docs/src/examples/basic-webinterface/pup.jsonc @@ -50,7 +50,7 @@ "plugins": [ { // Use full uri to plugin, e.g. jsr:@pup/plugin-web-interface - "url": "jsr:@pup/plugin-web-interface@^2.0.2", + "url": "jsr:@pup/plugin-web-interface", "options": { "port": 8002 } diff --git a/lib/common/sysinfo.ts b/lib/common/sysinfo.ts deleted file mode 100644 index f4b4968..0000000 --- a/lib/common/sysinfo.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { CurrentRuntime, Runtime } from "@cross/runtime" -import { ApiMemoryUsage, ApiSystemMemory } from "@pup/api-definitions" -import { freemem, loadavg, totalmem, uptime as nodeUptime } from "node:os" - -export function memoryUsage(): ApiMemoryUsage { - let memoryUsageResult: ApiMemoryUsage - if (CurrentRuntime === Runtime.Deno) { - //@ts-ignore cross-runtime - const { external, heapTotal, heapUsed, rss } = Deno.memoryUsage() - memoryUsageResult = { external, heapTotal, heapUsed, rss } - } else if ( - CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun - ) { - //@ts-ignore cross-runtime - const { external = 0, heapTotal, heapUsed, rss } = process.memoryUsage() - memoryUsageResult = { external, heapTotal, heapUsed, rss } - } else { - memoryUsageResult = { external: 0, heapTotal: 0, heapUsed: 0, rss: 0 } - } - return memoryUsageResult -} - -export function loadAvg(): number[] { - let loadAvgResult: number[] - if (CurrentRuntime === Runtime.Deno) { - loadAvgResult = Deno.loadavg() - } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { - // Node.js and Bun provide os module for loadAvg - loadAvgResult = loadavg() - } else { - // Unsupported runtime - loadAvgResult = [] - } - return loadAvgResult -} - -export function uptime(): number { - let uptimeResult: number - if (CurrentRuntime === Runtime.Deno) { - uptimeResult = Deno.osUptime() - } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { - // Node.js and Bun provide os module for uptime - uptimeResult = nodeUptime() - } else { - uptimeResult = -1 - } - return uptimeResult -} - -export function systemMemoryInfo(): ApiSystemMemory { - let memoryInfoResult: ApiSystemMemory - if (CurrentRuntime === Runtime.Deno) { - memoryInfoResult = Deno.systemMemoryInfo() - } else if (CurrentRuntime === Runtime.Node || CurrentRuntime === Runtime.Bun) { - // Node.js and Bun don't have a direct equivalent to Deno.systemMemoryInfo - // We can try to approximate values using os module (limited information) - memoryInfoResult = { - total: totalmem(), - free: freemem(), - available: -1, // Not directly available - buffers: -1, // Not directly available - cached: -1, // Not directly available - swapTotal: -1, // Approximate swap total - swapFree: -1, // Not directly available - } - } else { - // Unsupported runtime - memoryInfoResult = { - total: -1, - free: -1, - available: -1, - buffers: -1, - cached: -1, - swapTotal: -1, - swapFree: -1, - } - } - return memoryInfoResult -} diff --git a/lib/core/status.ts b/lib/core/status.ts index a8c7b89..274340a 100644 --- a/lib/core/status.ts +++ b/lib/core/status.ts @@ -12,7 +12,7 @@ import { ApiApplicationState, ApiProcessState } from "@pup/api-definitions" import { KV } from "@cross/kv" import { Prop } from "../common/prop.ts" -import { loadAvg, memoryUsage, systemMemoryInfo, uptime } from "../common/sysinfo.ts" +import { loadAvg, memoryUsage, systemMemoryInfo, uptime } from "@cross/utils/sysinfo" import { getCurrentOS, getCurrentRuntime, getCurrentVersion } from "@cross/runtime" import { pid } from "@cross/utils" From 022f0d1c26a4c27ba243610e48f2a24790bc34a6 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Fri, 7 Jun 2024 01:10:00 +0200 Subject: [PATCH 14/21] Add script to generate package.json, and publish to npm. --- .gitignore | 8 ++++- README.md | 12 +++++-- application.meta.ts | 1 + deno.json | 4 ++- tools/generate-package-json.ts | 62 ++++++++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 tools/generate-package-json.ts diff --git a/.gitignore b/.gitignore index 32354fd..ffc3218 100644 --- a/.gitignore +++ b/.gitignore @@ -45,9 +45,15 @@ coverage # Lumocs generated site _site -# Node files +# Node/npm files package-lock.json node_modules +.npmrc +package.json +*.tgz + +# Bun files +.bun.lockb # VSCode files .vscode \ No newline at end of file diff --git a/README.md b/README.md index 172dffe..a12a040 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![PUP](https://cdn.jsdelivr.net/gh/hexagon/pup@master/docs/src/resources/pup_dark.png) -Pup is a powerful universal process manager developed with Deno, designed to keep your scripts, applications and services alive. +Pup is a powerful universal process manager developed with JavaScript, designed to keep your scripts, applications and services alive. _For detailed documentation, visit [pup.56k.guru](https://pup.56k.guru)._ @@ -25,12 +25,20 @@ _For detailed documentation, visit [pup.56k.guru](https://pup.56k.guru)._ ### Installation -To install Pup, make sure you run the latest version of Deno (`deno upgrade`), then open your terminal and execute the following command: +To install Pup, make sure you run the latest version of your runtime environment, then open your terminal and execute the following command: + +**Deno**: ```bash deno run -Ar jsr:@pup/pup@1.0.0-rc.40 setup --channel prerelease ``` +**Bun**: + +```bash +bun run @hexagon/pup setup --channel prerelease +``` + This command downloads the latest version of Pup and installs it on your system. The `--channel prerelease` option is included as there is no stable version of Pup yet. Read more abour release channels [here](https://hexagon.github.io/pup/installation.html#release-channels). diff --git a/application.meta.ts b/application.meta.ts index 9de9049..a333afa 100644 --- a/application.meta.ts +++ b/application.meta.ts @@ -23,6 +23,7 @@ const Application = { name: "pup", version: "1.0.0-rc.40", url: "jsr:@pup/pup@$VERSION", + description: "Powerful universal process manager, designed to keep your scripts, applications and services alive.", canary_url: "https://raw.githubusercontent.com/Hexagon/pup/main/pup.ts", deno: "1.43.0", /* Minimum stable version of Deno required to run Pup (without --unstable-* flags) */ deno_unstable: "1.43.0", /* Minimum version of Deno required to run Pup (with --unstable-* flags) */ diff --git a/deno.json b/deno.json index febf147..b1cce77 100644 --- a/deno.json +++ b/deno.json @@ -30,7 +30,9 @@ "check-coverage": "deno task check && genhtml cov_profile.lcov --output-directory cov_profile/html && lcov --list cov_profile.lcov && deno run --allow-net --allow-read https://deno.land/std/http/file_server.ts cov_profile/html", "build-schema": "deno run --allow-write --allow-read --allow-env=XDG_DATA_HOME,HOME tools/build-schema.ts && deno fmt", "build-versions": "deno run --allow-read --allow-write --allow-env tools/release.ts && deno fmt", - "build": "deno task check && deno task build-schema && deno task build-versions" + "generate-package-json": "deno run -A tools/generate-package-json.ts", + "build": "deno task check && deno task build-schema && deno task build-versions && deno task build-package-json && npm pack", + "publish-npm": "deno task build && npm publish" }, "imports": { diff --git a/tools/generate-package-json.ts b/tools/generate-package-json.ts new file mode 100644 index 0000000..69eade0 --- /dev/null +++ b/tools/generate-package-json.ts @@ -0,0 +1,62 @@ +import { $ } from "jsr:@david/dax" + +import { Application } from "../application.meta.ts" + +import config from "../deno.json" with { type: "json" } + +// Remove existing package.json +try { + Deno.remove("./package.json") +} catch (_e) { + console.warn("Existing package.json could not be removed.") +} + +// Construct package.json Data +const packageJson = { + type: "module", + name: `@hexagon/${Application.name}`, + version: Application.version, + description: Application.description, + module: "./mod.ts", + bin: "./pup.ts", + files: [ + "lib/*", + "application.meta.ts", + "mod.ts", + "pup.ts", + "versions.json", + ], +} + +// Write initial package.json File +await Deno.writeTextFile("./package.json", JSON.stringify(packageJson, null, 2)) +console.log("package.json created successfully!") + +// 4. Install Dependencies Using jsr and npm +const npmDepdencies = [] +const jsrDepdencies = [] +for (const dependency in config.imports) { + if ((config.imports as Record)[dependency].startsWith("npm:")) { + npmDepdencies.push(dependency) + console.log(`Found npm dependency: ${dependency}`) + } else { + jsrDepdencies.push(dependency) + console.log(`Found jsr dependency: ${dependency}`) + } +} +if (npmDepdencies.length) { + await $.raw`npm i ${npmDepdencies.join(" ")}` +} +if (jsrDepdencies.length) { + await $.raw`npx jsr add ${jsrDepdencies.join(" ")}` +} + +// Read updated package.json +const updatedPackageJson = JSON.parse(await Deno.readTextFile("./package.json")) + +// Add all jsr deps as bundledDependencies +updatedPackageJson.bundledDependencies = jsrDepdencies + +// Write updated package.json +await Deno.writeTextFile("./package.json", JSON.stringify(updatedPackageJson, null, 2)) +console.log("package.json created successfully!") From be48f74e599f1dbed94804df4d3371083dc87231 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Fri, 7 Jun 2024 01:17:26 +0200 Subject: [PATCH 15/21] Add croner from npm --- deno.json | 2 +- lib/core/process.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deno.json b/deno.json index b1cce77..078c861 100644 --- a/deno.json +++ b/deno.json @@ -45,7 +45,6 @@ "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", "@cross/utils": "jsr:@cross/utils@~0.13.0", - "@hexagon/croner": "jsr:@hexagon/croner@~8.0.2", "@oak/oak": "jsr:@oak/oak@~16.0.0", "@pup/api-client": "jsr:@pup/api-client@~2.0.0", "@pup/api-definitions": "jsr:@pup/api-definitions@~2.0.0", @@ -57,6 +56,7 @@ "@std/io": "jsr:@std/io@~0.224.0", "@std/path": "jsr:@std/path@^0.225.1", "@std/semver": "jsr:@std/semver@~0.224.0", + "croner": "npm:croner@^8.0.2", "dax-sh": "npm:dax-sh@~0.41.0", "filesize": "npm:filesize@~10.1.1", "json5": "npm:json5@~2.2.3", diff --git a/lib/core/process.ts b/lib/core/process.ts index 3327389..c77ccc6 100644 --- a/lib/core/process.ts +++ b/lib/core/process.ts @@ -12,7 +12,7 @@ import type { ProcessConfiguration } from "./configuration.ts" import { Watcher } from "./watcher.ts" import type { ApiTelemetryData } from "@pup/api-definitions" -import { Cron } from "@hexagon/croner" +import { Cron } from "croner" import { delay } from "@std/async" import { ApiProcessState } from "@pup/api-definitions" From dd2206ff54da570748aca1f6063a7af19ac4a4b0 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Fri, 7 Jun 2024 21:49:05 +0200 Subject: [PATCH 16/21] Update @cross/kv. Use jsr croner instead of npm. --- deno.json | 4 ++-- lib/core/process.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deno.json b/deno.json index 078c861..e74dd6e 100644 --- a/deno.json +++ b/deno.json @@ -40,11 +40,12 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.15.8", + "@cross/kv": "jsr:@cross/kv@^0.15.9", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", "@cross/utils": "jsr:@cross/utils@~0.13.0", + "@hexagon/croner": "jsr:@hexagon/croner@^8.0.2", "@oak/oak": "jsr:@oak/oak@~16.0.0", "@pup/api-client": "jsr:@pup/api-client@~2.0.0", "@pup/api-definitions": "jsr:@pup/api-definitions@~2.0.0", @@ -56,7 +57,6 @@ "@std/io": "jsr:@std/io@~0.224.0", "@std/path": "jsr:@std/path@^0.225.1", "@std/semver": "jsr:@std/semver@~0.224.0", - "croner": "npm:croner@^8.0.2", "dax-sh": "npm:dax-sh@~0.41.0", "filesize": "npm:filesize@~10.1.1", "json5": "npm:json5@~2.2.3", diff --git a/lib/core/process.ts b/lib/core/process.ts index c77ccc6..3327389 100644 --- a/lib/core/process.ts +++ b/lib/core/process.ts @@ -12,7 +12,7 @@ import type { ProcessConfiguration } from "./configuration.ts" import { Watcher } from "./watcher.ts" import type { ApiTelemetryData } from "@pup/api-definitions" -import { Cron } from "croner" +import { Cron } from "@hexagon/croner" import { delay } from "@std/async" import { ApiProcessState } from "@pup/api-definitions" From f51db1bafaa9b350e89b0b2e4607d443fbba1490 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Fri, 7 Jun 2024 22:34:06 +0200 Subject: [PATCH 17/21] Update @cross/kv --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index e74dd6e..a1e7706 100644 --- a/deno.json +++ b/deno.json @@ -40,7 +40,7 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.15.9", + "@cross/kv": "jsr:@cross/kv@^0.15.10", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", From e729e3c6248e05c220869072af12f318a2b237de Mon Sep 17 00:00:00 2001 From: Hexagon Date: Mon, 10 Jun 2024 00:26:04 +0200 Subject: [PATCH 18/21] Dependency update + Small cross-runtime fix. --- deno.json | 2 +- lib/types/runner.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deno.json b/deno.json index a1e7706..cf32a22 100644 --- a/deno.json +++ b/deno.json @@ -40,7 +40,7 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@^0.15.10", + "@cross/kv": "jsr:@cross/kv@~0.15.11", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", diff --git a/lib/types/runner.ts b/lib/types/runner.ts index b4d0819..d829157 100644 --- a/lib/types/runner.ts +++ b/lib/types/runner.ts @@ -13,7 +13,7 @@ export type RunnerCallback = (pid?: number) => void export interface RunnerResult { success: boolean code?: number - signal?: Deno.Signal | null + signal?: unknown } export abstract class BaseRunner { protected readonly processConfig: ProcessConfiguration @@ -25,5 +25,5 @@ export abstract class BaseRunner { } abstract run(runningCallback: RunnerCallback): Promise - abstract kill(signal?: Deno.Signal): void + abstract kill(signal?: unknown): void } From f3c3ff387be6c6d661e2eaa35916a8c0dee63292 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Tue, 2 Jul 2024 22:05:18 +0200 Subject: [PATCH 19/21] Improve error resiliance in logger --- deno.json | 12 ++++++------ lib/core/logger.ts | 15 +++++++++------ lib/core/runner.ts | 1 + test/core/logger.test.ts | 12 ++++++++++++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/deno.json b/deno.json index cf32a22..533c458 100644 --- a/deno.json +++ b/deno.json @@ -25,8 +25,8 @@ }, "tasks": { - "update-deps": "deno run --allow-read=. --allow-net=jsr.io,registry.npmjs.org jsr:@check/deps", - "check": "deno fmt --check && deno lint && deno check pup.ts && deno test --allow-read --allow-write --allow-env --allow-net --allow-sys --allow-run --coverage=cov_profile && echo \"Generating coverage\" && deno coverage cov_profile --exclude=pup/test --lcov --output=cov_profile.lcov", + "check-deps": "deno run -r --allow-read=. --allow-net=jsr.io,registry.npmjs.org jsr:@check/deps", + "check": "deno fmt --check && deno lint && deno check pup.ts && deno test --trace-leaks --allow-read --allow-write --allow-env --allow-net --allow-sys --allow-run --coverage=cov_profile && echo \"Generating coverage\" && deno coverage cov_profile --exclude=pup/test --lcov --output=cov_profile.lcov", "check-coverage": "deno task check && genhtml cov_profile.lcov --output-directory cov_profile/html && lcov --list cov_profile.lcov && deno run --allow-net --allow-read https://deno.land/std/http/file_server.ts cov_profile/html", "build-schema": "deno run --allow-write --allow-read --allow-env=XDG_DATA_HOME,HOME tools/build-schema.ts && deno fmt", "build-versions": "deno run --allow-read --allow-write --allow-env tools/release.ts && deno fmt", @@ -40,20 +40,20 @@ "@cross/env": "jsr:@cross/env@~1.0.2", "@cross/fs": "jsr:@cross/fs@~0.1.11", "@cross/jwt": "jsr:@cross/jwt@~0.4.7", - "@cross/kv": "jsr:@cross/kv@~0.15.11", + "@cross/kv": "jsr:@cross/kv@^0.16.3", "@cross/runtime": "jsr:@cross/runtime@~1.0.0", "@cross/service": "jsr:@cross/service@~1.0.3", "@cross/test": "jsr:@cross/test@~0.0.9", "@cross/utils": "jsr:@cross/utils@~0.13.0", "@hexagon/croner": "jsr:@hexagon/croner@^8.0.2", - "@oak/oak": "jsr:@oak/oak@~16.0.0", + "@oak/oak": "jsr:@oak/oak@^16.1.0", "@pup/api-client": "jsr:@pup/api-client@~2.0.0", "@pup/api-definitions": "jsr:@pup/api-definitions@~2.0.0", "@pup/common": "jsr:@pup/common@~1.0.3", "@pup/plugin": "jsr:@pup/plugin@~1.0.1", - "@std/assert": "jsr:@std/assert@^0.225.2", + "@std/assert": "jsr:@std/assert@^0.226.0", "@std/async": "jsr:@std/async@~0.224.0", - "@std/encoding": "jsr:@std/encoding@~0.224.0", + "@std/encoding": "jsr:@std/encoding@^1.0.0", "@std/io": "jsr:@std/io@~0.224.0", "@std/path": "jsr:@std/path@^0.225.1", "@std/semver": "jsr:@std/semver@~0.224.0", diff --git a/lib/core/logger.ts b/lib/core/logger.ts index f304224..a097abe 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -42,6 +42,9 @@ class Logger { public async init(): Promise { await this.kv.open(this.storeName!) + + // Forcefully unlock ledger in case of a stale lock, this can be done as there is other means of preventing multiple running instances + await this.kv.forceUnlockLedger() } // Used for attaching the logger hook @@ -122,9 +125,9 @@ class Logger { timeStamp: timeStamp, } // Append a random uuid to the key, in case two logs should arrive at the same time - await this.kv.set(["logs_by_time", timeStamp, initiator, severity, crypto.randomUUID()], logObj) - } catch (e) { - console.error("Error while writing to log store", e) + await this.kv.defer(this.kv.set(["logs_by_time", timeStamp, initiator, severity, crypto.randomUUID()], logObj)) + } catch (_e) { + // console.error("Error while writing to log store", e) } } @@ -256,11 +259,11 @@ class Logger { } } /** - * Gracefully shut down the logger + * Gracefully shut down the logger, allowing an optional timeout */ - public async cleanup() { + public async cleanup(timeoutMs = 5000) { try { - await this.kv?.close() + await this.kv?.close(timeoutMs) } catch (e) { console.error("Error while closing kv store: " + e.message) } diff --git a/lib/core/runner.ts b/lib/core/runner.ts index b7f9b1f..9fdaac1 100644 --- a/lib/core/runner.ts +++ b/lib/core/runner.ts @@ -68,6 +68,7 @@ class Runner extends BaseRunner { */ public kill(signal: Deno.Signal = "SIGTERM") { try { + // @ts-ignore Wierd complaint about "SIGPOLL" this.process?.kill(signal) } catch (_e) { // Ignore diff --git a/test/core/logger.test.ts b/test/core/logger.test.ts index a949dad..389128b 100644 --- a/test/core/logger.test.ts +++ b/test/core/logger.test.ts @@ -39,6 +39,8 @@ test("Logger - Attachment of External Logger", async () => { assertEquals(externalLoggerCalled, true) assertEquals(externalLoggerText, expectedExteralLoggerText) + + await logger.cleanup() }) test("Logger - Logging with Different Methods", async () => { @@ -50,6 +52,8 @@ test("Logger - Logging with Different Methods", async () => { await logger.error("test", "Testing error method") assertEquals(true, true) // This is just to assert that the test passed if no errors are thrown + + await logger.cleanup() }) test("Logger - File Writing with writeFile Method", async () => { @@ -82,6 +86,8 @@ test("Logger - getLogContents: Fetch all logs", async () => { const logs = await logger.getLogContents() assertEquals(logs, expectedLogs) + + await logger.cleanup() }) test("Logger - getLogContents: Fetch logs by process ID", async () => { @@ -102,6 +108,8 @@ test("Logger - getLogContents: Fetch logs by process ID", async () => { const logs = await logger.getLogContents(processId) assertEquals(logs, expectedLogs) + + await logger.cleanup() }) test("Logger - getLogContents: Fetch logs by time range", async () => { @@ -123,6 +131,8 @@ test("Logger - getLogContents: Fetch logs by time range", async () => { const logs = await logger.getLogContents(undefined, startTimeStamp, endTimeStamp) assertEquals(logs, expectedLogs) + + await logger.cleanup() }) test("Logger - getLogContents: Fetch logs by process ID and time range", async () => { @@ -145,4 +155,6 @@ test("Logger - getLogContents: Fetch logs by process ID and time range", async ( const logs = await logger.getLogContents(processId, startTimeStamp, endTimeStamp) assertEquals(logs, expectedLogs) + + await logger.cleanup() }) From a67cd1a4638ec2bc525dd0e660cf52b09e1ec685 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Wed, 3 Jul 2024 00:41:03 +0200 Subject: [PATCH 20/21] Small changes for graceful shutdown --- lib/core/logger.ts | 8 ++++---- lib/core/pup.ts | 22 +++++++++++----------- test/core/logger.test.ts | 6 +++++- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/core/logger.ts b/lib/core/logger.ts index a097abe..37cb28c 100644 --- a/lib/core/logger.ts +++ b/lib/core/logger.ts @@ -126,8 +126,8 @@ class Logger { } // Append a random uuid to the key, in case two logs should arrive at the same time await this.kv.defer(this.kv.set(["logs_by_time", timeStamp, initiator, severity, crypto.randomUUID()], logObj)) - } catch (_e) { - // console.error("Error while writing to log store", e) + } catch (e) { + console.error("Error while writing to log store", e) } } @@ -248,10 +248,10 @@ class Logger { const logsByTimeSelector: KVQuery = ["logs_by_time", { to: startTime }] let rowsDeleted = 0 for await (const entry of this.kv.iterate(logsByTimeSelector)) { - await this.kv.delete(entry.key) + await this.kv.defer(this.kv.delete(entry.key)) rowsDeleted++ } - await this.kv.vacuum() + await this.kv.defer(this.kv.vacuum()) return rowsDeleted } catch (error) { this.log("error", `Failed to purge logs from store '${this.storeName}': ${error.message}`) diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 0545918..4719106 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -507,12 +507,12 @@ class Pup { // Block and stop all processes for (const process of this.processes) { process.block("terminating") - stoppingProcesses.push( - process.stop("terminating").then((result) => { - process.cleanup() - return result - }), - ) + const terminationPromise = process.stop("terminating") + stoppingProcesses.push(terminationPromise) + terminationPromise.then((result) => { + process.cleanup() + return result + }) } // Terminate api @@ -520,15 +520,15 @@ class Pup { await Promise.allSettled(stoppingProcesses) - // Cleanup - await this.cleanup() - // Allow some extra time to pass to allow untracked async tasks - // (such as logs about closing down) to finish + // (such as logs after killing a process) to finish // - But only if at least 500ms were used as grace period if (forceQuitMs >= 500) await delay(500) - // Deno should exit gracefully now + // Cleanup + await this.cleanup() + + // Pup should exit gracefully now } private registerGlobalErrorHandler() { diff --git a/test/core/logger.test.ts b/test/core/logger.test.ts index 389128b..369b09f 100644 --- a/test/core/logger.test.ts +++ b/test/core/logger.test.ts @@ -4,7 +4,7 @@ import type { ProcessConfiguration } from "../../mod.ts" import { test } from "@cross/test" import { readFile, tempfile, unlink } from "@cross/fs" -test("Logger - Creation with Global Configuration", () => { +test("Logger - Creation with Global Configuration", async () => { const globalConfig = { console: false, colors: true, @@ -16,6 +16,8 @@ test("Logger - Creation with Global Configuration", () => { const logger = new Logger(globalConfig) assertEquals(logger instanceof Logger, true) + + await logger.cleanup() }) test("Logger - Attachment of External Logger", async () => { @@ -67,6 +69,8 @@ test("Logger - File Writing with writeFile Method", async () => { assertEquals(fileContent, `${testText}\n`) await unlink(testFileName) + + await logger.cleanup() }) test("Logger - getLogContents: Fetch all logs", async () => { From 0a9fe25656e98b3aac15c9f2401098794e420d91 Mon Sep 17 00:00:00 2001 From: Hexagon Date: Wed, 3 Jul 2024 00:50:19 +0200 Subject: [PATCH 21/21] Cleanup --- .npmrc | 1 - README.md | 6 ------ docs/src/examples/telemetry/.npmrc | 1 - docs/src/examples/telemetry/package.json | 6 ------ lib/cli/main.ts | 2 +- lib/core/configuration.ts | 2 +- lib/core/pup.ts | 6 +++--- 7 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 .npmrc delete mode 100644 docs/src/examples/telemetry/.npmrc delete mode 100644 docs/src/examples/telemetry/package.json diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 691d217..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@jsr:registry=https://npm.jsr.io \ No newline at end of file diff --git a/README.md b/README.md index a12a040..9cc1e3c 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,6 @@ To install Pup, make sure you run the latest version of your runtime environment deno run -Ar jsr:@pup/pup@1.0.0-rc.40 setup --channel prerelease ``` -**Bun**: - -```bash -bun run @hexagon/pup setup --channel prerelease -``` - This command downloads the latest version of Pup and installs it on your system. The `--channel prerelease` option is included as there is no stable version of Pup yet. Read more abour release channels [here](https://hexagon.github.io/pup/installation.html#release-channels). diff --git a/docs/src/examples/telemetry/.npmrc b/docs/src/examples/telemetry/.npmrc deleted file mode 100644 index 41583e3..0000000 --- a/docs/src/examples/telemetry/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@jsr:registry=https://npm.jsr.io diff --git a/docs/src/examples/telemetry/package.json b/docs/src/examples/telemetry/package.json deleted file mode 100644 index b49fd43..0000000 --- a/docs/src/examples/telemetry/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": "module", - "dependencies": { - "@pup/telemetry": "npm:@jsr/pup__telemetry@^1.0.5" - } -} diff --git a/lib/cli/main.ts b/lib/cli/main.ts index 16cd1e4..5ad2024 100644 --- a/lib/cli/main.ts +++ b/lib/cli/main.ts @@ -429,7 +429,7 @@ async function main() { * Base argument: logs */ if (baseArgument === "logs") { - const logStore = `${await toPersistentPath(configFile as string)}/.main.log.ckvdb` + const logStore = `${await toPersistentPath(configFile as string)}/.main.db` const logger = new Logger(configuration!.logger || {}, logStore) await logger.init() const startTimestamp = checkedArgs.get("start") ? new Date(Date.parse(checkedArgs.get("start")!)).getTime() : undefined diff --git a/lib/core/configuration.ts b/lib/core/configuration.ts index 31b70c0..a74f72f 100644 --- a/lib/core/configuration.ts +++ b/lib/core/configuration.ts @@ -10,7 +10,7 @@ import { PluginConfiguration } from "@pup/plugin" // Logger constants export const DEFAULT_INTERNAL_LOG_HOURS = 48 -export const KV_LIMIT_STRING_LENGTH_BYTES = 12_000 +export const KV_LIMIT_STRING_LENGTH_BYTES = 65_536 // Core constants export const MAINTENANCE_INTERVAL_MS = 900_000 diff --git a/lib/core/pup.ts b/lib/core/pup.ts index 4719106..95fd568 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -82,17 +82,17 @@ class Pup { this.persistentStoragePath = persistentStoragePath - statusFile = `${this.persistentStoragePath}/.main.status.ckvdb` // Cross/KV store + statusFile = `${this.persistentStoragePath}/.main.status.db` // Cross/KV store secretFile = `${this.persistentStoragePath}/.main.secret` // Plain text file containing the JWT secret for the rest api portFile = `${this.temporaryStoragePath}/.main.port` // Plain text file containing the port number for the API - logStore = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store + logStore = `${this.persistentStoragePath}/.main.db` // Cross/KV store } // Throw on invalid configuration this.configuration = validateConfiguration(unvalidatedConfiguration) // Initialise core logger - this.logger = new Logger(this.configuration.logger ?? {}, logStore || "./main.log") + this.logger = new Logger(this.configuration.logger ?? {}, logStore || "./main.db") // Global error handler this.registerGlobalErrorHandler()