diff --git a/package-lock.json b/package-lock.json index 2edc9847c..def91bc18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2057,6 +2057,25 @@ "node": ">=0.1.90" } }, + "node_modules/@confluentinc/kafka-javascript": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@confluentinc/kafka-javascript/-/kafka-javascript-1.0.0.tgz", + "integrity": "sha512-rMcoeTnQ3YtyFaJ5CsHZvwRC7SCleIk6fZ5sNh5cDbZa2ocuukyO5J5pIhyrUW1FwHm/sauLuXPgwS8xrfiBDQ==", + "hasInstallScript": true, + "workspaces": [ + ".", + "schemaregistry", + "schemaregistry-examples" + ], + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.11", + "bindings": "^1.3.1", + "nan": "^2.22.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "dev": true, @@ -2110,6 +2129,10 @@ "resolved": "packages/dbos-compiler", "link": true }, + "node_modules/@dbos-inc/dbos-confluent-kafka": { + "resolved": "packages/dbos-confluent-kafka", + "link": true + }, "node_modules/@dbos-inc/dbos-datetime": { "resolved": "packages/communicator-datetime", "link": true @@ -3083,6 +3106,47 @@ "node": ">=8" } }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/@nestjs/common": { "version": "10.4.7", "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.4.7.tgz", @@ -5025,6 +5089,11 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", @@ -5072,6 +5141,17 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -5181,6 +5261,24 @@ "node": ">= 6.0.0" } }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==" + }, + "node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/arg": { "version": "4.1.3", "dev": true, @@ -5390,6 +5488,14 @@ "version": "2.4.3", "license": "MIT" }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -5801,6 +5907,14 @@ "node": "*" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, "node_modules/ci-info": { "version": "3.9.0", "dev": true, @@ -6017,6 +6131,14 @@ "simple-swizzle": "^0.2.2" } }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "bin": { + "color-support": "bin.js" + } + }, "node_modules/color/node_modules/color-convert": { "version": "1.9.3", "license": "MIT", @@ -6075,7 +6197,6 @@ }, "node_modules/concat-map": { "version": "0.0.1", - "dev": true, "license": "MIT" }, "node_modules/confbox": { @@ -6122,6 +6243,11 @@ "typedarray-to-buffer": "^3.1.5" } }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" + }, "node_modules/content-disposition": { "version": "0.5.4", "license": "MIT", @@ -6414,6 +6540,14 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "dev": true, @@ -7331,6 +7465,11 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/filelist": { "version": "1.0.4", "dev": true, @@ -7544,6 +7683,33 @@ "node": ">= 0.6" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs-minipass/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "license": "ISC" @@ -7569,6 +7735,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "dev": true, @@ -7633,7 +7819,6 @@ }, "node_modules/glob": { "version": "7.2.3", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -7663,7 +7848,6 @@ }, "node_modules/glob/node_modules/brace-expansion": { "version": "1.1.11", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -7672,7 +7856,6 @@ }, "node_modules/glob/node_modules/minimatch": { "version": "3.1.2", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -7816,6 +7999,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" + }, "node_modules/hasown": { "version": "2.0.2", "license": "MIT", @@ -7915,6 +8103,18 @@ "node": ">=10.19.0" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/human-signals": { "version": "2.1.0", "dev": true, @@ -9643,6 +9843,34 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/mkdirp": { "version": "2.1.6", "dev": true, @@ -9689,6 +9917,11 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nan": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==" + }, "node_modules/nanoid": { "version": "3.3.8", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", @@ -9720,6 +9953,25 @@ "node": ">= 0.6" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-int64": { "version": "0.4.0", "dev": true, @@ -9730,6 +9982,20 @@ "dev": true, "license": "MIT" }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "license": "MIT", @@ -9758,9 +10024,20 @@ "node": ">=8" } }, + "node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10059,7 +10336,6 @@ }, "node_modules/path-is-absolute": { "version": "1.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10885,6 +11161,21 @@ "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", "dev": true }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/rollup": { "version": "4.28.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.1.tgz", @@ -11135,6 +11426,11 @@ "node": ">= 0.8" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" + }, "node_modules/set-cookie-parser": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.1.tgz", @@ -11498,6 +11794,46 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/tarn": { "version": "3.0.2", "license": "MIT", @@ -11652,6 +11988,11 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/triple-beam": { "version": "1.4.1", "license": "MIT", @@ -12496,6 +12837,20 @@ "makeerror": "1.0.12" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "dev": true, @@ -12525,6 +12880,14 @@ "node": ">=8" } }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "node_modules/widest-line": { "version": "4.0.1", "license": "MIT", @@ -13056,6 +13419,25 @@ "node": ">=0.10.0" } }, + "packages/dbos-confluent-kafka": { + "name": "@dbos-inc/dbos-confluent-kafka", + "version": "0.0.0-placeholder", + "license": "MIT", + "dependencies": { + "@confluentinc/kafka-javascript": "^1.0.0" + }, + "devDependencies": { + "@types/jest": "^29.5.12", + "@types/supertest": "^6.0.2", + "jest": "^29.7.0", + "supertest": "^7.0.0", + "ts-jest": "^29.1.4", + "typescript": "^5.3.3" + }, + "peerDependencies": { + "@dbos-inc/dbos-sdk": "*" + } + }, "packages/dbos-kafkajs": { "name": "@dbos-inc/dbos-kafkajs", "version": "0.0.0-placeholder", diff --git a/packages/dbos-confluent-kafka/README.md b/packages/dbos-confluent-kafka/README.md new file mode 100644 index 000000000..7d7a48821 --- /dev/null +++ b/packages/dbos-confluent-kafka/README.md @@ -0,0 +1,59 @@ +# DBOS Kafka Client Library (Confluent Version) + +Publish/subscribe message queues are a common building block for distributed systems. Message queues allow processing to occur at a different place or time, perhaps in multiple client programming environments. Due to its performance, flexibility, and simple, scalable design, [Kafka](https://www.confluent.io/cloud-kafka) is a popular choice for publish/subscribe. + +This package includes a [DBOS](https://docs.dbos.dev/) [step](https://docs.dbos.dev/typescript/tutorials/step-tutorial) for sending Kafka messages, as well as an event receiver for exactly-once processing of incoming messages (even using standard queues). + +This package is based on [KafkaJS](https://kafka.js.org/). We are working on other client libraries for Kafka, please reach out to [us](https://www.dbos.dev/) if you are interested in a different client library. + +## Configuring a DBOS Application with Kafka +Ensure that the DBOS Kafka (Confluent version) package is installed into the application: +``` +npm install --save @dbos-inc/dbos-confluent-kafka +``` + +## Sending Messages + +### Imports +First, ensure that the package classes are imported: +```typescript +import { + KafkaProducer, + KafkaConfig, + logLevel, +} from "@dbos-inc/dbos-confluent-kafka"; +``` + +### Selecting A Configuration +`KafkaProducer` is a configured class. This means that the configuration (or config file key name) must be provided when a class instance is created, for example: +```typescript +const kafkaConfig: KafkaConfig = { + clientId: 'dbos-kafka-test', + brokers: [`${process.env['KAFKA_BROKER'] ?? 'localhost:9092'}`], + requestTimeout: 100, // FOR TESTING + retry: { // FOR TESTING + retries: 5 + }, + logLevel: logLevel.INFO, // FOR TESTING +} + +kafkaCfg = DBOS.configureInstance(KafkaProducer, 'defKafka', kafkaConfig, kafkaTopic); +``` + +### Sending +Within a [DBOS Transact Workflow](https://docs.dbos.dev/typescript/tutorials/workflow-tutorial), call the `KafkaProducer` function from a workflow: +```typescript +const sendRes = await kafkaCfg.send({value: ourMessage}); +``` + +## Receiving Messages +A tutorial for receiving and processing Kafka messages can be found [here](https://docs.dbos.dev/tutorials/requestsandevents/kafka-integration). This library provides an alternate implementation of the Kafka consumer that can be updated independently of the DBOS Transact core packages. + +## Simple Testing +The `confluent-kafkajs.test.ts` file included in the source repository demonstrates setting up topics, and sending and processing Kafka messages. Before running, set the following environment variables: +- `KAFKA_BROKER`: Broker URL + +## Next Steps +- To start a DBOS app from a template, visit our [quickstart](https://docs.dbos.dev/quickstart). +- For DBOS Transact programming tutorials, check out our [programming guide](https://docs.dbos.dev/typescript/programming-guide). +- To learn more about DBOS, take a look at [our documentation](https://docs.dbos.dev/) or our [source code](https://github.com/dbos-inc/dbos-transact). diff --git a/packages/dbos-confluent-kafka/confluent-kafka.test.ts b/packages/dbos-confluent-kafka/confluent-kafka.test.ts new file mode 100644 index 000000000..c44e6b3fd --- /dev/null +++ b/packages/dbos-confluent-kafka/confluent-kafka.test.ts @@ -0,0 +1,230 @@ +import { + DBOS, + parseConfigFile, +} from "@dbos-inc/dbos-sdk"; + +import { + KafkaProducer, + CKafkaConsume, + CKafka, + KafkaConfig, + Message, + logLevel, +} +from "./index"; + +import { + KafkaJS +} +from "@confluentinc/kafka-javascript" + +// These tests require local Kafka to run. +// Without it, they're automatically skipped. +// Here's a docker-compose script you can use to set up local Kafka: + +const _ = ` +version: "3.7" +services: + broker: + image: bitnami/kafka:latest + hostname: broker + container_name: broker + ports: + - '9092:9092' + - '29093:29093' + - '19092:19092' + environment: + KAFKA_CFG_NODE_ID: 1 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://broker:19092' + KAFKA_CFG_PROCESS_ROLES: 'broker,controller' + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_CFG_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' +` + +const kafkaConfig: KafkaConfig = { + clientId: 'dbos-kafka-test', + brokers: [`${process.env['KAFKA_BROKER'] ?? 'localhost:9092'}`], + //requestTimeout: 100, // FOR TESTING + retry: { // FOR TESTING + retries: 5 + }, + logLevel: logLevel.INFO, +} + +const ensureTopicExists = async (topicName: string) => { + const admin = new KafkaJS.Kafka({"bootstrap.servers": `${process.env['KAFKA_BROKER'] ?? 'localhost:9092'}`}).admin(); + + try { + // Connect to the admin client + await admin.connect(); + + // Check existing topics + const topics = await admin.listTopics(); + if (topics.includes(topicName)) { + console.log(`Topic "${topicName}" already exists.`); + return; + } + + // Create the topic + console.log(`Creating topic "${topicName}"...`); + await admin.createTopics({ + topics: [ + { + topic: topicName, + numPartitions: 1, + replicationFactor: 1, + }, + ], + }); + + console.log(`Topic "${topicName}" created successfully.`); + } catch (e) { + const error = e as Error; + console.error(`Failed to ensure topic exists: ${error.message}`); + } finally { + // Disconnect from the admin client + await admin.disconnect(); + } +}; + +const wf1Topic = 'dbos-test-wf-topic'; +const wf2Topic = 'dbos-test-wf-topic2'; +const wfMessage = 'dbos-wf' +let wfCounter = 0; + +const patternTopic = new RegExp(/^dbos-test-.*/); +let patternTopicCounter = 0; + +const arrayTopics = [wf1Topic, wf2Topic]; +let arrayTopicsCounter = 0; + +describe("kafka-tests", () => { + let kafkaIsAvailable = true; + let wfKafkaCfg: KafkaProducer | undefined = undefined; + let wf2KafkaCfg: KafkaProducer | undefined = undefined; + + beforeAll(async () => { + // Check if Kafka is available, skip the test if it's not + if (process.env['KAFKA_BROKER']) { + kafkaIsAvailable = true; + } else { + kafkaIsAvailable = false; + return; + } + + await ensureTopicExists(wf1Topic); + await ensureTopicExists(wf2Topic); + + const [cfg, rtCfg] = parseConfigFile({configfile: 'confluentkafka-test-dbos-config.yaml'}); + DBOS.setConfig(cfg, rtCfg); + + // This would normally be a global or static or something + wfKafkaCfg = DBOS.configureInstance(KafkaProducer, 'wfKafka', kafkaConfig, wf1Topic); + wf2KafkaCfg = DBOS.configureInstance(KafkaProducer, 'wf2Kafka', kafkaConfig, wf2Topic); + await DBOS.launch(); + }, 30000); + + afterAll(async() => { + await wfKafkaCfg?.disconnect(); + await wf2KafkaCfg?.disconnect(); + await DBOS.shutdown(); + }, 30000); + + test("txn-kafka", async () => { + if (!kafkaIsAvailable) { + console.log("Kafka unavailable, skipping Kafka tests") + return + } + else { + console.log("Kafka tests running...") + } + // Create a producer to send a message + await wf2KafkaCfg!.sendMessage({ + value: wfMessage, + }); + await wfKafkaCfg!.sendMessage({ + value: wfMessage, + }); + console.log("Messages sent"); + + // Check that both messages are consumed + console.log("Waiting for regular topic"); + await DBOSTestClass.wfPromise; + expect(wfCounter).toBe(1); + + console.log("Waiting for pattern topic"); + await DBOSTestClass.patternTopicPromise; + expect(patternTopicCounter).toBe(2); + + console.log("Waiting for array topic"); + await DBOSTestClass.arrayTopicsPromise; + expect(arrayTopicsCounter).toBe(2); + + console.log("Done"); + }, 30000); +}); + +@CKafka(kafkaConfig) +class DBOSTestClass { + static wfResolve: () => void; + static wfPromise = new Promise((r) => { + DBOSTestClass.wfResolve = r; + }); + + static patternTopicResolve: () => void; + static patternTopicPromise = new Promise((r) => { + DBOSTestClass.patternTopicResolve = r; + }); + + static arrayTopicsResolve: () => void; + static arrayTopicsPromise = new Promise((r) => { + DBOSTestClass.arrayTopicsResolve = r; + }); + + @CKafkaConsume(wf1Topic) + @DBOS.workflow() + static async testWorkflow(topic: string, _partition: number, message: Message) { + console.log(`got something 1 ${topic} ${message.value?.toString()}`); + if (topic === wf1Topic && message.value?.toString() === wfMessage) { + wfCounter = wfCounter + 1; + DBOSTestClass.wfResolve(); + } + else { + console.warn(`Got strange message on wf1Topic: ${JSON.stringify(message)}`); + } + await DBOSTestClass.wfPromise; + } + + @DBOS.workflow() + @CKafkaConsume(patternTopic) + static async testConsumeTopicsByPattern(topic: string, _partition: number, message: Message) { + console.log(`got something 2 ${topic}`); + const isWfMessage = topic === wf1Topic && message.value?.toString() === wfMessage; + const isWf2Message = topic === wf2Topic && message.value?.toString() === wfMessage; + if ( isWfMessage || isWf2Message ) { + patternTopicCounter = patternTopicCounter + 1; + if (patternTopicCounter === 2) { + DBOSTestClass.patternTopicResolve(); + } + } + await DBOSTestClass.patternTopicPromise; + } + + @CKafkaConsume(arrayTopics) + @DBOS.workflow() + static async testConsumeTopicsArray(topic: string, _partition: number, message: Message) { + console.log(`got something 3 ${topic}`); + const isWfMessage = topic === wf1Topic && message.value?.toString() === wfMessage; + const isWf2Message = topic === wf2Topic && message.value?.toString() === wfMessage; + if ( isWfMessage || isWf2Message) { + arrayTopicsCounter = arrayTopicsCounter + 1; + if (arrayTopicsCounter === 2) { + DBOSTestClass.arrayTopicsResolve(); + } + } + await DBOSTestClass.arrayTopicsPromise; + } +} diff --git a/packages/dbos-confluent-kafka/confluentkafka-test-dbos-config.yaml b/packages/dbos-confluent-kafka/confluentkafka-test-dbos-config.yaml new file mode 100644 index 000000000..3e5dc1177 --- /dev/null +++ b/packages/dbos-confluent-kafka/confluentkafka-test-dbos-config.yaml @@ -0,0 +1,17 @@ +# To enable auto-completion and validation for this file in VSCode, install the RedHat YAML extension +# https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml + +# yaml-language-server: $schema=https://raw.githubusercontent.com/dbos-inc/dbos-transact/main/dbos-config.schema.json + +database: + hostname: 'localhost' + port: 5432 + username: 'postgres' + password: ${PGPASSWORD} + app_db_name: 'hello' + connectionTimeoutMillis: 3000 + app_db_client: 'knex' + migrate: + - npx knex migrate:latest + rollback: + - npx knex migrate:rollback diff --git a/packages/dbos-confluent-kafka/index.ts b/packages/dbos-confluent-kafka/index.ts new file mode 100644 index 000000000..761414c7e --- /dev/null +++ b/packages/dbos-confluent-kafka/index.ts @@ -0,0 +1,232 @@ +import { + DBOS, + InitContext, + ConfiguredInstance, + Error as DBOSError, + + DBOSEventReceiver, + DBOSExecutorContext, + WorkflowFunction, + TransactionFunction, + associateClassWithEventReceiver, + associateMethodWithEventReceiver, +} from '@dbos-inc/dbos-sdk'; + +import { + KafkaJS, + LibrdKafkaError as KafkaError +} from "@confluentinc/kafka-javascript"; + +export type KafkaConfig = KafkaJS.KafkaConfig; +export const logLevel = KafkaJS.logLevel; +export type Message = KafkaJS.Message; + +const sleepms = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +type KafkaArgs = [string, number, KafkaJS.KafkaMessage] + +//////////////////////// +/* Kafka Management */ +/////////////////////// + +export class DBOSConfluentKafka implements DBOSEventReceiver { + readonly consumers: KafkaJS.Consumer[] = []; + + executor?: DBOSExecutorContext = undefined; + + constructor() { } + + async initialize(dbosExecI: DBOSExecutorContext) { + this.executor = dbosExecI; + const regops = this.executor.getRegistrationsFor(this); + for (const registeredOperation of regops) { + const ro = registeredOperation.methodConfig as KafkaRegistrationInfo; + if (ro.kafkaTopics) { + const defaults = registeredOperation.classConfig as KafkaDefaults; + const method = registeredOperation.methodReg; + const cname = method.className; + const mname = method.name; + console.log(`Registering ${cname}.${mname}:${JSON.stringify(ro.kafkaTopics)}`) + if (!method.txnConfig && !method.workflowConfig) { + throw new DBOSError.DBOSError(`Error registering method ${cname}.${mname}: A CKafka decorator can only be assigned to a transaction or workflow.`) + } + if (!defaults.kafkaConfig) { + throw new DBOSError.DBOSError(`Error registering method ${cname}.${mname}: Kafka configuration not found. (Does class ${cname} have an @CKafka decorator?)`) + } + const topics: Array = []; + if (Array.isArray(ro.kafkaTopics) ) { + topics.push(...ro.kafkaTopics) + } + else if (ro.kafkaTopics) { + topics.push(ro.kafkaTopics) + } + let clientId = defaults.kafkaConfig.clientId ?? 'dbos-kafka'; + clientId = clientId + `${cname}-${mname}`; + const kafka = new KafkaJS.Kafka({kafkaJS: {...defaults.kafkaConfig, clientId: clientId }}); + const consumerConfig = ro.consumerConfig + ? {...ro.consumerConfig, 'auto.offset.reset': 'earliest'} + : { "group.id": `${this.safeGroupName(topics)}`, 'auto.offset.reset': 'earliest' }; + const consumer = kafka.consumer(consumerConfig); + await consumer.connect(); + // Unclear if we need this: + // A temporary workaround for https://github.com/tulios/kafkajs/pull/1558 until it gets fixed + // If topic autocreation is on and you try to subscribe to a nonexistent topic, KafkaJS should retry until the topic is created. + // However, it has a bug where it won't. Thus, we retry instead. + const maxRetries = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.retries ?? 5 : 5; + let retryTime = defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.maxRetryTime ?? 300 : 300; + const multiplier = /* defaults.kafkaConfig.retry ? defaults.kafkaConfig.retry.multiplier ?? 2 :*/ 2; + for (let i = 0; i < maxRetries; i++) { + try { + console.log(`Subscribe to ${JSON.stringify(topics)}`); + await consumer.subscribe({ topics: topics }); + break; + } catch (error) { + const e = error as KafkaError; + if (e.code === 3 && i + 1 < maxRetries) { // UNKNOWN_TOPIC_OR_PARTITION + await sleepms(retryTime); + retryTime *= multiplier; + continue; + } else { + throw error; + } + } + } + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + // This combination uniquely identifies a message for a given Kafka cluster + const workflowUUID = `kafka-unique-id-${topic}-${partition}-${message.offset}` + const wfParams = { workflowUUID: workflowUUID, configuredInstance: null }; + // All operations annotated with Kafka decorators must take in these three arguments + const args: KafkaArgs = [topic, partition, message]; + // We can only guarantee exactly-once-per-message execution of transactions and workflows. + if (method.txnConfig) { + // Execute the transaction + await this.executor!.transaction(method.registeredFunction as TransactionFunction, wfParams, ...args); + } else if (method.workflowConfig) { + // Safely start the workflow + await this.executor!.workflow(method.registeredFunction as unknown as WorkflowFunction, wfParams, ...args); + } + }, + }) + this.consumers.push(consumer); + } + } + } + + async destroy() { + for (const consumer of this.consumers) { + //await consumer.stop(); + await consumer.disconnect(); + } + } + + safeGroupName(topics: Array) { + const safeGroupIdPart = topics + .map(r => r.toString()) + .map( r => r.replaceAll(/[^a-zA-Z0-9\\-]/g, '')) + .join('-'); + return `dbos-kafka-group-${safeGroupIdPart}`.slice(0, 255); + } + + logRegisteredEndpoints() { + if (!this.executor) return; + const logger = this.executor.logger; + logger.info("Kafka endpoints supported:"); + const regops = this.executor.getRegistrationsFor(this); + regops.forEach((registeredOperation) => { + const ro = registeredOperation.methodConfig as KafkaRegistrationInfo; + if (ro.kafkaTopics) { + const cname = registeredOperation.methodReg.className; + const mname = registeredOperation.methodReg.name; + if (Array.isArray(ro.kafkaTopics)) { + ro.kafkaTopics.forEach( kafkaTopic => { + logger.info(` ${kafkaTopic} -> ${cname}.${mname}`); + }); + } else { + logger.info(` ${ro.kafkaTopics} -> ${cname}.${mname}`); + } + } + }); + } +} + +///////////////////////////// +/* Kafka Method Decorators */ +///////////////////////////// + +let kafkaInst: DBOSConfluentKafka | undefined = undefined; + +export interface KafkaRegistrationInfo { + kafkaTopics?: string | RegExp | Array; + consumerConfig?: KafkaConfig; +} + +export function CKafkaConsume(topics: string | RegExp | Array, consumerConfig?: KafkaConfig) { + function kafkadec( + target: object, + propertyKey: string, + inDescriptor: TypedPropertyDescriptor<(this: This, ...args: KafkaArgs) => Promise> + ) + { + if (!kafkaInst) kafkaInst = new DBOSConfluentKafka(); + const {descriptor, receiverInfo} = associateMethodWithEventReceiver(kafkaInst, target, propertyKey, inDescriptor); + + const kafkaRegistration = receiverInfo as KafkaRegistrationInfo; + kafkaRegistration.kafkaTopics = topics; + kafkaRegistration.consumerConfig = consumerConfig; + + return descriptor; + } + return kafkadec; +} + +///////////////////////////// +/* Kafka Class Decorators */ +///////////////////////////// + +export interface KafkaDefaults { + kafkaConfig?: KafkaConfig; +} + +export function CKafka(kafkaConfig: KafkaConfig) { + function clsdec(ctor: T) { + if (!kafkaInst) kafkaInst = new DBOSConfluentKafka(); + const kafkaInfo = associateClassWithEventReceiver(kafkaInst, ctor) as KafkaDefaults; + kafkaInfo.kafkaConfig = kafkaConfig; + } + return clsdec; +} + +////////////////////////////// +/* Kafka Producer */ +////////////////////////////// +export class KafkaProducer extends ConfiguredInstance +{ + producer: KafkaJS.Producer | undefined = undefined; + topic: string = ""; + + constructor(name: string, readonly cfg: KafkaJS.ProducerConfig, topic: string) { + super(name); + this.topic = topic; + } + + async initialize(_ctx: InitContext): Promise { + const kafka = new KafkaJS.Kafka({}); + this.producer = kafka.producer({kafkaJS: this.cfg}); + await this.producer.connect(); + } + + @DBOS.step() + async sendMessage(msg: KafkaJS.Message) { + return await this.producer?.send({topic: this.topic, messages:[msg]}); + } + + @DBOS.step() + async sendMessages(msg: KafkaJS.Message[]) { + return await this.producer?.send({topic: this.topic, messages:msg}); + } + + async disconnect() { + await this.producer?.disconnect(); + } +} \ No newline at end of file diff --git a/packages/dbos-confluent-kafka/jest.config.js b/packages/dbos-confluent-kafka/jest.config.js new file mode 100644 index 000000000..26529263a --- /dev/null +++ b/packages/dbos-confluent-kafka/jest.config.js @@ -0,0 +1,8 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testRegex: '((\\.|/)(test|spec))\\.ts?$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePaths: ["./"], +}; diff --git a/packages/dbos-confluent-kafka/package.json b/packages/dbos-confluent-kafka/package.json new file mode 100644 index 000000000..73ee96e4c --- /dev/null +++ b/packages/dbos-confluent-kafka/package.json @@ -0,0 +1,33 @@ +{ + "name": "@dbos-inc/dbos-confluent-kafka", + "version": "0.0.0-placeholder", + "description": "Communicator/event receiver library - Kafka via @confluentinc/kafka-javascript", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/dbos-inc/dbos-transact", + "directory": "packages/dbos-confluent-kafka" + }, + "homepage": "https://docs.dbos.dev/", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc --project tsconfig.json", + "test": "echo 'no tests'", + "testkafka": "npm run build && jest --detectOpenHandles" + }, + "devDependencies": { + "@types/jest": "^29.5.12", + "@types/supertest": "^6.0.2", + "jest": "^29.7.0", + "supertest": "^7.0.0", + "ts-jest": "^29.1.4", + "typescript": "^5.3.3" + }, + "peerDependencies": { + "@dbos-inc/dbos-sdk": "*" + }, + "dependencies": { + "@confluentinc/kafka-javascript": "^1.0.0" + } +} diff --git a/packages/dbos-confluent-kafka/tsconfig.json b/packages/dbos-confluent-kafka/tsconfig.json new file mode 100644 index 000000000..ec45f0403 --- /dev/null +++ b/packages/dbos-confluent-kafka/tsconfig.json @@ -0,0 +1,24 @@ +/* Visit https://aka.ms/tsconfig to read more about this file */ +{ + "compilerOptions": { + "target": "esnext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + "module": "Node16", /* Specify what module code is generated. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declarationMap": true, /* Create sourcemaps for d.ts files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "newLine": "lf", /* Set the newline character for emitting files. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + "strict": true, /* Enable all strict type-checking options. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": [ /* Specifies an array of filenames or patterns to include in the program. */ + "." + ], + "exclude": [ + "dist", + ], +} diff --git a/packages/dbos-kafkajs/README.md b/packages/dbos-kafkajs/README.md index d0c746e1f..472007641 100644 --- a/packages/dbos-kafkajs/README.md +++ b/packages/dbos-kafkajs/README.md @@ -7,7 +7,7 @@ This package includes a [DBOS](https://docs.dbos.dev/) [step](https://docs.dbos. This package is based on [KafkaJS](https://kafka.js.org/). We are working on other client libraries for Kafka, please reach out to [us](https://www.dbos.dev/) if you are interested in a different client library. ## Configuring a DBOS Application with Kafka -Ensure that the DBOS SQS package is installed into the application: +Ensure that the DBOS Kafka (KafkaJS version) package is installed into the application: ``` npm install --save @dbos-inc/dbos-kafkajs ``` diff --git a/src/dbos.ts b/src/dbos.ts index 48f29e5a0..0c11ccd75 100644 --- a/src/dbos.ts +++ b/src/dbos.ts @@ -25,7 +25,7 @@ import { DBOSError, DBOSExecutorNotInitializedError, DBOSInvalidWorkflowTransiti import { parseConfigFile } from "./dbos-runtime/config"; import { DBOSRuntimeConfig } from "./dbos-runtime/runtime"; import { DBOSScheduler, ScheduledArgs, SchedulerConfig, SchedulerRegistrationBase } from "./scheduler/scheduler"; -import { configureInstance, getOrCreateClassRegistration, getRegisteredOperations, MethodRegistration, registerAndWrapContextFreeFunction, registerFunctionWrapper } from "./decorators"; +import { configureInstance, getOrCreateClassRegistration, getRegisteredOperations, MethodRegistration, registerAndWrapDBOSFunction, registerFunctionWrapper } from "./decorators"; import { sleepms } from "./utils"; import { DBOSHttpServer } from "./httpServer/server"; import { koaTracingMiddleware, expressTracingMiddleware } from "./httpServer/middleware"; @@ -114,7 +114,7 @@ function httpApiDec(verb: APITypes, url: string) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise> ) { - const { descriptor, registration } = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); const handlerRegistration = registration as unknown as HandlerRegistrationBase; handlerRegistration.apiURL = url; handlerRegistration.apiType = verb; @@ -697,7 +697,7 @@ export class DBOS { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: ScheduledArgs) => Promise> ) { - const { descriptor, registration } = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); const schedRegistration = registration as unknown as SchedulerRegistrationBase; schedRegistration.schedulerConfig = schedulerConfig; @@ -719,7 +719,7 @@ export class DBOS { inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise> ) { - const { descriptor, registration } = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); registration.workflowConfig = config; const invokeWrapper = async function (this: This, ...rawArgs: Args): Promise { @@ -818,7 +818,7 @@ export class DBOS { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); registration.txnConfig = config; const invokeWrapper = async function (this: This, ...rawArgs: Args): Promise { @@ -901,7 +901,7 @@ export class DBOS { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); registration.commConfig = config; const invokeWrapper = async function (this: This, ...rawArgs: Args): Promise { @@ -1013,7 +1013,7 @@ export class DBOS { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { - const {descriptor, registration} = registerAndWrapContextFreeFunction(target, propertyKey, inDescriptor); + const {descriptor, registration} = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); registration.requiredRole = anyOf; return descriptor; @@ -1030,13 +1030,13 @@ export class DBOS { } // Function registration - static registerAndWrapContextFreeFunction( + static registerAndWrapDBOSFunction( target: object, propertyKey: string, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>, ) { - return registerAndWrapContextFreeFunction(target, propertyKey, descriptor); + return registerAndWrapDBOSFunction(target, propertyKey, descriptor); } static async executeWorkflowById(workflowId: string, startNewWorkflow: boolean = false): Promise> diff --git a/src/decorators.ts b/src/decorators.ts index 73c88fe10..7fcc6084b 100644 --- a/src/decorators.ts +++ b/src/decorators.ts @@ -176,12 +176,15 @@ implements MethodRegistrationBase requiredRole: string[] | undefined = undefined; args: MethodParameter[] = []; + passContext: boolean = false; - constructor(origFunc: (this: This, ...args: Args) => Promise, isInstance: boolean, readonly passContext: boolean) + constructor(origFunc: (this: This, ...args: Args) => Promise, isInstance: boolean, passContext: boolean) { this.origFunction = origFunc; this.isInstance = isInstance; + this.passContext = passContext; } + needInitialized: boolean = true; isInstance: boolean; origFunction: (this: This, ...args: Args) => Promise; @@ -341,6 +344,13 @@ function getOrCreateMethodRegistration( classReg.registeredOperations.set(fname, new MethodRegistration(descriptor.value!, isInstance, passContext)); } const methReg: MethodRegistration = classReg.registeredOperations.get(fname)! as MethodRegistration; + + // Note: We cannot tell if the method takes a context or not. + // Our @Workflow, @Transaction, and @Step decorators are the only ones that would know to set passContext. + // So, if passContext is indicated, add it to the registration. + if (passContext && !methReg.passContext) { + methReg.passContext = true; + } if (methReg.needInitialized) { methReg.needInitialized = false; @@ -440,7 +450,7 @@ function getOrCreateMethodRegistration( return methReg; } -export function registerAndWrapFunction(target: object, propertyKey: string, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { +export function registerAndWrapFunctionTakingContext(target: object, propertyKey: string, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { if (!descriptor.value) { throw Error("Use of decorator when original method is undefined"); } @@ -450,7 +460,7 @@ export function registerAndWrapFunction(ta return { descriptor, registration }; } -export function registerAndWrapContextFreeFunction(target: object, propertyKey: string, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { +export function registerAndWrapDBOSFunction(target: object, propertyKey: string, descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { if (!descriptor.value) { throw Error("Use of decorator when original method is undefined"); } @@ -500,7 +510,7 @@ export function associateClassWithEventReceiver(rcvr: DBOSEventReceiver, target: object, propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); if (!registration.eventReceiverInfo.has(rcvr)) { registration.eventReceiverInfo.set(rcvr, {}); } @@ -615,7 +625,7 @@ export function RequiredRole(anyOf: string[]) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: Ctx, ...args: Args) => Promise>) { - const {descriptor, registration} = registerAndWrapFunction(target, propertyKey, inDescriptor); + const {descriptor, registration} = registerAndWrapDBOSFunction(target, propertyKey, inDescriptor); registration.requiredRole = anyOf; return descriptor; @@ -629,7 +639,7 @@ export function Workflow(config: WorkflowConfig={}) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: WorkflowContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.workflowConfig = config; return descriptor; } @@ -643,7 +653,7 @@ export function Transaction(config: TransactionConfig={}) { // eslint-disable-next-line @typescript-eslint/no-explicit-any inDescriptor: TypedPropertyDescriptor<(this: This, ctx: TransactionContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.txnConfig = config; return descriptor; } @@ -656,7 +666,7 @@ export function StoredProcedure(config: StoredProcedureConfig={}) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: StoredProcedureContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.procConfig = config; return descriptor; } @@ -669,7 +679,7 @@ export function Step(config: StepConfig={}) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: StepContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.commConfig = config; return descriptor; } @@ -693,7 +703,7 @@ export function DBOSInitializer() { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: InitContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.init = true; return descriptor; } @@ -707,7 +717,7 @@ export function DBOSDeploy() { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: InitContext, ...args: Args) => Promise>) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); registration.init = true; return descriptor; } diff --git a/src/httpServer/handler.ts b/src/httpServer/handler.ts index 8c698f5d6..7a0857d58 100644 --- a/src/httpServer/handler.ts +++ b/src/httpServer/handler.ts @@ -1,4 +1,4 @@ -import { MethodParameter, registerAndWrapFunction, getOrCreateMethodArgsRegistration, MethodRegistrationBase, getRegisteredOperations, ConfiguredInstance } from "../decorators"; +import { MethodParameter, registerAndWrapFunctionTakingContext, getOrCreateMethodArgsRegistration, MethodRegistrationBase, getRegisteredOperations, ConfiguredInstance } from "../decorators"; import { DBOSExecutor, OperationType } from "../dbos-executor"; import { DBOSContext, DBOSContextImpl } from "../context"; import Koa from "koa"; @@ -248,7 +248,7 @@ function generateApiDec(verb: APITypes, url: string) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: Ctx, ...args: Args) => Promise> ) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); const handlerRegistration = registration as unknown as HandlerRegistrationBase; handlerRegistration.apiURL = url; handlerRegistration.apiType = verb; diff --git a/src/scheduler/scheduler.ts b/src/scheduler/scheduler.ts index bf782d5d3..f07366746 100644 --- a/src/scheduler/scheduler.ts +++ b/src/scheduler/scheduler.ts @@ -1,6 +1,6 @@ import { DBOSEventReceiverState, WorkflowContext } from ".."; import { DBOSExecutor } from "../dbos-executor"; -import { MethodRegistrationBase, registerAndWrapFunction } from "../decorators"; +import { MethodRegistrationBase, registerAndWrapFunctionTakingContext } from "../decorators"; import { TimeMatcher } from "./crontab"; import { Workflow } from "../workflow"; @@ -37,7 +37,7 @@ export function Scheduled(schedulerConfig: SchedulerConfig) { propertyKey: string, inDescriptor: TypedPropertyDescriptor<(this: This, ctx: Ctx, ...args: ScheduledArgs) => Promise> ) { - const { descriptor, registration } = registerAndWrapFunction(target, propertyKey, inDescriptor); + const { descriptor, registration } = registerAndWrapFunctionTakingContext(target, propertyKey, inDescriptor); const schedRegistration = registration as unknown as SchedulerRegistrationBase; schedRegistration.schedulerConfig = schedulerConfig;