diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 880fc76..b763242 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -150,6 +150,7 @@ Solid on Rails comes with a built in CLI which supports: * `solid-on-rails db:revert` Reverting migrations * `solid-on-rails queues:deleteAll` Deleting all background job queues * `solid-on-rails queues:delete` Deleting specific background job queues +* `solid-on-rails queues:removeCompleted` Removes all completed jobs from a queue You may prefer to add helpers in your `package.json` to set the configuration options for each of these commands like so: ```json @@ -164,6 +165,7 @@ You may prefer to add helpers in your `package.json` to set the configuration op "db:revert": "npx solid-on-rails db:revert -c ./config/storage-accessor.json -m .", "queues:deleteAll": "npx solid-on-rails queues:deleteAll -c ./config/queue-accessor.json -m .", "queues:delete": "npx solid-on-rails queues:delete -c ./config/queue-accessor.json -m .", + "queues:removeCompleted": "npx solid-on-rails queues:removeCompleted -c ./config/queue-accessor.json -m .", "start": "npx solid-on-rails -c ./config/test.json -m .", ... } diff --git a/package-lock.json b/package-lock.json index 6520147..61ff3fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@comake/solid-on-rails", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@comake/solid-on-rails", - "version": "0.8.0", + "version": "0.9.0", "dependencies": { "arrayify-stream": "^2.0.0", - "bull": "^4.8.5", + "bull": "^4.11.3", "componentsjs": "^5.4.0", "cors": "^2.8.5", "dotenv": "^16.0.1", @@ -36,7 +36,6 @@ "@commitlint/cli": "^17.0.3", "@commitlint/config-conventional": "^17.0.3", "@tsconfig/node14": "^1.0.3", - "@types/bull": "^3.15.9", "@types/cors": "^2.8.12", "@types/ejs": "^3.1.1", "@types/end-of-stream": "^1.4.1", @@ -1447,9 +1446,7 @@ "node_modules/@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "optional": true, - "peer": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -2149,16 +2146,6 @@ "@babel/types": "^7.3.0" } }, - "node_modules/@types/bull": { - "version": "3.15.9", - "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz", - "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==", - "dev": true, - "dependencies": { - "@types/ioredis": "*", - "@types/redis": "^2.8.0" - } - }, "node_modules/@types/cookiejar": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz", @@ -2212,15 +2199,6 @@ "@types/node": "*" } }, - "node_modules/@types/ioredis": { - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -2331,15 +2309,6 @@ "safe-buffer": "~5.1.1" } }, - "node_modules/@types/redis": { - "version": "2.8.32", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", - "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/semver": { "version": "7.3.10", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.10.tgz", @@ -3130,55 +3099,34 @@ } }, "node_modules/bull": { - "version": "4.8.5", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.8.5.tgz", - "integrity": "sha512-2Z630e4f6VsLJnWMAtfEHwIqJYmND4W3dcG48RIbXeWpvb4UnYtpe/zxEdslJu0PKrltB4IkFj5YtBsdeQRn8w==", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.11.3.tgz", + "integrity": "sha512-DhS0XtiAuejkAY08iGOdDK35eex/yGNoezlWqGJTu9FqWFF/oBjUhpsusE9SXiI4culyDbOoFs+l3ar0VXhFqQ==", "dependencies": { "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", "get-port": "^5.1.1", - "ioredis": "^4.28.5", + "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", + "semver": "^7.5.2", "uuid": "^8.3.0" }, "engines": { - "node": ">=10.1" - } - }, - "node_modules/bull/node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" + "node": ">=12" } }, - "node_modules/bull/node_modules/ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "node_modules/bull/node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" + "lru-cache": "^6.0.0" }, - "engines": { - "node": ">=6" + "bin": { + "semver": "bin/semver.js" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" + "engines": { + "node": ">=10" } }, "node_modules/call-bind": { @@ -3815,14 +3763,6 @@ } } }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", - "engines": { - "node": "*" - } - }, "node_modules/decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -3904,8 +3844,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "optional": true, - "peer": true, "engines": { "node": ">=0.10" } @@ -6239,16 +6177,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.2.tgz", - "integrity": "sha512-wryKc1ur8PcCmNwfcGkw5evouzpbDXxxkMkzPK8wl4xQfQf7lHe11Jotell5ikMVAtikXJEu/OJVaoV51BggRQ==", - "optional": true, - "peer": true, + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -7427,11 +7363,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -8085,14 +8016,6 @@ "node": ">= 0.8.0" } }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8123,25 +8046,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", - "engines": { - "node": ">=6" - } - }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -9007,11 +8911,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/redis-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", - "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" - }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -11740,9 +11639,7 @@ "@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "optional": true, - "peer": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -12300,16 +12197,6 @@ "@babel/types": "^7.3.0" } }, - "@types/bull": { - "version": "3.15.9", - "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz", - "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==", - "dev": true, - "requires": { - "@types/ioredis": "*", - "@types/redis": "^2.8.0" - } - }, "@types/cookiejar": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz", @@ -12363,15 +12250,6 @@ "@types/node": "*" } }, - "@types/ioredis": { - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -12482,15 +12360,6 @@ "safe-buffer": "~5.1.1" } }, - "@types/redis": { - "version": "2.8.32", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", - "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/semver": { "version": "7.3.10", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.10.tgz", @@ -13022,42 +12891,25 @@ "dev": true }, "bull": { - "version": "4.8.5", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.8.5.tgz", - "integrity": "sha512-2Z630e4f6VsLJnWMAtfEHwIqJYmND4W3dcG48RIbXeWpvb4UnYtpe/zxEdslJu0PKrltB4IkFj5YtBsdeQRn8w==", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.11.3.tgz", + "integrity": "sha512-DhS0XtiAuejkAY08iGOdDK35eex/yGNoezlWqGJTu9FqWFF/oBjUhpsusE9SXiI4culyDbOoFs+l3ar0VXhFqQ==", "requires": { "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", "get-port": "^5.1.1", - "ioredis": "^4.28.5", + "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", + "semver": "^7.5.2", "uuid": "^8.3.0" }, "dependencies": { - "denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" - }, - "ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "requires": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" + "lru-cache": "^6.0.0" } } } @@ -13535,11 +13387,6 @@ "ms": "2.1.2" } }, - "debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==" - }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -13601,9 +13448,7 @@ "denque": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "optional": true, - "peer": true + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, "depd": { "version": "1.1.2", @@ -15311,16 +15156,14 @@ } }, "ioredis": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.2.tgz", - "integrity": "sha512-wryKc1ur8PcCmNwfcGkw5evouzpbDXxxkMkzPK8wl4xQfQf7lHe11Jotell5ikMVAtikXJEu/OJVaoV51BggRQ==", - "optional": true, - "peer": true, + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -16194,11 +16037,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" }, - "lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -16697,11 +16535,6 @@ "word-wrap": "^1.2.3" } }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -16720,19 +16553,6 @@ "p-limit": "^3.0.2" } }, - "p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" - }, - "p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "requires": { - "p-finally": "^1.0.0" - } - }, "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -17417,11 +17237,6 @@ "readable-stream": "^3.6.0" } }, - "redis-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", - "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" - }, "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", diff --git a/package.json b/package.json index df69483..9835082 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@comake/solid-on-rails", - "version": "0.8.0", + "version": "0.9.0", "description": "Runs a Node.js server using componentsjs preset with configurations for working with Solid.", "repository": { "type": "git", @@ -64,7 +64,6 @@ "@commitlint/cli": "^17.0.3", "@commitlint/config-conventional": "^17.0.3", "@tsconfig/node14": "^1.0.3", - "@types/bull": "^3.15.9", "@types/cors": "^2.8.12", "@types/ejs": "^3.1.1", "@types/end-of-stream": "^1.4.1", @@ -96,7 +95,7 @@ }, "dependencies": { "arrayify-stream": "^2.0.0", - "bull": "^4.8.5", + "bull": "^4.11.3", "componentsjs": "^5.4.0", "cors": "^2.8.5", "dotenv": "^16.0.1", diff --git a/src/cli/Cli.ts b/src/cli/Cli.ts index 811c325..63a5213 100644 --- a/src/cli/Cli.ts +++ b/src/cli/Cli.ts @@ -75,6 +75,7 @@ export class Cli { .command('db:revert', 'Revert the last executed migration') .command('queues:deleteAll', 'Delete all queues') .command('queues:delete', 'Delete a specific queue') + .command('queues:removeCompleted', 'Removes all completed jobs from a queue') .example('solid-on-rails storages:seed -c ./path/to/config.json -m .', 'Seed the storages with a custom config') .options(cliParameters) .wrap(120) @@ -125,5 +126,9 @@ export class Cli { const runner = new QueueAdapterAccessorRunner(); return runner.deleteQueue(params, argv); } + if (command === 'queues:removeCompleted') { + const runner = new QueueAdapterAccessorRunner(); + return runner.removeCompletedInQueue(params, argv); + } } } diff --git a/src/cli/QueueAdapterAccessorRunner.ts b/src/cli/QueueAdapterAccessorRunner.ts index 0b4edb4..c85d881 100644 --- a/src/cli/QueueAdapterAccessorRunner.ts +++ b/src/cli/QueueAdapterAccessorRunner.ts @@ -1,4 +1,5 @@ import type { CliArgv } from '../init/variables/Types'; +import type { QueueAdapter } from '../jobs/adapter/QueueAdapter'; import { getLoggerFor } from '../logging/LogUtil'; import type { CliParameters } from '../util/ComponentsJsUtil'; import type { AsyncronousAppRunnerCallbackArgs } from './AsyncronousAppAccessorRunner'; @@ -37,4 +38,18 @@ export class QueueAdapterAccessorRunner extends AsyncronousAppAccessorRunner { }; await this.runAppAndExecuteCallbackWithInstancesAndEnv(params, argv, callback); } + + public async removeCompletedInQueue( + params: CliParameters, + argv: CliArgv, + ): Promise { + const callback = async({ instances: { queueAdapter }}: AsyncronousAppRunnerCallbackArgs): Promise => { + const indexOfQueuesDeleteCommand = argv.indexOf('queues:removeCompleted'); + const queueName = argv[indexOfQueuesDeleteCommand + 1]; + this.logger.info(`Removing completed in the ${queueName} queue`); + await (queueAdapter as QueueAdapter).removeCompletedInQueue(queueName); + this.logger.info(`Successfully removed completed in the ${queueName} queue.`); + }; + await this.runAppAndExecuteCallbackWithInstancesAndEnv(params, argv, callback); + } } diff --git a/src/jobs/JobOptions.ts b/src/jobs/JobOptions.ts index 417f5e8..3b7921d 100644 --- a/src/jobs/JobOptions.ts +++ b/src/jobs/JobOptions.ts @@ -23,6 +23,18 @@ export interface JobOptions { * If this field is not specified, the job will not be auto-retried. */ retryAttempts?: number; + /** + * Whether jobs of this type should be removed from storage after they are completed + */ + disableRemoveOnComplete?: boolean; + /** + * The number of seconds after which to remove failed jobs + */ + removeOnFailAge?: number; + /** + * The number of failed jobs to keep + */ + removeOnFailCount?: number; } export type ConfiguredJobOptions = Omit; diff --git a/src/jobs/adapter/BullQueueAdapter.ts b/src/jobs/adapter/BullQueueAdapter.ts index 9f90805..0fb093c 100644 --- a/src/jobs/adapter/BullQueueAdapter.ts +++ b/src/jobs/adapter/BullQueueAdapter.ts @@ -1,5 +1,5 @@ /* eslint-disable tsdoc/syntax */ -import type { Queue } from 'bull'; +import type { Queue, Job as BullJob } from 'bull'; import Bull from 'bull'; import { getLoggerFor } from '../../logging/LogUtil'; import type { Job } from '../Job'; @@ -122,6 +122,15 @@ export class BullQueueAdapter implements QueueAdapter { private jobOptionsToBullOptions(options: JobOptions): Bull.JobOptions { const bullOptions: Bull.JobOptions = {}; + bullOptions.removeOnComplete = !options.disableRemoveOnComplete; + + if (options.removeOnFailAge ?? options.removeOnFailCount) { + bullOptions.removeOnFail = { + age: options.removeOnFailAge, + count: options.removeOnFailCount, + }; + } + if (options.every && typeof options.every === 'string') { bullOptions.repeat = { cron: options.every }; this.addDelayFieldToObject(bullOptions.repeat, 'startDate', options.at ?? options.in); @@ -173,4 +182,16 @@ export class BullQueueAdapter implements QueueAdapter { .map((queueName: string): Promise => this.deleteQueue(queueName)), ); } + + public async removeCompletedInQueue(queueName: string): Promise { + const queue = this.queues[queueName]; + if (!queue) { + throw new Error(`No queue named ${queueName} found`); + } + const getCompletedFn = queue.getCompleted as (start?: number, end?: number, opts?: any) => Promise; + const completed = await getCompletedFn(undefined, undefined, { excludeData: true }); + await Promise.all( + completed.map((job): Promise => job.remove()), + ); + } } diff --git a/src/jobs/adapter/QueueAdapter.ts b/src/jobs/adapter/QueueAdapter.ts index b1b0671..c4bb64d 100644 --- a/src/jobs/adapter/QueueAdapter.ts +++ b/src/jobs/adapter/QueueAdapter.ts @@ -13,5 +13,7 @@ export interface QueueAdapter extends Finalizable { deleteQueue: (queueName: string) => Promise; + removeCompletedInQueue: (queueName: string) => Promise; + deleteAllQueues: () => Promise; } diff --git a/test/unit/cli/Cli.test.ts b/test/unit/cli/Cli.test.ts index 7e76608..01f8475 100644 --- a/test/unit/cli/Cli.test.ts +++ b/test/unit/cli/Cli.test.ts @@ -14,14 +14,15 @@ jest.mock('../../../src/init/AppRunner'); const HELP_MESSAGE = `solid-on-rails [] Commands: - script task Run a task from the tasks folder - script storages:seed Seed the storages from the ./db/seeds.js file - script storages:drop Drop all data from the DataMapper and KeyValue Storages - script db:setup Setup the database from configured entity schemas - script db:migrate Run all pending migrations in the ./db/migrations folder - script db:revert Revert the last executed migration - script queues:deleteAll Delete all queues - script queues:delete Delete a specific queue + script task Run a task from the tasks folder + script storages:seed Seed the storages from the ./db/seeds.js file + script storages:drop Drop all data from the DataMapper and KeyValue Storages + script db:setup Setup the database from configured entity schemas + script db:migrate Run all pending migrations in the ./db/migrations folder + script db:revert Revert the last executed migration + script queues:deleteAll Delete all queues + script queues:delete Delete a specific queue + script queues:removeCompleted Removes all completed jobs from a queue Options: --help Show help [boolean] --version Show version number [boolean] @@ -124,6 +125,23 @@ describe('The Cli', (): void => { ); }); + it('runs the queues:removeCompleted command.', async(): Promise => { + const removeCompletedInQueue = jest.fn().mockReturnValue(Promise.resolve()); + (QueueAdapterAccessorRunner as jest.Mock).mockImplementation((): any => ({ removeCompletedInQueue })); + // eslint-disable-next-line no-sync + new Cli().runCliSync({ argv: [ 'node', 'script', 'queues:removeCompleted', 'default' ]}); + // Wait until app.start has been called, because we can't await AppRunner.run. + await flushPromises(); + expect(QueueAdapterAccessorRunner).toHaveBeenCalledTimes(1); + expect(removeCompletedInQueue).toHaveBeenCalledTimes(1); + expect(removeCompletedInQueue).toHaveBeenCalledWith( + expect.objectContaining({ + loggingLevel: 'info', + }), + [ 'node', 'script', 'queues:removeCompleted', 'default' ], + ); + }); + it('runs the db:setup command.', async(): Promise => { const setupDatabase = jest.fn().mockReturnValue(Promise.resolve()); (StorageAccessorRunner as jest.Mock).mockImplementation((): any => ({ setupDatabase })); diff --git a/test/unit/cli/QueueAdapterAccessorRunner.test.ts b/test/unit/cli/QueueAdapterAccessorRunner.test.ts index 0f03aa9..8eb72c9 100644 --- a/test/unit/cli/QueueAdapterAccessorRunner.test.ts +++ b/test/unit/cli/QueueAdapterAccessorRunner.test.ts @@ -16,6 +16,7 @@ const app: jest.Mocked = { const queueAdapter: jest.Mocked = { deleteAllQueues: jest.fn(), deleteQueue: jest.fn(), + removeCompletedInQueue: jest.fn(), } as any; const instances = { @@ -147,4 +148,41 @@ describe('QueueAdapterAccessorRunner', (): void => { expect(app.stop).toHaveBeenCalledTimes(1); }); }); + + describe('removeCompleted', (): void => { + it('runs the server and removed completed jobs from a queue.', async(): Promise => { + const runner = new QueueAdapterAccessorRunner(); + await expect(runner.removeCompletedInQueue(params, [ 'node', 'queues:removeCompleted', 'default' ])) + .resolves.toBeUndefined(); + expect(ComponentsManager.build).toHaveBeenCalledTimes(1); + expect(ComponentsManager.build).toHaveBeenCalledWith({ + dumpErrorState: true, + logLevel: 'info', + mainModulePath: joinFilePath(__dirname, '../../../'), + typeChecking: false, + }); + expect(manager.configRegistry.register).toHaveBeenCalledTimes(1); + expect(manager.configRegistry.register).toHaveBeenCalledWith('/var/cwd/config.json'); + expect(manager.instantiate).toHaveBeenCalledTimes(2); + expect(manager.instantiate).toHaveBeenNthCalledWith( + 1, + 'urn:solid-on-rails-setup:default:CliResolver', + { variables: { 'urn:solid-on-rails:default:variable:modulePathPlaceholder': '@SoR:' }}, + ); + expect(cliExtractor.handleSafe).toHaveBeenCalledTimes(1); + expect(cliExtractor.handleSafe).toHaveBeenCalledWith({ + argv: [ 'node', 'queues:removeCompleted', 'default' ], + envVarPrefix: '', + }); + expect(settingsResolver.handleSafe).toHaveBeenCalledTimes(1); + expect(settingsResolver.handleSafe).toHaveBeenCalledWith(defaultParameters); + expect(manager.instantiate).toHaveBeenNthCalledWith(2, + 'urn:solid-on-rails:queue-accessor:Instances', + { variables: defaultVariables }); + expect(app.start).toHaveBeenCalledTimes(1); + expect(queueAdapter.removeCompletedInQueue).toHaveBeenCalledTimes(1); + expect(queueAdapter.removeCompletedInQueue).toHaveBeenCalledWith('default'); + expect(app.stop).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/test/unit/jobs/adapter/BullQueueAdapter.test.ts b/test/unit/jobs/adapter/BullQueueAdapter.test.ts index 09b5626..00595f8 100644 --- a/test/unit/jobs/adapter/BullQueueAdapter.test.ts +++ b/test/unit/jobs/adapter/BullQueueAdapter.test.ts @@ -20,6 +20,7 @@ describe('A BullQueueAdapter', (): void => { let close: any; let add: any; let obliterate: any; + let getCompleted: any; const redisConfig = { port: 6379, host: '127.0.0.1' }; let adapter: BullQueueAdapter; let queueProcessor: BullQueueProcessor; @@ -36,11 +37,12 @@ describe('A BullQueueAdapter', (): void => { close = jest.fn(); obliterate = jest.fn(); + getCompleted = jest.fn(); add = jest.fn(); queueProcessor = { processJobsOnQueues: jest.fn() } as any; (Bull as jest.Mock).mockImplementation( - (name: string): Bull.Queue => ({ process, add, name, close, obliterate } as any), + (name: string): Bull.Queue => ({ process, add, name, close, obliterate, getCompleted } as any), ); }); @@ -50,7 +52,7 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, {}); + expect(add).toHaveBeenCalledWith('example', {}, { removeOnComplete: true }); }); it('initializes queues with only one total concurrent job.', async(): Promise => { @@ -66,7 +68,7 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, {}); + expect(add).toHaveBeenCalledWith('example', {}, { removeOnComplete: true }); await adapter.finalize(); expect(close).toHaveBeenCalledTimes(1); }); @@ -91,7 +93,7 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', data, {}); + expect(add).toHaveBeenCalledWith('example', data, { removeOnComplete: true }); }); it('adds the job on a cron schedule.', async(): Promise => { @@ -101,7 +103,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { repeat: { cron: '5 4 * * *' }}); + expect(add).toHaveBeenCalledWith('example', {}, { + repeat: { cron: '5 4 * * *' }, + removeOnComplete: true, + }); }); it('adds the job on a cron schedule with a start time at a specific date.', async(): Promise => { @@ -111,7 +116,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { repeat: { cron: '5 4 * * *', startDate: dayInMs }}); + expect(add).toHaveBeenCalledWith('example', {}, { + repeat: { cron: '5 4 * * *', startDate: dayInMs }, + removeOnComplete: true, + }); }); it('adds the job on a cron schedule with a start time after a delay.', async(): Promise => { @@ -121,7 +129,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { repeat: { cron: '5 4 * * *', startDate: 1000 }}); + expect(add).toHaveBeenCalledWith('example', {}, { + repeat: { cron: '5 4 * * *', startDate: 1000 }, + removeOnComplete: true, + }); }); it('adds the job on a certain millisecond schedule.', async(): Promise => { @@ -131,7 +142,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { repeat: { every: 1000 }}); + expect(add).toHaveBeenCalledWith('example', {}, { + repeat: { every: 1000 }, + removeOnComplete: true, + }); }); it('adds the job to be run at a specific date.', async(): Promise => { @@ -141,7 +155,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { delay: dayInMs }); + expect(add).toHaveBeenCalledWith('example', {}, { + delay: dayInMs, + removeOnComplete: true, + }); }); it('adds the job to be run after a delay.', async(): Promise => { @@ -151,7 +168,10 @@ describe('A BullQueueAdapter', (): void => { expect(Bull).toHaveBeenCalledTimes(1); expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); expect(add).toHaveBeenCalledTimes(1); - expect(add).toHaveBeenCalledWith('example', {}, { delay: 1000 }); + expect(add).toHaveBeenCalledWith('example', {}, { + delay: 1000, + removeOnComplete: true, + }); }); it('adds the job with an auto-retry backoff setting.', async(): Promise => { @@ -164,6 +184,51 @@ describe('A BullQueueAdapter', (): void => { expect(add).toHaveBeenCalledWith('example', {}, { attempts: 3, backoff: { type: 'exponential', delay: 2000 }, + removeOnComplete: true, + }); + }); + + it('adds the job with remove on complete setting disabled.', async(): Promise => { + adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor); + await expect(adapter.performLater('example', {}, { disableRemoveOnComplete: true })) + .resolves.toBeUndefined(); + expect(Bull).toHaveBeenCalledTimes(1); + expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); + expect(add).toHaveBeenCalledTimes(1); + expect(add).toHaveBeenCalledWith('example', {}, { + removeOnComplete: false, + }); + }); + + it('adds the job with a removeOnFail age.', async(): Promise => { + adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor); + await expect(adapter.performLater('example', {}, { removeOnFailAge: 1000 })) + .resolves.toBeUndefined(); + expect(Bull).toHaveBeenCalledTimes(1); + expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); + expect(add).toHaveBeenCalledTimes(1); + expect(add).toHaveBeenCalledWith('example', {}, { + removeOnComplete: true, + removeOnFail: { + age: 1000, + count: undefined, + }, + }); + }); + + it('adds the job with a removeOnFail count.', async(): Promise => { + adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor); + await expect(adapter.performLater('example', {}, { removeOnFailCount: 1000 })) + .resolves.toBeUndefined(); + expect(Bull).toHaveBeenCalledTimes(1); + expect(Bull).toHaveBeenCalledWith('default', { redis: redisConfig, settings: {}}); + expect(add).toHaveBeenCalledTimes(1); + expect(add).toHaveBeenCalledWith('example', {}, { + removeOnComplete: true, + removeOnFail: { + age: undefined, + count: 1000, + }, }); }); @@ -180,6 +245,25 @@ describe('A BullQueueAdapter', (): void => { expect(obliterate).toHaveBeenCalledTimes(0); }); + it('removes completed jobs in a queue.', async(): Promise => { + const remove = jest.fn(); + getCompleted.mockResolvedValue([ + { remove }, + { remove }, + ]); + adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor); + await expect(adapter.removeCompletedInQueue('default')).resolves.toBeUndefined(); + expect(getCompleted).toHaveBeenCalledTimes(1); + expect(getCompleted).toHaveBeenCalledWith(undefined, undefined, { excludeData: true }); + expect(remove).toHaveBeenCalledTimes(2); + }); + + it('throws an error when removing completed jobs from a queue that does not exist.', async(): Promise => { + adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor); + await expect(adapter.removeCompletedInQueue('otherQueue')).rejects.toThrow('No queue named otherQueue found'); + expect(getCompleted).toHaveBeenCalledTimes(0); + }); + it('deletes all queues.', async(): Promise => { queues = { queue: {}, secondQueue: {}}; adapter = new BullQueueAdapter(jobs, queues, redisConfig, queueProcessor);