From 9e75e378203b5739eaa7a76fa586c671e8b00a2f Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Mon, 5 Aug 2024 15:57:46 +0200 Subject: [PATCH] clean up corrupted versions after 1 minute --- package-lock.json | 20 ++++++ package.json | 3 +- .../components-cache/components-list.ts | 66 ++++++++++++++----- 3 files changed, 73 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index a65603cd6..30444ab45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "dotenv": "^16.4.5", "errorhandler": "^1.5.1", "express": "^4.19.2", + "fastq": "^1.17.1", "form-data": "^4.0.0", "fs-extra": "^11.2.0", "getport": "^0.1.0", @@ -5189,6 +5190,15 @@ "url": "https://paypal.me/naturalintelligence" } }, + "node_modules/fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -8290,6 +8300,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, "node_modules/right-align": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", diff --git a/package.json b/package.json index 24cdacab1..2e07ba830 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "dotenv": "^16.4.5", "errorhandler": "^1.5.1", "express": "^4.19.2", + "fastq": "^1.17.1", "form-data": "^4.0.0", "fs-extra": "^11.2.0", "getport": "^0.1.0", @@ -122,4 +123,4 @@ "universalify": "^2.0.0", "yargs": "^17.7.2" } -} \ No newline at end of file +} diff --git a/src/registry/domain/components-cache/components-list.ts b/src/registry/domain/components-cache/components-list.ts index f24fcbad1..a25a6edff 100644 --- a/src/registry/domain/components-cache/components-list.ts +++ b/src/registry/domain/components-cache/components-list.ts @@ -1,3 +1,5 @@ +import * as fastq from 'fastq'; +import type { queueAsPromised } from 'fastq'; import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp'; import type { StorageAdapter } from 'oc-storage-adapters-utils'; import semver from 'semver'; @@ -5,9 +7,45 @@ import type { ComponentsList, Config } from '../../../types'; import pLimit from '../../../utils/pLimit'; import eventsHandler from '../events-handler'; +const delay = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms)); +const validateComponentVersion = + (conf: Config, cdn: StorageAdapter) => + (componentName: string, componentVersion: string) => { + return cdn + .getJson( + // Check integrity of the package by checking existence of package.json + // OC will upload always the package.json last when publishing + `${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json` + ) + .then(() => true) + .catch(() => false); + }; + +interface QueueTask { + conf: Config; + cdn: StorageAdapter; + name: string; + version: string; +} +const cleanupQueue: queueAsPromised = fastq.promise( + cleanupWorker, + 1 +); + +async function cleanupWorker(arg: QueueTask): Promise { + const validator = validateComponentVersion(arg.conf, arg.cdn); + const isValid = await validator(arg.name, arg.version); + if (!isValid) { + await arg.cdn.removeDir( + `${arg.conf.storage.options.componentsDir}/${arg.name}/${arg.version}` + ); + } +} + export default function componentsList(conf: Config, cdn: StorageAdapter) { const filePath = (): string => `${conf.storage.options.componentsDir}/components.json`; + const validator = validateComponentVersion(conf, cdn); const componentsList = { getFromJson: (): Promise => cdn.getJson(filePath(), true), @@ -17,20 +55,6 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) { ): Promise => { const componentsInfo: Record = {}; - const validateComponentVersion = ( - componentName: string, - componentVersion: string - ) => { - return cdn - .getJson( - // Check integrity of the package by checking existence of package.json - // OC will upload always the package.json last when publishing - `${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json` - ) - .then(() => true) - .catch(() => false); - }; - const getVersionsForComponent = async ( componentName: string ): Promise => { @@ -45,7 +69,7 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) { await Promise.all( unCheckedVersions.map((unCheckedVersion) => limit(async () => { - const isValid = await validateComponentVersion( + const isValid = await validator( componentName, unCheckedVersion ); @@ -63,6 +87,18 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) { ', ' )}.` }); + delay(60_000).then(() => { + for (const invalidVersion of invalidVersions) { + cleanupQueue + .push({ + conf, + cdn, + name: componentName, + version: invalidVersion + }) + .catch(() => {}); + } + }); } const validVersions = allVersions.filter(