From a8a1aec8e6d7c57c0c0f7020a3f017b3a68d0f00 Mon Sep 17 00:00:00 2001 From: Sviat Date: Fri, 14 Nov 2025 14:45:32 +0200 Subject: [PATCH 1/2] feat: add sitemap-scraper --- .gitignore | 2 + sitemap-scraper/.actor/actor.json | 13 + sitemap-scraper/.dockerignore | 10 + sitemap-scraper/Dockerfile | 32 + sitemap-scraper/INPUT_SCHEMA.json | 176 ++ sitemap-scraper/package-lock.json | 2388 +++++++++++++++++ sitemap-scraper/package.json | 40 + sitemap-scraper/src/internals/consts.ts | 45 + .../src/internals/crawler_setup.ts | 427 +++ sitemap-scraper/src/internals/index.ts | 2 + sitemap-scraper/src/main.ts | 5 + sitemap-scraper/tsconfig.json | 12 + 12 files changed, 3152 insertions(+) create mode 100644 sitemap-scraper/.actor/actor.json create mode 100644 sitemap-scraper/.dockerignore create mode 100644 sitemap-scraper/Dockerfile create mode 100644 sitemap-scraper/INPUT_SCHEMA.json create mode 100644 sitemap-scraper/package-lock.json create mode 100644 sitemap-scraper/package.json create mode 100644 sitemap-scraper/src/internals/consts.ts create mode 100644 sitemap-scraper/src/internals/crawler_setup.ts create mode 100644 sitemap-scraper/src/internals/index.ts create mode 100644 sitemap-scraper/src/main.ts create mode 100644 sitemap-scraper/tsconfig.json diff --git a/.gitignore b/.gitignore index 515503dc..ce88fdff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .idea +storage apify_storage +dist node_modules .nyc_output coverage diff --git a/sitemap-scraper/.actor/actor.json b/sitemap-scraper/.actor/actor.json new file mode 100644 index 00000000..dd848644 --- /dev/null +++ b/sitemap-scraper/.actor/actor.json @@ -0,0 +1,13 @@ +{ + "actorSpecification": 1, + "name": "sitemap-scraper", + "version": "0.1", + "buildTag": "latest", + "storages": { + "dataset": { + "actorSpecification": 1, + "fields": {}, + "views": {} + } + } +} diff --git a/sitemap-scraper/.dockerignore b/sitemap-scraper/.dockerignore new file mode 100644 index 00000000..ceb85b1c --- /dev/null +++ b/sitemap-scraper/.dockerignore @@ -0,0 +1,10 @@ +# configurations +.idea + +# crawlee and apify storage folders +apify_storage +crawlee_storage +storage + +# installed files +node_modules diff --git a/sitemap-scraper/Dockerfile b/sitemap-scraper/Dockerfile new file mode 100644 index 00000000..d166d657 --- /dev/null +++ b/sitemap-scraper/Dockerfile @@ -0,0 +1,32 @@ +FROM apify/actor-node:22 AS builder + +COPY package*.json ./ + +RUN npm install --include=dev --audit=false + +COPY . ./ + +RUN npm run build + +FROM apify/actor-node:22 + +COPY --from=builder /usr/src/app/dist ./dist + +COPY package*.json ./ + +RUN rm -rf node_modules \ + && npm --quiet set progress=false \ + && npm install --omit=dev --omit=optional \ + && echo "Installed NPM packages:" \ + && (npm list --omit=dev --all || true) \ + && echo "Node.js version:" \ + && node --version \ + && echo "NPM version:" \ + && npm --version \ + && rm -r ~/.npm + +COPY . ./ + +ENV APIFY_DISABLE_OUTDATED_WARNING=1 + +CMD npm run start:prod --silent diff --git a/sitemap-scraper/INPUT_SCHEMA.json b/sitemap-scraper/INPUT_SCHEMA.json new file mode 100644 index 00000000..b11547f4 --- /dev/null +++ b/sitemap-scraper/INPUT_SCHEMA.json @@ -0,0 +1,176 @@ +{ + "title": "Sitemap Scraper Input", + "type": "object", + "description": "", + "schemaVersion": 1, + "properties": { + "startUrls": { + "sectionCaption": "Basic configuration", + "title": "Sitemap URLs", + "type": "array", + "description": "A static list of sitemap URLs to scrape.

For details, see the Start URLs section in the README.", + "prefill": [ + { + "url": "https://docs.apify.com/sitemap.xml" + } + ], + "editor": "requestListSources" + }, + "proxyConfiguration": { + "sectionCaption": "Proxy and HTTP configuration", + "title": "Proxy configuration", + "type": "object", + "description": "Specifies proxy servers that will be used by the scraper in order to hide its origin.

For details, see Proxy configuration in README.", + "prefill": { + "useApifyProxy": true + }, + "default": { + "useApifyProxy": true + }, + "editor": "proxy" + }, + "proxyRotation": { + "title": "Proxy rotation", + "type": "string", + "description": "This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.", + "default": "RECOMMENDED", + "editor": "select", + "enum": [ + "RECOMMENDED", + "PER_REQUEST", + "UNTIL_FAILURE" + ], + "enumTitles": [ + "Use recommended settings", + "Rotate proxy after each request", + "Use one proxy until failure" + ] + }, + "sessionPoolName": { + "title": "Session pool name", + "type": "string", + "description": "Use only english alphanumeric characters dashes and underscores. A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple Actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new Actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.", + "editor": "textfield", + "minLength": 3, + "maxLength": 200, + "pattern": "[0-9A-z-]" + }, + "initialCookies": { + "title": "Initial cookies", + "type": "array", + "description": "A JSON array with cookies that will be send with every HTTP request made by the Sitemap Scraper, in the format accepted by the tough-cookie NPM package. This option is useful for transferring a logged-in session from an external web browser. For details how to do this, read this help article.", + "default": [], + "prefill": [], + "editor": "json" + }, + "suggestResponseEncoding": { + "title": "Suggest response encoding", + "type": "string", + "description": "The scraper automatically determines response encoding from the response headers. If the headers are invalid or information is missing, malformed responses may be produced. Use the Suggest response encoding option to provide a fall-back encoding to the Scraper for cases where it could not be determined.", + "editor": "textfield" + }, + "forceResponseEncoding": { + "title": "Force response encoding", + "type": "boolean", + "description": "If enabled, the suggested response encoding will be used even if a valid response encoding is provided by the target website. Use this only when you've inspected the responses thoroughly and are sure that they are the ones doing it wrong.", + "default": false + }, + "ignoreSslErrors": { + "title": "Ignore SSL errors", + "type": "boolean", + "description": "If enabled, the scraper will ignore SSL/TLS certificate errors. Use at your own risk.", + "default": false, + "groupCaption": "Security" + }, + "preNavigationHooks": { + "sectionCaption": "Advanced configuration", + "title": "Pre-navigation hooks", + "type": "string", + "description": "Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `requestAsBrowserOptions`, which are passed to the `requestAsBrowser()` function the crawler calls to navigate.", + "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept two arguments: the \"crawlingContext\" object\n// and \"requestAsBrowserOptions\" which are passed to the `requestAsBrowser()`\n// function the crawler calls to navigate..\n[\n async (crawlingContext, requestAsBrowserOptions) => {\n // ...\n }\n]", + "editor": "javascript" + }, + "postNavigationHooks": { + "title": "Post-navigation hooks", + "type": "string", + "description": "Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.", + "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept a single argument: the \"crawlingContext\" object.\n[\n async (crawlingContext) => {\n // ...\n },\n]", + "editor": "javascript" + }, + "maxRequestRetries": { + "title": "Max request retries", + "type": "integer", + "description": "The maximum number of times the scraper will retry to load each web page on error, in case of a page load error or an exception thrown by the Page function.

If set to 0, the page will be considered failed right after the first error.", + "minimum": 0, + "default": 3, + "unit": "retries" + }, + "maxPagesPerCrawl": { + "title": "Max pages per run", + "type": "integer", + "description": "The maximum number of pages that the scraper will load. The scraper will stop when this limit is reached. It is always a good idea to set this limit in order to prevent excess platform usage for misconfigured scrapers. Note that the actual number of pages loaded might be slightly higher than this value.

If set to 0, there is no limit.", + "minimum": 0, + "default": 0, + "unit": "pages" + }, + "maxResultsPerCrawl": { + "title": "Max result records", + "type": "integer", + "description": "The maximum number of records that will be saved to the resulting dataset. The scraper will stop when this limit is reached.

If set to 0, there is no limit.", + "minimum": 0, + "default": 0, + "unit": "results" + }, + "maxCrawlingDepth": { + "title": "Max crawling depth", + "type": "integer", + "description": "Specifies how many links away from the Start URLs the scraper will descend. This value is a safeguard against infinite crawling depths for misconfigured scrapers. Note that pages added using context.enqueuePage() in Page function are not subject to the maximum depth constraint.

If set to 0, there is no limit.", + "minimum": 0, + "default": 0 + }, + "maxConcurrency": { + "title": "Max concurrency", + "type": "integer", + "description": "Specifies the maximum number of pages that can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. This option enables you to set an upper limit, for example to reduce the load on a target web server.", + "minimum": 1, + "default": 50 + }, + "pageLoadTimeoutSecs": { + "title": "Page load timeout", + "type": "integer", + "description": "The maximum amount of time the scraper will wait for a web page to load, in seconds. If the web page does not load in this timeframe, it is considered to have failed and will be retried (subject to Max page retries), similarly as with other page load errors.", + "minimum": 1, + "default": 60, + "unit": "seconds" + }, + "debugLog": { + "title": "Enable debug log", + "type": "boolean", + "description": "If enabled, the Actor log will include debug messages. Beware that this can be quite verbose. Use context.log.debug('message') to log your own debug messages from the Page function.", + "default": false, + "groupCaption": "Logging" + }, + "datasetName": { + "title": "Dataset name", + "type": "string", + "description": "Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.", + "editor": "textfield" + }, + "keyValueStoreName": { + "title": "Key-value store name", + "type": "string", + "description": "Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.", + "editor": "textfield" + }, + "requestQueueName": { + "title": "Request queue name", + "type": "string", + "description": "Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.", + "editor": "textfield" + } + }, + "required": [ + "startUrls", + "proxyConfiguration" + ] +} diff --git a/sitemap-scraper/package-lock.json b/sitemap-scraper/package-lock.json new file mode 100644 index 00000000..57f95cc3 --- /dev/null +++ b/sitemap-scraper/package-lock.json @@ -0,0 +1,2388 @@ +{ + "name": "@apify/actor-sitemap-scraper", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@apify/actor-sitemap-scraper", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "@apify/scraper-tools": "^1.1.4", + "@crawlee/http": "^3.14.1", + "@crawlee/utils": "^3.15.3", + "apify": "^3.2.6" + }, + "devDependencies": { + "@apify/tsconfig": "^0.1.0", + "@types/node": "^24.0.0", + "tsx": "^4.19.1", + "typescript": "~5.9.0" + } + }, + "node_modules/@apify/consts": { + "version": "2.47.0", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.47.0.tgz", + "integrity": "sha512-VFVrNn/S3bLPrS3v9x7Td5b8YtxPbrepuXLWu27n06T34qQeogysbnAWIBNnTQ59qRKlxrAeFR4ezI+fRXTMNQ==", + "license": "Apache-2.0" + }, + "node_modules/@apify/datastructures": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@apify/datastructures/-/datastructures-2.0.3.tgz", + "integrity": "sha512-E6yQyc/XZDqJopbaGmhzZXMJqwGf96ELtDANZa0t68jcOAJZS+pF7YUfQOLszXq6JQAdnRvTH2caotL6urX7HA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/input_secrets": { + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@apify/input_secrets/-/input_secrets-1.2.14.tgz", + "integrity": "sha512-6PtVmjfO0o4QttQjj7/hKc3L5PqoRWk3jwPcyAN27E1hz0rPo/Iup31oMyT8w2XCrjLYH4matPtsVJzc+Um3Dg==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.5.26", + "@apify/utilities": "^2.23.0", + "ow": "^0.28.2" + } + }, + "node_modules/@apify/log": { + "version": "2.5.26", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.26.tgz", + "integrity": "sha512-o/Ciki7UwaXwdJ+tvTg7WG8Q3C+hZXYUpo2FrTG7Upr1PQW45PGfwLDuJxteIBsEXjIHtWitlLi4AZwG7mtRMQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.47.0", + "ansi-colors": "^4.1.1" + } + }, + "node_modules/@apify/ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@apify/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-VHIswI7rD/R4bToeIDuJ9WJXt+qr5SdhfoZ9RzdjmCs9mgy7l0P4RugQEUCcU+WB4sfImbd4CKwzXcn0uYx1yw==", + "license": "MIT", + "dependencies": { + "event-stream": "3.3.4" + }, + "bin": { + "ps-tree": "bin/ps-tree.js" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@apify/pseudo_url": { + "version": "2.0.67", + "resolved": "https://registry.npmjs.org/@apify/pseudo_url/-/pseudo_url-2.0.67.tgz", + "integrity": "sha512-2MEtpxQVCRmIG97sRxAPnwf7niCUl+rC31j/9gKQhy6q3Wwho2IOrWLxPJE43K/clDdnmzm0LdfJkLwf2LsADw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.5.26" + } + }, + "node_modules/@apify/scraper-tools": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@apify/scraper-tools/-/scraper-tools-1.1.4.tgz", + "integrity": "sha512-qAKl1Iz+vIoSTOBQSvEYCKT2F25z1sXJHyBfv6KZ8qpB0dHyAuA83wMbfLp2OA6M7MJPTY7RUV8SpFyw0ClXuQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "ajv": "^8.12.0", + "content-type": "^1.0.5", + "tslib": "^2.6.1" + }, + "peerDependencies": { + "@crawlee/browser-pool": "^3.8.2", + "@crawlee/core": "^3.8.2", + "@crawlee/types": "^3.8.2", + "@crawlee/utils": "^3.8.2", + "apify": "^3.1.8" + }, + "peerDependenciesMeta": { + "@crawlee/playwright": { + "optional": true + }, + "@crawlee/puppeteer": { + "optional": true + } + } + }, + "node_modules/@apify/timeout": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@apify/timeout/-/timeout-0.3.2.tgz", + "integrity": "sha512-JnOLIOpqfm366q7opKrA6HrL0iYRpYYDn8Mi77sMR2GZ1fPbwMWCVzN23LJWfJV7izetZbCMrqRUXsR1etZ7dA==", + "license": "Apache-2.0" + }, + "node_modules/@apify/tsconfig": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@apify/tsconfig/-/tsconfig-0.1.1.tgz", + "integrity": "sha512-cS7mwN2UW1UXcluGXRDHH0Vr2VsSLkw2DwLTwoSBkcJSe8fvCr3MPryTSq0uod4MashpMURxJ7CsLKxs82VmOQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@apify/utilities": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.23.0.tgz", + "integrity": "sha512-WAyenlKvtXtvd6V8D2fYwbsmc3dMn3z02JaOhZNx/p8u0NuacNgoLk/+PW4IPWrNxhqgDQZqde4cpNfZOktvsQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.47.0", + "@apify/log": "^2.5.26" + } + }, + "node_modules/@borewit/text-codec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.1.1.tgz", + "integrity": "sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@crawlee/basic": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.15.3.tgz", + "integrity": "sha512-+j0rhP16Gx84eFFXnG2t0YxmwkIwz5cWFnJ6CFyj1F7ElQ5JmVkzxyIWoyKBWCmLpPlafySht1tKq7L/4EZ1AQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "ow": "^0.28.1", + "tldts": "^7.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/browser-pool": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.15.3.tgz", + "integrity": "sha512-a+QPQyHhLOO2cVzqjA8c9nuu++omzS9PWXRq248z46F+CnmAyYbClhuKiuBwhaNSTDKv7mgD8u1HikpzSN1duA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/timeout": "^0.3.0", + "@crawlee/core": "3.15.3", + "@crawlee/types": "3.15.3", + "fingerprint-generator": "^2.1.68", + "fingerprint-injector": "^2.1.68", + "lodash.merge": "^4.6.2", + "nanoid": "^3.3.4", + "ow": "^0.28.1", + "p-limit": "^3.1.0", + "proxy-chain": "^2.0.1", + "quick-lru": "^5.1.1", + "tiny-typed-emitter": "^2.1.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/@crawlee/core": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.15.3.tgz", + "integrity": "sha512-cBglpY4KVlKnozeO8K4lw6/TDWajtJjPj4aNfckxUeEzapJgvTaxg6ZI4zxir6vic8sTeK9Olp3qG3wLnlrtXw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.20.0", + "@apify/datastructures": "^2.0.0", + "@apify/log": "^2.4.0", + "@apify/pseudo_url": "^2.0.30", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/memory-storage": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@sapphire/async-queue": "^1.5.1", + "@vladfrangu/async_event_emitter": "^2.2.2", + "csv-stringify": "^6.2.0", + "fs-extra": "^11.0.0", + "got-scraping": "^4.0.0", + "json5": "^2.2.3", + "minimatch": "^9.0.0", + "ow": "^0.28.1", + "stream-json": "^1.8.0", + "tldts": "^7.0.0", + "tough-cookie": "^6.0.0", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/http": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.15.3.tgz", + "integrity": "sha512-NvD9khVsji6gX/t1YNBgTSlVCMRgzVhkL/oygyXPjfihfX42adAWJGi/ztK5/drA+7nNZlGY304W9Kheo5SqCQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/basic": "3.15.3", + "@crawlee/types": "3.15.3", + "@crawlee/utils": "3.15.3", + "@types/content-type": "^1.1.5", + "cheerio": "1.0.0-rc.12", + "content-type": "^1.0.4", + "got-scraping": "^4.0.0", + "iconv-lite": "^0.7.0", + "mime-types": "^2.1.35", + "ow": "^0.28.1", + "tslib": "^2.4.0", + "type-fest": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/memory-storage": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.15.3.tgz", + "integrity": "sha512-iOUOGBTZNyl2srDsrAJwhYu4+leOxQSlx9uAtGt88kC7srlrn2B/OgXjhzTv0Vo7+kkAiTkxUMAfZ2eNW+UbSw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@crawlee/types": "3.15.3", + "@sapphire/async-queue": "^1.5.0", + "@sapphire/shapeshift": "^3.0.0", + "content-type": "^1.0.4", + "fs-extra": "^11.0.0", + "json5": "^2.2.3", + "mime-types": "^2.1.35", + "proper-lockfile": "^4.1.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/@crawlee/types": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.15.3.tgz", + "integrity": "sha512-RvgVPXrsQw4GQIUXrC1z1aNOedUPJnZ/U/8n+jZ0fu1Iw9moJVMuiuIxSI8q1P6BA84aWZdalyfDWBZ3FMjsiw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@crawlee/utils": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.15.3.tgz", + "integrity": "sha512-guldTIfG+No6zoNmi5CKwABJDnrN8NqgwB9PFMR8kD+5r//TPFENfU9I3w4tQXx/pefnSZ99JrVZMUL3zenpJA==", + "license": "Apache-2.0", + "dependencies": { + "@apify/log": "^2.4.0", + "@apify/ps-tree": "^1.2.0", + "@crawlee/types": "3.15.3", + "@types/sax": "^1.2.7", + "cheerio": "1.0.0-rc.12", + "file-type": "^20.0.0", + "got-scraping": "^4.0.3", + "ow": "^0.28.1", + "robots-parser": "^3.0.1", + "sax": "^1.4.1", + "tslib": "^2.4.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "license": "MIT" + }, + "node_modules/@sapphire/async-queue": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@sapphire/async-queue/-/async-queue-1.5.5.tgz", + "integrity": "sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==", + "license": "MIT", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@sapphire/shapeshift": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.9.7.tgz", + "integrity": "sha512-4It2mxPSr4OGn4HSQWGmhFMsNFGfFVhWeRPCRwbH972Ek2pzfGRZtb0pJ4Ze6oIzcyh2jw7nUDa6qGlWofgd9g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=v16" + } + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.1.1.tgz", + "integrity": "sha512-rO92VvpgMc3kfiTjGT52LEtJ8Yc5kCWhZjLQ3LwlA4pSgPpQO7bVpYXParOD8Jwf+cVQECJo3yP/4I8aZtUQTQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, + "node_modules/@types/content-type": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@types/content-type/-/content-type-1.1.9.tgz", + "integrity": "sha512-Hq9IMnfekuOCsEmYl4QX2HBrT+XsfXiupfrLLY8Dcf3Puf4BkBOxSbWYTITSOQAhJoYPBez+b4MJRpIYL65z8A==", + "license": "MIT" + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vladfrangu/async_event_emitter": { + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@vladfrangu/async_event_emitter/-/async_event_emitter-2.4.7.tgz", + "integrity": "sha512-Xfe6rpCTxSxfbswi/W/Pz7zp1WWSNn4A0eW4mLkQUewCrXXtMj31lCg+iQyTkh/CkusZSq9eDflu7tjEDXUY6g==", + "license": "MIT", + "engines": { + "node": ">=v14.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/apify": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/apify/-/apify-3.5.2.tgz", + "integrity": "sha512-ypuAc/k9XduUSq6NZpqfOEdsY+vkT/anhw5X4SqGRUyB+b9JA/23WhWSxwmgGcxf020FNctBAAz0wxDcI8YCeg==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.23.0", + "@apify/input_secrets": "^1.2.0", + "@apify/log": "^2.4.3", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.14.1", + "@crawlee/types": "^3.14.1", + "@crawlee/utils": "^3.14.1", + "apify-client": "^2.17.0", + "fs-extra": "^11.2.0", + "ow": "^0.28.2", + "semver": "^7.5.4", + "tslib": "^2.6.2", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/apify-client": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.19.0.tgz", + "integrity": "sha512-cDYwbygx/OplyF9MXTeb70nKwZHQQIp5OodsPeikWrh8sHmfeKWUu9jUUzeiWpHIPcwroYrP7KxA6UDSBGY3kQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.42.0", + "@apify/log": "^2.2.6", + "@apify/utilities": "^2.18.0", + "@crawlee/types": "^3.3.0", + "agentkeepalive": "^4.2.1", + "async-retry": "^1.3.3", + "axios": "^1.6.7", + "content-type": "^1.0.5", + "ow": "^0.28.2", + "tslib": "^2.5.0", + "type-fest": "^4.0.0" + } + }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", + "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.28", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.28.tgz", + "integrity": "sha512-gYjt7OIqdM0PcttNYP2aVrr2G0bMALkBaoehD4BuRGjAOtipg0b6wHg1yNL+s5zSnLZZrGHOw4IrND8CD+3oIQ==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/byte-counter": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/byte-counter/-/byte-counter-0.1.0.tgz", + "integrity": "sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/cacheable-request": { + "version": "13.0.14", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-13.0.14.tgz", + "integrity": "sha512-2hjaCKFHLKCpzhNLjmOr5ODlbAypwpZqh3c2UusxzxKfQ899y0SvFHQJRYMYbefOYak3njSNBS5PmqCJjrmaBg==", + "license": "MIT", + "dependencies": { + "@types/http-cache-semantics": "^4.0.4", + "get-stream": "^9.0.1", + "http-cache-semantics": "^4.2.0", + "keyv": "^5.5.3", + "mimic-response": "^4.0.0", + "normalize-url": "^8.1.0", + "responselike": "^4.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001754", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001754.tgz", + "integrity": "sha512-x6OeBXueoAceOmotzx3PO4Zpt4rzpeIFsSr6AAePTZxSkXiYDUmpypEl7e2+8NCd9bD7bXjqyef8CJYPC1jfxg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/csv-stringify": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz", + "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-response": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-10.0.0.tgz", + "integrity": "sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==", + "license": "MIT", + "dependencies": { + "mimic-response": "^4.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "license": "MIT", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.252", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.252.tgz", + "integrity": "sha512-53uTpjtRgS7gjIxZ4qCgFdNO2q+wJt/Z8+xAvxbCqXPJrY6h7ighUkadQmNMXH96crtpa6gPFNP7BF4UBGDuaA==", + "license": "ISC" + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, + "node_modules/file-type": { + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.5.0.tgz", + "integrity": "sha512-BfHZtG/l9iMm4Ecianu7P8HRD2tBHLtjXinm4X62XBOYzi7CYA7jyqfJzOvXHqzVrVPYqBo2/GvbARMaaJkKVg==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/fingerprint-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.77.tgz", + "integrity": "sha512-wR15VUEZnwozFiSDRV+40zxlEt3ZV3JNYvLx0CSF9D9smov4pUC6MJZJnlxtDr+Ir4oppU8vn1JXApLk/Qr5Uw==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "generative-bayesian-network": "^2.1.77", + "header-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/fingerprint-injector": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.1.77.tgz", + "integrity": "sha512-R778SIyrqgWO0P+UWKzIFWUWZz13EGu6UmV7CX3vuFDbsYIL1xiH+s+/nzPSOqFdhXyLo7d8aTOjbGbRLULoQQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "fingerprint-generator": "^2.1.77", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "playwright": "^1.22.2", + "puppeteer": ">= 9.x" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.1.0.tgz", + "integrity": "sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "license": "MIT" + }, + "node_modules/fs-extra": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", + "integrity": "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/generative-bayesian-network": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.77.tgz", + "integrity": "sha512-viU4CRPsmgiklR94LhvdMndaY73BkCH1pGjmOjWbLR/ZwcUd06gKF3TCcsS3npRl74o33YSInSixxm16wIukcA==", + "license": "Apache-2.0", + "dependencies": { + "adm-zip": "^0.5.9", + "tslib": "^2.4.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-tsconfig": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", + "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/got": { + "version": "14.6.4", + "resolved": "https://registry.npmjs.org/got/-/got-14.6.4.tgz", + "integrity": "sha512-DjsLab39NUMf5iYlK9asVCkHMhaA2hEhrlmf+qXRhjEivuuBHWYbjmty9DA3OORUwZgENTB+6vSmY2ZW8gFHVw==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^7.0.1", + "byte-counter": "^0.1.0", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^13.0.12", + "decompress-response": "^10.0.0", + "form-data-encoder": "^4.0.2", + "http2-wrapper": "^2.2.1", + "keyv": "^5.5.3", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^4.0.1", + "responselike": "^4.0.2", + "type-fest": "^4.26.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/got-scraping": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.1.2.tgz", + "integrity": "sha512-LtVwPM5YLnNY7HVT/AK/yDBUg/4yOZSlAjjug2ovrHQseS43QCmO1XosKKXcXrfc6OMX8OnDbAWIauFMcaJ5TQ==", + "license": "Apache-2.0", + "dependencies": { + "got": "^14.2.1", + "header-generator": "^2.1.41", + "http2-wrapper": "^2.2.0", + "mimic-response": "^4.0.0", + "ow": "^1.1.1", + "quick-lru": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/got-scraping/node_modules/@sindresorhus/is": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", + "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/got-scraping/node_modules/callsites": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", + "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "license": "MIT", + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/ow": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ow/-/ow-1.1.1.tgz", + "integrity": "sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^5.3.0", + "callsites": "^4.0.0", + "dot-prop": "^7.2.0", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/quick-lru": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/header-generator": { + "version": "2.1.77", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.77.tgz", + "integrity": "sha512-ggSG/mfkFMu8CO7xP591G8kp1IJCBvgXu7M8oxTjC9u914JsIzE6zIfoFsXzA+pf0utWJhUsdqU0oV/DtQ4DFQ==", + "license": "Apache-2.0", + "dependencies": { + "browserslist": "^4.21.1", + "generative-bayesian-network": "^2.1.77", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http2-wrapper": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "license": "MIT", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.2.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/keyv": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.4.tgz", + "integrity": "sha512-eohl3hKTiVyD1ilYdw9T0OiB4hnjef89e3dMYKz+mVKDzj+5IteTseASUsOB+EU9Tf6VNTCjDePcP6wkDGmLKQ==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "license": "MIT", + "peer": true + }, + "node_modules/lowercase-keys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", + "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-response": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", + "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "license": "MIT" + }, + "node_modules/normalize-url": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.1.0.tgz", + "integrity": "sha512-X06Mfd/5aKsRHc0O0J5CUedwnPmnDtLF2+nq+KN9KSDlJHkPuh0JUviWjEWMe0SW/9TDdSLVPuk7L5gGTIA1/w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/ow": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz", + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.2.0", + "callsites": "^3.1.0", + "dot-prop": "^6.0.1", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/p-cancelable": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz", + "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "license": [ + "MIT", + "Apache2" + ], + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/proxy-chain": { + "version": "2.5.9", + "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.5.9.tgz", + "integrity": "sha512-DZZKtRz92WuXd7fzRTKgI/oGhjmSgGMgT3FweLunCztpaG5jDVOJp1jgRPAVLQD1SG6HhkOyRkj6RTF3A214bg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "socks": "^2.8.3", + "socks-proxy-agent": "^8.0.3", + "tslib": "^2.3.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "license": "MIT" + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/responselike": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-4.0.2.tgz", + "integrity": "sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==", + "license": "MIT", + "dependencies": { + "lowercase-keys": "^3.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.3.tgz", + "integrity": "sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ==", + "license": "BlueOak-1.0.0" + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "peer": true, + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "peer": true, + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "license": "MIT", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1" + } + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, + "node_modules/strtok3": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz", + "integrity": "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==", + "license": "MIT", + "peer": true + }, + "node_modules/tldts": { + "version": "7.0.17", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.17.tgz", + "integrity": "sha512-Y1KQBgDd/NUc+LfOtKS6mNsC9CCaH+m2P1RoIZy7RAPo3C3/t8X45+zgut31cRZtZ3xKPjfn3TkGTrctC2TQIQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.0.17" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.0.17", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.17.tgz", + "integrity": "sha512-DieYoGrP78PWKsrXr8MZwtQ7GLCUeLxihtjC1jZsW1DnvSMdKPitJSe8OSYDM2u5H6g3kWJZpePqkp43TfLh0g==", + "license": "MIT" + }, + "node_modules/token-types": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.1.tgz", + "integrity": "sha512-kh9LVIWH5CnL63Ipf0jhlBIy0UsrMj/NJDfpsy1SqOXlLKEVyXXYrnFxFT1yOOYVGBSApeVnjPw/sBz5BfEjAQ==", + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.1.0", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tough-cookie": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", + "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tsx": { + "version": "4.20.6", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", + "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.25.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vali-date": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/sitemap-scraper/package.json b/sitemap-scraper/package.json new file mode 100644 index 00000000..df054ab1 --- /dev/null +++ b/sitemap-scraper/package.json @@ -0,0 +1,40 @@ +{ + "name": "@apify/actor-sitemap-scraper", + "version": "0.0.1", + "private": true, + "description": "Crawl web pages from a sitemap using HTTP HEAD requests", + "type": "module", + "dependencies": { + "@apify/scraper-tools": "^1.1.4", + "@crawlee/http": "^3.14.1", + "@crawlee/utils": "^3.15.3", + "apify": "^3.2.6" + }, + "devDependencies": { + "@apify/tsconfig": "^0.1.0", + "@types/node": "^24.0.0", + "tsx": "^4.19.1", + "typescript": "~5.9.0" + }, + "scripts": { + "start": "npm run start:dev", + "start:prod": "node dist/main.js", + "start:dev": "tsx src/main.ts", + "build": "tsc" + }, + "repository": { + "type": "git", + "url": "https://github.com/apify/actor-scraper" + }, + "author": { + "name": "Apify Technologies", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [ + "Marek Trunkat ", + "Ondra Urban " + ], + "license": "Apache-2.0", + "homepage": "https://github.com/apify/actor-scraper" +} diff --git a/sitemap-scraper/src/internals/consts.ts b/sitemap-scraper/src/internals/consts.ts new file mode 100644 index 00000000..aca7a05c --- /dev/null +++ b/sitemap-scraper/src/internals/consts.ts @@ -0,0 +1,45 @@ +import type { + Dictionary, + ProxyConfigurationOptions, + RequestOptions, + Session, +} from '@crawlee/http'; + +export const enum ProxyRotation { + Recommended = 'RECOMMENDED', + PerRequest = 'PER_REQUEST', + UntilFailure = 'UNTIL_FAILURE', +} + +/** + * Replicates the INPUT_SCHEMA with JavaScript types for quick reference + * and IDE type check integration. + */ +export interface Input { + startUrls: RequestOptions[]; + keepUrlFragments: boolean; + respectRobotsTxtFile: boolean; + pageFunction: string; + preNavigationHooks?: string; + postNavigationHooks?: string; + proxyConfiguration: ProxyConfigurationOptions; + proxyRotation: ProxyRotation; + sessionPoolName?: string; + initialCookies: Parameters[0]; + additionalMimeTypes: string[]; + suggestResponseEncoding?: string; + forceResponseEncoding: boolean; + ignoreSslErrors: boolean; + maxRequestRetries: number; + maxPagesPerCrawl: number; + maxResultsPerCrawl: number; + maxCrawlingDepth: number; + maxConcurrency: number; + pageLoadTimeoutSecs: number; + pageFunctionTimeoutSecs: number; + debugLog: boolean; + customData: Dictionary; + datasetName?: string; + keyValueStoreName?: string; + requestQueueName?: string; +} diff --git a/sitemap-scraper/src/internals/crawler_setup.ts b/sitemap-scraper/src/internals/crawler_setup.ts new file mode 100644 index 00000000..daa572b3 --- /dev/null +++ b/sitemap-scraper/src/internals/crawler_setup.ts @@ -0,0 +1,427 @@ +import { readFile } from 'node:fs/promises'; +import type { IncomingMessage } from 'node:http'; +import { dirname } from 'node:path'; +import { fileURLToPath, URL } from 'node:url'; + +import type { + AutoscaledPool, + Awaitable, + Dictionary, + HttpCrawlerOptions, + HttpCrawlingContext, + InternalHttpCrawlingContext, + ProxyConfiguration, + Request, + RequestOptions, +} from '@crawlee/http'; +import { + Dataset, + HttpCrawler, + KeyValueStore, + log, + RequestList, + RequestQueueV2, +} from '@crawlee/http'; +import type { ApifyEnv, RequestQueue } from 'apify'; +import { Actor } from 'apify'; + +import type { CrawlerSetupOptions } from '@apify/scraper-tools'; +import { + constants as scraperToolsConstants, + tools, +} from '@apify/scraper-tools'; + +import type { Input } from './consts.js'; +import { ProxyRotation } from './consts.js'; +import { Sitemap } from '@crawlee/utils'; + +const { SESSION_MAX_USAGE_COUNTS } = scraperToolsConstants; +const SCHEMA = JSON.parse( + await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'), +); + +const MAX_EVENT_LOOP_OVERLOADED_RATIO = 0.9; +const SESSION_STORE_NAME = 'APIFY-HTTP-SCRAPER-SESSION-STORE'; +const REQUEST_QUEUE_INIT_FLAG_KEY = 'REQUEST_QUEUE_INITIALIZED'; + +/** + * Holds all the information necessary for constructing a crawler + * instance and creating a context for a pageFunction invocation. + */ +export class CrawlerSetup implements CrawlerSetupOptions { + name = 'Sitemap Scraper'; + rawInput: string; + env: ApifyEnv; + /** + * Used to store data that persist navigations + */ + globalStore = new Map(); + requestQueue: RequestQueueV2; + keyValueStore: KeyValueStore; + customData: unknown; + input: Input; + maxSessionUsageCount: number; + evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable)[]; + evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable)[]; + datasetName?: string; + keyValueStoreName?: string; + requestQueueName?: string; + + crawler!: HttpCrawler; + dataset!: Dataset; + pagesOutputted!: number; + proxyConfiguration?: ProxyConfiguration; + private initPromise: Promise; + protected readonly schema: object = SCHEMA; + + constructor(input: Input) { + // Set log level early to prevent missed messages. + if (input.debugLog) log.setLevel(log.LEVELS.DEBUG); + + // Keep this as string to be immutable. + this.rawInput = JSON.stringify(input); + + // Validate INPUT if not running on Apify Cloud Platform. + if (!Actor.isAtHome()) tools.checkInputOrThrow(input, this.schema); + + this.input = input; + this.env = Actor.getEnv(); + + // Validations + this.input.initialCookies.forEach((cookie) => { + if (!tools.isPlainObject(cookie)) { + throw new Error( + 'The initialCookies Array must only contain Objects.', + ); + } + }); + + // solving proxy rotation settings + this.maxSessionUsageCount = + SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation]; + + if (this.input.preNavigationHooks) { + this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow( + this.input.preNavigationHooks, + 'preNavigationHooks', + ); + } else { + this.evaledPreNavigationHooks = []; + } + + if (this.input.postNavigationHooks) { + this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow( + this.input.postNavigationHooks, + 'postNavigationHooks', + ); + } else { + this.evaledPostNavigationHooks = []; + } + + // Named storages + this.datasetName = this.input.datasetName; + this.keyValueStoreName = this.input.keyValueStoreName; + this.requestQueueName = this.input.requestQueueName; + + // Initialize async operations. + this.crawler = null!; + this.requestQueue = null!; + this.dataset = null!; + this.keyValueStore = null!; + this.proxyConfiguration = null!; + this.initPromise = this._initializeAsync(); + } + + private async _initializeAsync() { + // RequestList + const startUrls = this.input.startUrls.map((req) => { + req.useExtendedUniqueKey = true; + req.keepUrlFragment = this.input.keepUrlFragments; + return req; + }); + + // KeyValueStore + this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName); + + // RequestQueue + this.requestQueue = await RequestQueueV2.open(this.requestQueueName); + + if ( + !(await this.keyValueStore.recordExists( + REQUEST_QUEUE_INIT_FLAG_KEY, + )) + ) { + const requests: Request[] = []; + for await (const request of await RequestList.open( + null, + startUrls, + )) { + if ( + this.input.maxResultsPerCrawl > 0 && + requests.length >= 1.5 * this.input.maxResultsPerCrawl + ) { + break; + } + requests.push(request); + } + + const { waitForAllRequestsToBeAdded } = + await this.requestQueue.addRequestsBatched(requests); + + void waitForAllRequestsToBeAdded.then(async () => { + await this.keyValueStore.setValue( + REQUEST_QUEUE_INIT_FLAG_KEY, + '1', + ); + }); + } + + // Dataset + this.dataset = await Dataset.open(this.datasetName); + const info = await this.dataset.getInfo(); + this.pagesOutputted = info?.itemCount ?? 0; + + // Proxy configuration + this.proxyConfiguration = (await Actor.createProxyConfiguration( + this.input.proxyConfiguration, + )) as any as ProxyConfiguration; + } + + /** + * Resolves to a `HttpCrawler` instance. + */ + async createCrawler() { + await this.initPromise; + + const options: HttpCrawlerOptions = { + proxyConfiguration: this.proxyConfiguration, + requestHandler: this._requestHandler.bind(this), + preNavigationHooks: [], + postNavigationHooks: [], + requestQueue: this.requestQueue, + navigationTimeoutSecs: this.input.pageLoadTimeoutSecs, + ignoreSslErrors: this.input.ignoreSslErrors, + failedRequestHandler: this._failedRequestHandler.bind(this), + respectRobotsTxtFile: this.input.respectRobotsTxtFile, + maxRequestRetries: this.input.maxRequestRetries, + maxRequestsPerCrawl: + this.input.maxPagesPerCrawl === 0 + ? undefined + : this.input.maxPagesPerCrawl, + additionalMimeTypes: this.input.additionalMimeTypes, + autoscaledPoolOptions: { + maxConcurrency: this.input.maxConcurrency, + systemStatusOptions: { + maxEventLoopOverloadedRatio: + MAX_EVENT_LOOP_OVERLOADED_RATIO, + }, + }, + useSessionPool: true, + persistCookiesPerSession: true, + sessionPoolOptions: { + persistStateKeyValueStoreId: this.input.sessionPoolName + ? SESSION_STORE_NAME + : undefined, + persistStateKey: this.input.sessionPoolName, + sessionOptions: { + maxUsageCount: this.maxSessionUsageCount, + }, + }, + experiments: { + requestLocking: true, + }, + }; + + this._createNavigationHooks(options); + + if (this.input.proxyRotation === ProxyRotation.UntilFailure) { + options.sessionPoolOptions!.maxPoolSize = 1; + } + + if (this.input.suggestResponseEncoding) { + if (this.input.forceResponseEncoding) { + options.forceResponseEncoding = + this.input.suggestResponseEncoding; + } else { + options.suggestResponseEncoding = + this.input.suggestResponseEncoding; + } + } + + this.crawler = new HttpCrawler(options); + + return this.crawler; + } + + private _createNavigationHooks(options: HttpCrawlerOptions) { + options.preNavigationHooks!.push(async ({ request, session }) => { + // Normalize headers + request.headers = Object.entries(request.headers ?? {}).reduce( + (newHeaders, [key, value]) => { + newHeaders[key.toLowerCase()] = value; + return newHeaders; + }, + {} as Dictionary, + ); + + // Add initial cookies, if any. + if (this.input.initialCookies && this.input.initialCookies.length) { + const cookiesToSet = session + ? tools.getMissingCookiesFromSession( + session, + this.input.initialCookies, + request.url, + ) + : this.input.initialCookies; + if (cookiesToSet?.length) { + // setting initial cookies that are not already in the session and page + session?.setCookies(cookiesToSet, request.url); + } + } + }); + + options.preNavigationHooks!.push( + ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks), + ); + options.postNavigationHooks!.push( + ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks), + ); + } + + private _runHookWithEnhancedContext( + hooks: ((...args: unknown[]) => Awaitable)[], + ) { + return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => { + const { customData } = this.input; + return hook({ ...ctx, Apify: Actor, Actor, customData }, ...args); + }); + } + + private async _failedRequestHandler({ request }: HttpCrawlingContext) { + const lastError = + request.errorMessages[request.errorMessages.length - 1]; + const errorMessage = lastError ? lastError.split('\n')[0] : 'no error'; + log.error( + `Request ${request.url} failed and will not be retried anymore. Marking as failed.\nLast Error Message: ${errorMessage}`, + ); + return this._handleResult(request, undefined, undefined, true); + } + + /** + * Parses the sitemap if it's one and enqueues HEAD requests. Otherwise pushes + * the response data to the dataset. + */ + protected async _requestHandler(crawlingContext: HttpCrawlingContext) { + const { + request, + response, + body, + crawler, + } = crawlingContext; + + // Make sure that an object containing internal metadata + // is present on every request. + tools.ensureMetaData(request); + + // Abort the crawler if the maximum number of results was reached. + const aborted = await this._handleMaxResultsPerCrawl( + crawler.autoscaledPool, + ); + if (aborted) return; + + if (request.method === 'GET') { + log.info('Processing sitemap', { url: request.url }); + const sitemap = await Sitemap.fromXmlString(body.toString()); + await this.enqueueRequests({ + currentRequest: request, + requestQueue: this.requestQueue, + requestsOpts: sitemap.urls.map((url) => ({ + url, + method: 'HEAD' + })), + }); + log.info(`Enqueued ${sitemap.urls.length} URLs from sitemap`, { url: request.url }); + return; + } + + const result = { + url: request.url, + status: response.statusCode, + } + + // Save the `pageFunction`s result to the default dataset. + await this._handleResult( + request, + response, + result, + ); + } + + private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) { + if ( + !this.input.maxResultsPerCrawl || + this.pagesOutputted < this.input.maxResultsPerCrawl + ) + return false; + if (!autoscaledPool) return false; + log.info( + `User set limit of ${this.input.maxResultsPerCrawl} results was reached.Finishing the crawl.`, + ); + await autoscaledPool.abort(); + return true; + } + + private async _handleResult( + request: Request, + response?: IncomingMessage, + pageFunctionResult?: Dictionary, + isError?: boolean, + ) { + const payload = tools.createDatasetPayload( + request, + response, + pageFunctionResult, + isError, + ); + await this.dataset.pushData(payload); + + if (this.pagesOutputted % 100 === 0) { + log.info(`Pushed ${this.pagesOutputted} items to the dataset so far.`); + } + this.pagesOutputted++; + } + + /** + * Same as `Context.enqueueRequest`, but works for multiple of them + */ + async enqueueRequests({ + currentRequest, + requestQueue, + requestsOpts, + options = {}, + }: { + currentRequest: Request, + requestQueue: RequestQueue + requestsOpts: RequestOptions[]; + options?: {}; + }) { + const defaultRequestOpts = { + useExtendedUniqueKey: true, + keepUrlFragment: this.input.keepUrlFragments, + }; + const defaultUserData = { + [scraperToolsConstants.META_KEY]: { + parentRequestId: currentRequest.id || currentRequest.uniqueKey, + depth: (currentRequest.userData?.[scraperToolsConstants.META_KEY]).depth ?? 0 + 1, + }, + }; + + const newRequests = requestsOpts.map((requestOpts) => ({ + ...defaultRequestOpts, + ...requestOpts, + userData: { ...defaultUserData, ...requestOpts.userData }, + })); + + return requestQueue.addRequests(newRequests, options); + } +} diff --git a/sitemap-scraper/src/internals/index.ts b/sitemap-scraper/src/internals/index.ts new file mode 100644 index 00000000..1fa9bbd8 --- /dev/null +++ b/sitemap-scraper/src/internals/index.ts @@ -0,0 +1,2 @@ +export type { Input } from './consts.js'; +export { CrawlerSetup } from './crawler_setup.js'; diff --git a/sitemap-scraper/src/main.ts b/sitemap-scraper/src/main.ts new file mode 100644 index 00000000..0d3d9d20 --- /dev/null +++ b/sitemap-scraper/src/main.ts @@ -0,0 +1,5 @@ +import { runActor } from '@apify/scraper-tools'; + +import { CrawlerSetup } from './internals/crawler_setup.js'; + +runActor(CrawlerSetup); diff --git a/sitemap-scraper/tsconfig.json b/sitemap-scraper/tsconfig.json new file mode 100644 index 00000000..72d2e896 --- /dev/null +++ b/sitemap-scraper/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@apify/tsconfig", + "compilerOptions": { + "outDir": "dist", + "module": "ESNext", + "allowJs": true, + "skipLibCheck": true + }, + "include": [ + "src" + ] +} From adeef7dd9b5830ebfb883610d5c09e8df856763e Mon Sep 17 00:00:00 2001 From: Sviat Date: Tue, 18 Nov 2025 12:01:31 +0200 Subject: [PATCH 2/2] chore: remove unused --- sitemap-scraper/src/internals/crawler_setup.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sitemap-scraper/src/internals/crawler_setup.ts b/sitemap-scraper/src/internals/crawler_setup.ts index daa572b3..7b9f2c04 100644 --- a/sitemap-scraper/src/internals/crawler_setup.ts +++ b/sitemap-scraper/src/internals/crawler_setup.ts @@ -1,7 +1,6 @@ import { readFile } from 'node:fs/promises'; import type { IncomingMessage } from 'node:http'; -import { dirname } from 'node:path'; -import { fileURLToPath, URL } from 'node:url'; +import { URL } from 'node:url'; import type { AutoscaledPool,