feat: enable migration testing (#1583)

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -598,6 +598,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         await this._init();
         await this.stats.startCapturing();
 
+        process.once('SIGINT', async () => {
+            this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm run start');
+            await this._pauseOnMigration();
+            await this.autoscaledPool!.abort();
+        });
+
         try {
             this.log.info('Starting the crawl');
             await this.autoscaledPool!.run();

test/e2e/migration/actor/.gitignore

@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage

test/e2e/migration/actor/Dockerfile

@@ -0,0 +1,16 @@
+FROM apify/actor-node:16-beta
+
+COPY packages ./packages
+COPY package*.json ./
+
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && npm update \
+ && echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+COPY . ./

test/e2e/migration/actor/apify.json

@@ -0,0 +1,6 @@
+{
+	"name": "test-migration",
+	"version": "0.0",
+	"buildTag": "latest",
+	"env": null
+}

test/e2e/migration/actor/main.js

@@ -0,0 +1,35 @@
+import { Actor } from 'apify';
+import { CheerioCrawler, Configuration, Dataset } from '@crawlee/cheerio';
+import { ApifyStorageLocal } from '@apify/storage-local';
+
+process.env.CRAWLEE_PURGE_ON_START = '0';
+
+const mainOptions = {
+    exit: Actor.isAtHome(),
+    storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
+};
+
+const run = async () => {
+    await Actor.main(async () => {
+        const crawler = new CheerioCrawler({
+            maxConcurrency: 1,
+            maxRequestsPerCrawl: 5,
+            async requestHandler({ enqueueLinks, request }) {
+                const { url } = request;
+                await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });
+
+                await Dataset.pushData({ url });
+
+                process.emit('SIGINT');
+            },
+        });
+
+        // eslint-disable-next-line no-underscore-dangle
+        Configuration.getGlobalConfig().getStorageClient().__purged = false;
+
+        await crawler.run(['https://apify.com']);
+    }, mainOptions);
+};
+
+await run();
+await run();

test/e2e/migration/actor/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "test-migration",
+    "version": "0.0.1",
+    "description": "Migration Test",
+    "dependencies": {
+        "apify": "next",
+        "@apify/storage-local": "^2.1.0",
+        "@crawlee/basic": "file:./packages/basic-crawler",
+        "@crawlee/browser-pool": "file:./packages/browser-pool",
+        "@crawlee/http": "file:./packages/http-crawler",
+        "@crawlee/cheerio": "file:./packages/cheerio-crawler",
+        "@crawlee/core": "file:./packages/core",
+        "@crawlee/memory-storage": "file:./packages/memory-storage",
+        "@crawlee/types": "file:./packages/types",
+        "@crawlee/utils": "file:./packages/utils"
+    },
+    "overrides": {
+        "apify": {
+            "@crawlee/core": "file:./packages/core",
+            "@crawlee/utils": "file:./packages/utils"
+        }
+    },
+    "scripts": {
+        "start": "node main.js"
+    },
+    "type": "module",
+    "license": "ISC"
+}

test/e2e/migration/test.mjs

@@ -0,0 +1,10 @@
+import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname);
+
+await expect(datasetItems.length === 2, 'Number of dataset items');
+await expect(validateDataset(datasetItems, ['url']), 'Dataset items validation');
+await expect(datasetItems[0].url !== datasetItems[1].url, 'Dataset items unique');