Persist cookies + localStorage plugin

packages/puppeteer-extra-plugin-session-persistence/ava.config-ts.js

@@ -0,0 +1,9 @@
+export default {
+  compileEnhancements: false,
+  environmentVariables: {
+    TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}'
+  },
+  files: ['src/**.test.ts', 'src/storage/**.test.ts'],
+  extensions: ['ts'],
+  require: ['ts-node/register']
+}

packages/puppeteer-extra-plugin-session-persistence/ava.config.js

@@ -0,0 +1,3 @@
+export default {
+  files: ['dist/*.test.js']
+}

packages/puppeteer-extra-plugin-session-persistence/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "puppeteer-extra-plugin-session-persistence",
+  "version": "1.0.2",
+  "description": "A puppeteer-extra plugin to persist sessions.",
+  "main": "dist/index.cjs.js",
+  "module": "dist/index.esm.js",
+  "typings": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "repository": "berstend/puppeteer-extra",
+  "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-session-persistence#readme",
+  "author": "kangoo13",
+  "license": "MIT",
+  "scripts": {
+    "clean": "rimraf dist/*",
+    "tscheck": "tsc --pretty --noEmit",
+    "prebuild": "run-s clean",
+    "build": "run-s build:tsc build:rollup",
+    "build:tsc": "tsc --module commonjs",
+    "build:rollup": "rollup -c rollup.config.ts",
+    "docs": "node -e 0",
+    "test": "ava -v --config ava.config-ts.js",
+    "pretest-ci": "run-s build",
+    "test-ci-back": "ava --concurrency 1 --serial --fail-fast -v",
+    "test-ci": "exit 0"
+  },
+  "engines": {
+    "node": ">=8"
+  },
+  "prettier": {
+    "printWidth": 80,
+    "semi": false,
+    "singleQuote": true
+  },
+  "keywords": [
+    "puppeteer",
+    "puppeteer-extra",
+    "puppeteer-extra-plugin",
+    "sessions",
+    "cookies",
+    "localstorage",
+    "persistence"
+  ],
+  "devDependencies": {
+    "@types/debug": "^4.1.5",
+    "@types/node-fetch": "^2.5.4",
+    "@types/psl": "^1.1.0",
+    "@types/puppeteer": "*",
+    "ava": "^2.4.0",
+    "npm-run-all": "^4.1.5",
+    "puppeteer": "9",
+    "puppeteer-extra": "^3.3.6",
+    "rimraf": "^3.0.0",
+    "rollup": "^1.27.5",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-node-resolve": "^5.2.0",
+    "rollup-plugin-sourcemaps": "^0.4.2",
+    "rollup-plugin-typescript2": "^0.25.2",
+    "ts-node": "^8.5.4",
+    "tslint": "^5.20.1",
+    "tslint-config-prettier": "^1.18.0",
+    "tslint-config-standard": "^9.0.0",
+    "typescript": "^4.3.3"
+  },
+  "dependencies": {
+    "debug": "^4.1.1",
+    "devtools-protocol": "^0.0.1138159",
+    "node-fetch": "^2.6.0",
+    "psl": "^1.9.0",
+    "puppeteer-extra-plugin": "^3.2.3",
+    "redis": "^4.6.6"
+  },
+  "peerDependencies": {
+    "puppeteer": "*",
+    "puppeteer-core": "*",
+    "puppeteer-extra": "*"
+  },
+  "peerDependenciesMeta": {
+    "puppeteer": {
+      "optional": true
+    },
+    "puppeteer-core": {
+      "optional": true
+    },
+    "puppeteer-extra": {
+      "optional": true
+    }
+  },
+  "gitHead": "72fe830c158f1e971c8499fdd5232338dd53c220"
+}

packages/puppeteer-extra-plugin-session-persistence/readme.md

@@ -0,0 +1,77 @@
+# Puppeteer Extra Plugin Session Persistence
+
+This TypeScript library provides a Puppeteer Extra plugin for persisting sessions in Puppeteer. It saves and loads cookies and localStorage data from different storage engines.
+
+## Features
+
+- Save and load cookies and localStorage data
+- Supports different storage engines (default: filesystem)
+- Merge cookies and localStorage data from constructor and storage
+- Automatically set cookies and localStorage data when a new page is created
+
+## Installation
+
+```bash
+npm install puppeteer-extra-plugin-session-persistence
+```
+
+## Strategies
+
+The plugin supports different strategies for persisting session data, all activated by default:
+- A polling strategy, update very X seconds the cookies from every page (default 1000 ms), very useful for XHR requests that sets cookies by JS
+- On HTTP response, update the cookies from the response thanks to the 'set-cookie' header
+- Using onFrameNavigated event, update the cookies and localStorage data from every frame
+
+## Usage
+
+```javascript
+const puppeteer = require('puppeteer-extra')
+puppeteer.use(require('puppeteer-extra-plugin-session-persistence')())
+// or
+puppeteer.use(require('puppeteer-extra-plugin-session-persistence')({
+  persistCookies: true,
+  persistLocalStorage: true,
+  storage: {
+    name: 'filesystem',
+    options: {
+      localStorageDataFile: './localStorageData.json',
+      cookiesFile: './cookies.json'
+    }
+  }
+}))
+const browser = await puppeteer.launch()
+```
+
+## Options
+
+- `persistCookies` (boolean, default: true): Allow or disallow cookies persistence.
+- `persistLocalStorage` (boolean, default: true): Allow or disallow local storage persistence.
+- `storage` (StorageConfig, default: filesystem storage): Storage options.
+- `localStorageData` (LocalStorageData, default: {}): Local storage data to load.
+- `cookies` (Cookie[], default: []): Cookies to load.
+
+## Storage Engines
+
+The plugin supports different storage engines for persisting session data:
+
+- File System Storage (default)
+- Redis Storage
+- In-Memory Storage
+
+To use a specific storage engine, pass the corresponding configuration object to the plugin constructor. For example, to use Redis storage:
+
+```javascript
+puppeteer.use(require('puppeteer-extra-plugin-session-persistence')({
+  storage: {
+    name: 'redis',
+    options: {
+      host: 'localhost',
+      port: 6379
+    }
+  }
+}))
+```
+
+## License
+
+This library is released under the MIT License.

packages/puppeteer-extra-plugin-session-persistence/rollup.config.ts

@@ -0,0 +1,65 @@
+import resolve from 'rollup-plugin-node-resolve'
+import sourceMaps from 'rollup-plugin-sourcemaps'
+import typescript from 'rollup-plugin-typescript2'
+
+const pkg = require('./package.json')
+
+const entryFile = 'index'
+const banner = `
+/*!
+ * ${pkg.name} v${pkg.version} by ${pkg.author}
+ * ${pkg.homepage || `https://github.com/${pkg.repository}`}
+ * @license ${pkg.license}
+ */
+`.trim()
+
+const defaultExportOutro = `
+  module.exports = exports.default || {}
+  Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value })
+`
+
+export default {
+  input: `src/${entryFile}.ts`,
+  output: [
+    {
+      file: pkg.main,
+      format: 'cjs',
+      sourcemap: true,
+      exports: 'named',
+      outro: defaultExportOutro,
+      banner
+    },
+    {
+      file: pkg.module,
+      format: 'es',
+      sourcemap: true,
+      exports: 'named',
+      banner
+    }
+  ],
+  // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+  external: [
+    ...Object.keys(pkg.dependencies || {}),
+    ...Object.keys(pkg.peerDependencies || {}),
+    'fs',
+    'os',
+    'path'
+  ],
+  watch: {
+    include: 'src/**'
+  },
+  plugins: [
+    // Compile TypeScript files
+    typescript({ useTsconfigDeclarationDir: true }),
+    // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+    // commonjs(),
+    // Allow node_modules resolution, so you can use 'external' to control
+    // which external modules to include in the bundle
+    // https://github.com/rollup/rollup-plugin-node-resolve#usage
+    resolve({
+      preferBuiltins: true
+    }),
+    // Resolve source maps to the original source
+    sourceMaps()
+  ]
+}

packages/puppeteer-extra-plugin-session-persistence/src/ambient.d.ts

@@ -0,0 +1,8 @@
+export {}
+
+// https://github.com/sindresorhus/type-fest/issues/19
+declare global {
+  interface SymbolConstructor {
+    readonly observable: symbol
+  }
+}

packages/puppeteer-extra-plugin-session-persistence/src/helpers.test.ts

@@ -0,0 +1,29 @@
+import test from 'ava';
+
+import { getDomainFromUrl, getBaseDomainFromUrl } from './helpers';
+
+test('getDomainFromUrl', (t) => {
+    t.is(getDomainFromUrl('https://www.example.com'), 'www.example.com');
+    t.is(getDomainFromUrl('http://www.example.com'), 'www.example.com');
+    t.is(getDomainFromUrl('https://example.com'), 'example.com');
+    t.is(getDomainFromUrl('http://example.com'), 'example.com');
+    t.is(getDomainFromUrl('https://www.example.co.uk'), 'www.example.co.uk');
+    t.is(getDomainFromUrl('http://www.example.co.uk'), 'www.example.co.uk');
+    t.is(getDomainFromUrl('https://example.co.uk'), 'example.co.uk');
+    t.is(getDomainFromUrl('http://example.co.uk'), 'example.co.uk');
+    t.is(getDomainFromUrl('https://subdomain.example.com'), 'subdomain.example.com');
+    t.is(getDomainFromUrl('http://subdomain.example.com'), 'subdomain.example.com');
+});
+
+test('getBaseDomainFromUrl', (t) => {
+    t.is(getBaseDomainFromUrl('https://www.example.com'), 'example.com');
+    t.is(getBaseDomainFromUrl('http://www.example.com'), 'example.com');
+    t.is(getBaseDomainFromUrl('https://example.com'), 'example.com');
+    t.is(getBaseDomainFromUrl('http://example.com'), 'example.com');
+    t.is(getBaseDomainFromUrl('https://www.example.co.uk'), 'example.co.uk');
+    t.is(getBaseDomainFromUrl('http://www.example.co.uk'), 'example.co.uk');
+    t.is(getBaseDomainFromUrl('https://example.co.uk'), 'example.co.uk');
+    t.is(getBaseDomainFromUrl('http://example.co.uk'), 'example.co.uk');
+    t.is(getBaseDomainFromUrl('https://subdomain.example.com'), 'example.com');
+    t.is(getBaseDomainFromUrl('http://subdomain.example.com'), 'example.com');
+});

packages/puppeteer-extra-plugin-session-persistence/src/helpers.ts

@@ -0,0 +1,24 @@
+import debug from "debug";
+
+const psl = require('psl');
+
+export function getDomainFromUrl(url: string): string {
+    try {
+        const parsedUrl = new URL(url);
+        return parsedUrl.hostname;
+    } catch (error) {
+        debug(`puppeteer-extra-plugin:session-persistence`).log('getDomainFromUrl() Error parsing url', url, error);
+        return '';
+    }
+}
+
+export function getBaseDomainFromUrl(url: string): string {
+    try {
+        const parsedUrl = new URL(url);
+        const parsedDomain = psl.parse(parsedUrl.hostname);
+        return parsedDomain.domain || '';
+    } catch (error) {
+        debug(`puppeteer-extra-plugin:session-persistence`).log('getBaseDomainFromUrl() Error parsing url', url, error);
+        return '';
+    }
+}