diff --git a/packages/puppeteer-extra-plugin-session-persistence/ava.config-ts.js b/packages/puppeteer-extra-plugin-session-persistence/ava.config-ts.js new file mode 100644 index 00000000..a3bce5d6 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/ava.config-ts.js @@ -0,0 +1,9 @@ +export default { + compileEnhancements: false, + environmentVariables: { + TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}' + }, + files: ['src/**.test.ts', 'src/storage/**.test.ts'], + extensions: ['ts'], + require: ['ts-node/register'] +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/ava.config.js b/packages/puppeteer-extra-plugin-session-persistence/ava.config.js new file mode 100644 index 00000000..bfde4dbe --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/ava.config.js @@ -0,0 +1,3 @@ +export default { + files: ['dist/*.test.js'] +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/package.json b/packages/puppeteer-extra-plugin-session-persistence/package.json new file mode 100644 index 00000000..c6de3e7f --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/package.json @@ -0,0 +1,91 @@ +{ + "name": "puppeteer-extra-plugin-session-persistence", + "version": "1.0.2", + "description": "A puppeteer-extra plugin to persist sessions.", + "main": "dist/index.cjs.js", + "module": "dist/index.esm.js", + "typings": "dist/index.d.ts", + "files": [ + "dist" + ], + "repository": "berstend/puppeteer-extra", + "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-session-persistence#readme", + "author": "kangoo13", + "license": "MIT", + "scripts": { + "clean": "rimraf dist/*", + "tscheck": "tsc --pretty --noEmit", + "prebuild": "run-s clean", + "build": "run-s build:tsc build:rollup", + "build:tsc": "tsc --module commonjs", + "build:rollup": "rollup -c rollup.config.ts", + "docs": "node -e 0", + "test": "ava -v --config ava.config-ts.js", + "pretest-ci": "run-s build", + "test-ci-back": "ava --concurrency 1 --serial --fail-fast -v", + "test-ci": "exit 0" + }, + "engines": { + "node": ">=8" + }, + "prettier": { + "printWidth": 80, + "semi": false, + "singleQuote": true + }, + "keywords": [ + "puppeteer", + "puppeteer-extra", + "puppeteer-extra-plugin", + "sessions", + "cookies", + "localstorage", + "persistence" + ], + "devDependencies": { + "@types/debug": "^4.1.5", + "@types/node-fetch": "^2.5.4", + "@types/psl": "^1.1.0", + "@types/puppeteer": "*", + "ava": "^2.4.0", + "npm-run-all": "^4.1.5", + "puppeteer": "9", + "puppeteer-extra": "^3.3.6", + "rimraf": "^3.0.0", + "rollup": "^1.27.5", + "rollup-plugin-commonjs": "^10.1.0", + "rollup-plugin-node-resolve": "^5.2.0", + "rollup-plugin-sourcemaps": "^0.4.2", + "rollup-plugin-typescript2": "^0.25.2", + "ts-node": "^8.5.4", + "tslint": "^5.20.1", + "tslint-config-prettier": "^1.18.0", + "tslint-config-standard": "^9.0.0", + "typescript": "^4.3.3" + }, + "dependencies": { + "debug": "^4.1.1", + "devtools-protocol": "^0.0.1138159", + "node-fetch": "^2.6.0", + "psl": "^1.9.0", + "puppeteer-extra-plugin": "^3.2.3", + "redis": "^4.6.6" + }, + "peerDependencies": { + "puppeteer": "*", + "puppeteer-core": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "puppeteer": { + "optional": true + }, + "puppeteer-core": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + }, + "gitHead": "72fe830c158f1e971c8499fdd5232338dd53c220" +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/readme.md b/packages/puppeteer-extra-plugin-session-persistence/readme.md new file mode 100644 index 00000000..de9527ea --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/readme.md @@ -0,0 +1,77 @@ +# Puppeteer Extra Plugin Session Persistence + +This TypeScript library provides a Puppeteer Extra plugin for persisting sessions in Puppeteer. It saves and loads cookies and localStorage data from different storage engines. + +## Features + +- Save and load cookies and localStorage data +- Supports different storage engines (default: filesystem) +- Merge cookies and localStorage data from constructor and storage +- Automatically set cookies and localStorage data when a new page is created + +## Installation + +```bash +npm install puppeteer-extra-plugin-session-persistence +``` + +## Strategies + +The plugin supports different strategies for persisting session data, all activated by default: +- A polling strategy, update very X seconds the cookies from every page (default 1000 ms), very useful for XHR requests that sets cookies by JS +- On HTTP response, update the cookies from the response thanks to the 'set-cookie' header +- Using onFrameNavigated event, update the cookies and localStorage data from every frame + +## Usage + +```javascript +const puppeteer = require('puppeteer-extra') +puppeteer.use(require('puppeteer-extra-plugin-session-persistence')()) +// or +puppeteer.use(require('puppeteer-extra-plugin-session-persistence')({ + persistCookies: true, + persistLocalStorage: true, + storage: { + name: 'filesystem', + options: { + localStorageDataFile: './localStorageData.json', + cookiesFile: './cookies.json' + } + } +})) +const browser = await puppeteer.launch() +``` + +## Options + +- `persistCookies` (boolean, default: true): Allow or disallow cookies persistence. +- `persistLocalStorage` (boolean, default: true): Allow or disallow local storage persistence. +- `storage` (StorageConfig, default: filesystem storage): Storage options. +- `localStorageData` (LocalStorageData, default: {}): Local storage data to load. +- `cookies` (Cookie[], default: []): Cookies to load. + +## Storage Engines + +The plugin supports different storage engines for persisting session data: + +- File System Storage (default) +- Redis Storage +- In-Memory Storage + +To use a specific storage engine, pass the corresponding configuration object to the plugin constructor. For example, to use Redis storage: + +```javascript +puppeteer.use(require('puppeteer-extra-plugin-session-persistence')({ + storage: { + name: 'redis', + options: { + host: 'localhost', + port: 6379 + } + } +})) +``` + +## License + +This library is released under the MIT License. \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/rollup.config.ts b/packages/puppeteer-extra-plugin-session-persistence/rollup.config.ts new file mode 100644 index 00000000..83e63d7b --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/rollup.config.ts @@ -0,0 +1,65 @@ +import resolve from 'rollup-plugin-node-resolve' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' + +const pkg = require('./package.json') + +const entryFile = 'index' +const banner = ` +/*! + * ${pkg.name} v${pkg.version} by ${pkg.author} + * ${pkg.homepage || `https://github.com/${pkg.repository}`} + * @license ${pkg.license} + */ +`.trim() + +const defaultExportOutro = ` + module.exports = exports.default || {} + Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) +` + +export default { + input: `src/${entryFile}.ts`, + output: [ + { + file: pkg.main, + format: 'cjs', + sourcemap: true, + exports: 'named', + outro: defaultExportOutro, + banner + }, + { + file: pkg.module, + format: 'es', + sourcemap: true, + exports: 'named', + banner + } + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [ + ...Object.keys(pkg.dependencies || {}), + ...Object.keys(pkg.peerDependencies || {}), + 'fs', + 'os', + 'path' + ], + watch: { + include: 'src/**' + }, + plugins: [ + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + // commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + preferBuiltins: true + }), + // Resolve source maps to the original source + sourceMaps() + ] +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/ambient.d.ts b/packages/puppeteer-extra-plugin-session-persistence/src/ambient.d.ts new file mode 100644 index 00000000..7adc0f7d --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/ambient.d.ts @@ -0,0 +1,8 @@ +export {} + +// https://github.com/sindresorhus/type-fest/issues/19 +declare global { + interface SymbolConstructor { + readonly observable: symbol + } +} \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/helpers.test.ts b/packages/puppeteer-extra-plugin-session-persistence/src/helpers.test.ts new file mode 100644 index 00000000..23b7ec36 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/helpers.test.ts @@ -0,0 +1,29 @@ +import test from 'ava'; + +import { getDomainFromUrl, getBaseDomainFromUrl } from './helpers'; + +test('getDomainFromUrl', (t) => { + t.is(getDomainFromUrl('https://www.example.com'), 'www.example.com'); + t.is(getDomainFromUrl('http://www.example.com'), 'www.example.com'); + t.is(getDomainFromUrl('https://example.com'), 'example.com'); + t.is(getDomainFromUrl('http://example.com'), 'example.com'); + t.is(getDomainFromUrl('https://www.example.co.uk'), 'www.example.co.uk'); + t.is(getDomainFromUrl('http://www.example.co.uk'), 'www.example.co.uk'); + t.is(getDomainFromUrl('https://example.co.uk'), 'example.co.uk'); + t.is(getDomainFromUrl('http://example.co.uk'), 'example.co.uk'); + t.is(getDomainFromUrl('https://subdomain.example.com'), 'subdomain.example.com'); + t.is(getDomainFromUrl('http://subdomain.example.com'), 'subdomain.example.com'); +}); + +test('getBaseDomainFromUrl', (t) => { + t.is(getBaseDomainFromUrl('https://www.example.com'), 'example.com'); + t.is(getBaseDomainFromUrl('http://www.example.com'), 'example.com'); + t.is(getBaseDomainFromUrl('https://example.com'), 'example.com'); + t.is(getBaseDomainFromUrl('http://example.com'), 'example.com'); + t.is(getBaseDomainFromUrl('https://www.example.co.uk'), 'example.co.uk'); + t.is(getBaseDomainFromUrl('http://www.example.co.uk'), 'example.co.uk'); + t.is(getBaseDomainFromUrl('https://example.co.uk'), 'example.co.uk'); + t.is(getBaseDomainFromUrl('http://example.co.uk'), 'example.co.uk'); + t.is(getBaseDomainFromUrl('https://subdomain.example.com'), 'example.com'); + t.is(getBaseDomainFromUrl('http://subdomain.example.com'), 'example.com'); +}); diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/helpers.ts b/packages/puppeteer-extra-plugin-session-persistence/src/helpers.ts new file mode 100644 index 00000000..2eeb2d02 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/helpers.ts @@ -0,0 +1,24 @@ +import debug from "debug"; + +const psl = require('psl'); + +export function getDomainFromUrl(url: string): string { + try { + const parsedUrl = new URL(url); + return parsedUrl.hostname; + } catch (error) { + debug(`puppeteer-extra-plugin:session-persistence`).log('getDomainFromUrl() Error parsing url', url, error); + return ''; + } +} + +export function getBaseDomainFromUrl(url: string): string { + try { + const parsedUrl = new URL(url); + const parsedDomain = psl.parse(parsedUrl.hostname); + return parsedDomain.domain || ''; + } catch (error) { + debug(`puppeteer-extra-plugin:session-persistence`).log('getBaseDomainFromUrl() Error parsing url', url, error); + return ''; + } +} \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/index.ts b/packages/puppeteer-extra-plugin-session-persistence/src/index.ts new file mode 100644 index 00000000..3375dc21 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/index.ts @@ -0,0 +1,396 @@ +import {PluginData, PuppeteerExtraPlugin} from 'puppeteer-extra-plugin'; +import {Page, HTTPResponse} from 'puppeteer'; +import { + Storage, + PluginOptions, + LocalStorageData, createStorage, + Cookie, +} from './types'; +import {FileSystemStorage} from './storage/fileSystemStorage'; +import {Protocol} from 'devtools-protocol'; +import CookieSourceScheme = Protocol.Network.CookieSourceScheme; +import CookiePriority = Protocol.Network.CookiePriority; +import {getBaseDomainFromUrl, getDomainFromUrl} from './helpers'; +import CookieSameSite = Protocol.Network.CookieSameSite; + + +/** + * Persist sessions in puppeteer. This plugin will save/load the cookies and localStorage from different storage engines. + * + * @function + * @param {Object} opts - Options + * @param {boolean} [opts.persistCookies=true] - Allow or disallow cookies persistence. + * @param {boolean} [opts.persistLocalStorage=true] - Allow or disallow local storage persistence. + * @param {StorageConfig} [opts.storage] - Storage options. Default to filesystem storage. + * @param {LocalStorageData} [localStorageData={}] - Local storage data to load. + * @param {Cookie[]} [cookies=[]] - Cookies to load. + * @returns {Object} The puppeteer-extra-plugin-session-persistence instance. + * + * @example + * const puppeteer = require('puppeteer-extra') + * puppeteer.use(require('puppeteer-extra-plugin-session-persistence')()) + * // or + * puppeteer.use(require('puppeteer-extra-plugin-session-persistence')({ + * persistCookies: true, + * persistLocalStorage: true, + * storage: { + * name: 'redis', + * options: { + * host: 'localhost', + * port: 6379 + * } + * } + * })) + * const browser = await puppeteer.launch() + */ +export class PuppeteerExtraPluginSessionPersistence extends PuppeteerExtraPlugin { + private localStorageData: LocalStorageData = {}; + private cookies: Cookie[] = []; + private storage: Storage; + private domainCookiesTrigger: string[] = []; + private needUpdateCookies: boolean = false; + private pageList: Page[] = []; + private pollingInterval: NodeJS.Timeout | null = null; + + constructor(opts: Partial, localStorageData: LocalStorageData = {}, cookies: Cookie[] = []) { + super(opts) + this.storage = opts.storage ? createStorage(opts.storage) : new FileSystemStorage(); + this.localStorageData = localStorageData; + this.cookies = cookies; + this.debug('constructor', {opts: this.opts, localStorageData: this.localStorageData, cookies: this.cookies.map((c) => c.name)}); + } + + get name() { + return 'session-persistence' + } + + // todo: make a PR to puppeteer-extra to change the PluginData interface, name and value should not be an object but any instead. + get data(): PluginData[] { + return [ + { + name: { + name: 'cookies' + }, + value: { + value: this.cookies + } + }, + { + name: { + name: 'localStorageData' + }, + value: { + value: this.localStorageData + } + } + ] + } + + get defaults(): PluginOptions { + return { + persistCookiesEnabled: true, + persistLocalStorageEnabled: true, + cookiesPollingEnabled: true, + cookiesPollingInterval: 1000, + storage: { + name: "filesystem", + options: {}, + }, + } + } + + async onPluginRegistered() { + this.debug('onPluginRegistered'); + await this.loadCookies(); + await this.loadLocalStorageData(); + if (this.opts.cookiesPollingEnabled && this.opts.persistCookiesEnabled && !this.pollingInterval) { + this.debug('onPluginRegistered starting cookies polling'); + this.pollingInterval = setInterval(() => { + this.debug('setInterval polling cookies', {needUpdateCookies: this.needUpdateCookies}); + if (this.needUpdateCookies) { + this.pageList.forEach(async (page) => { + if (!page.isClosed()) { + try { + this.debug("Updating cookies for page (polling strategy)", page.url()); + await this.mergePageCookies(page); + } catch (error) { + this.debug('setInterval error with cookies, removing page from the list', {error}); + this.pageList = this.pageList.filter((p) => p !== page); + } + } else { + this.debug('setInterval page is closed, removing page from the list'); + this.pageList = this.pageList.filter((p) => p !== page); + } + }); + this.needUpdateCookies = false; + } + }, this.opts.cookiesPollingInterval || 1000); + } else { + this.debug('onPluginRegistered cookies polling disabled'); + } + this.debug('onPluginRegistered ended', {localStorageData: this.localStorageData}); + } + + async onClose() { + this.debug('onClose', {localStorageData: this.localStorageData}); + if (this.opts.persistCookiesEnabled) { + await this.saveCookies(); + } + if (this.opts.persistLocalStorageEnabled) { + await this.saveLocalStorageData(); + } + if (this.pollingInterval) { + this.debug('onClose clearing cookies polling interval'); + clearInterval(this.pollingInterval); + } + } + + // onPageCreated create all the event listeners for the page. + async onPageCreated(page: Page) { + this.debug('onPageCreated adding event listeners'); + if (this.opts.cookiesPollingEnabled && this.opts.persistCookiesEnabled) { + this.pageList.push(page); + } + await this.setPageCookies(page); + await page.setBypassCSP(true); + page.on('framenavigated', () => this.onFrameNavigated(page)); + page.on('response', this.onResponseReceived.bind(this)); + + } + + // loadLocalStorageData loads the localStorage data from the localStorageData file, if the file does not exist, it will try to create it. + // It also merges the localStorageData from the file with the localStorageData from the constructor if any localStorageData are given. + loadLocalStorageData() { + this.debug('loadLocalStorageData'); + this.localStorageData = {...this.storage.loadLocalStorageData(), ...this.localStorageData}; + } + + + // saveLocalStorageData saves the localStorage data to the localStorageData file. + async saveLocalStorageData() { + this.debug('saveLocalStorageData'); + await this.storage.saveLocalStorageData(this.localStorageData); + } + + // loadCookies loads the cookies from the cookies file, if the file does not exist, it will try to create it. + // It also merges the cookies from the file with the cookies from the constructor if any cookies are given. + async loadCookies() { + this.debug('loadCookies'); + try { + const cookies = await this.storage.loadCookies(); + this.mergeCookies(cookies); + this.debug('loadCookies loaded', {cookies: this.cookies.map((c) => c.name)}); + } catch (err) { + this.debug('loadCookies ended with error', {err}); + await this.saveCookies(); + } + } + + // saveCookies saves the cookies to the cookies file. + async saveCookies() { + this.debug('saveCookies'); + await this.storage.saveCookies(this.cookies); + } + + async setPageCookies(page: Page) { + this.debug('setPageCookies', this.cookies.map((c) => c.name)); + const pageTarget = page.target(); + const client = await pageTarget.createCDPSession(); + const rtValue = await client.send('Network.setCookies', {cookies: this.cookies}); + this.debug('setPageCookies ended', rtValue); + } + + async extractCookiesFromResponse(cookies: string, url: string): Promise { + this.debug('extractCookiesFromResponse', cookies.toString()); + + const cookiesArray = cookies.split(/,(?=\s\S+=\S+)/); + + return cookiesArray.map((cookie: string) => { + const cookieParts = cookie.split(';'); + const parsedCookie = cookieParts[0].split('='); + const domain = url.split('/')[2]; + + let expires = 0; + let httpOnly = false; + let secure = false; + let session = false; + let path = '/'; + let sameSite: CookieSameSite = 'Lax'; + let sameParty = false; + let sourceScheme: CookieSourceScheme = 'Secure'; + let sourcePort = 443; + let priority: CookiePriority = 'Low'; + + cookieParts.slice(1).forEach((part) => { + const [key, value] = part.trim().split('='); + + switch (key.toLowerCase()) { + case 'expires': + expires = value ? new Date(value).getTime() : 0; + break; + case 'path': + path = value; + break; + case 'httponly': + httpOnly = true; + break; + case 'secure': + secure = true; + break; + case 'samesite': + sameSite = value as 'Lax' | 'Strict' | 'None'; + break; + case 'sameparty': + sameParty = true; + break; + case 'sourcescheme': + sourceScheme = value as CookieSourceScheme || 'Secure'; + break; + case 'sourceport': + sourcePort = parseInt(value, 10) || 443; + break; + case 'priority': + priority = value as CookiePriority || 'Low'; + break; + } + }); + + const newCookie: Cookie = { + name: parsedCookie[0].trim(), + value: parsedCookie[1], + domain: domain, + path: path, + expires: expires, + size: cookie.length, + httpOnly: httpOnly, + secure: secure, + session: session, + sameSite: sameSite, + sameParty: sameParty, + sourceScheme: sourceScheme as CookieSourceScheme, + sourcePort: sourcePort, + priority: priority as CookiePriority, + }; + + return newCookie; + }); + }; + + async onResponseReceived(response: HTTPResponse) { + this.debug('onResponseReceived', {response}); + const headers = response.headers(); + const cookies = headers['set-cookie']; + if (cookies) { + const parsedCookies = await this.extractCookiesFromResponse(cookies, response.url()); + await this.mergeCookies(parsedCookies); + } + if (this.domainCookiesTrigger.includes(getBaseDomainFromUrl(response.url()))) { + this.needUpdateCookies = true; + } + } + + // mergeCookies merges the cookies we have with incoming cookies. + // the farthest cookies with expiration date will remain if there are duplicates. + mergeCookies(cookies: Cookie[]) { + const cookieMap = new Map(); + + for (const cookie of this.cookies) { + const key = `${cookie.domain}-${cookie.name}`; + cookieMap.set(key, cookie); + } + + for (const cookie of cookies) { + const key = `${cookie.domain}-${cookie.name}`; + const existingCookie = cookieMap.get(key); + + if (existingCookie) { + const existingExpires = new Date(existingCookie.expires || 0); + const newExpires = new Date(cookie.expires || 0); + + if (newExpires > existingExpires) { + cookieMap.set(key, cookie); + } + } else { + cookieMap.set(key, cookie); + } + } + + this.cookies = Array.from(cookieMap.values()); + } + + async onFrameNavigated(page: Page) { + this.debug('onFrameNavigated'); + const domainUrl = getDomainFromUrl(page.url()); + const baseDomainUrl = getBaseDomainFromUrl(page.url()); + if (!this.domainCookiesTrigger.includes(baseDomainUrl)) { + this.domainCookiesTrigger.push(baseDomainUrl); + } + + try { + await this.setLocalStorageValues(page, domainUrl); + await this.updateLocalStorageData(page, domainUrl); + } catch (error) { + this.debug('onFrameNavigated error with localStorage', {error}); + } + + try { + await this.mergePageCookies(page); + } catch (error) { + this.debug('onFrameNavigated error with cookies', {error}); + } + } + + async setLocalStorageValues(page: Page, domainUrl: string) { + for (const key in this.localStorageData[domainUrl]) { + if (this.localStorageData[domainUrl].hasOwnProperty(key)) { + const value = this.localStorageData[domainUrl][key]; + this.debug('setLocalStorageValues', {key, value}); + await page.evaluate((key: string, value: string) => { + localStorage.setItem(key, value); + }, key, value); + } + } + } + + async updateLocalStorageData(page: Page, domainUrl: string) { + const localStorageAccessible = await page.evaluate(() => { + try { + return !!window.localStorage; + } catch (error) { + return false; + } + }); + + if (localStorageAccessible) { + this.debug('onFrameNavigated localStorage accessible'); + const localStorageData = await page.evaluate(() => { + const data: { [key: string]: string | null } = {}; + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i); + if (key) { + data[key] = localStorage.getItem(key); + } + } + return data; + }); + this.localStorageData[domainUrl] = {...this.localStorageData[domainUrl], ...localStorageData}; + } else { + this.debug('onFrameNavigated localStorage not accessible'); + } + } + + async mergePageCookies(page: Page) { + const pageTarget = page.target(); + const client = await pageTarget.createCDPSession(); + const cookies = await client.send('Network.getAllCookies'); + this.debug('onRequestFinished merging cookies'); + this.mergeCookies(cookies.cookies); + } + +} + +const defaultExport = (options?: Partial, localStorageData: LocalStorageData = {}, cookies: Cookie[] = []) => { + return new PuppeteerExtraPluginSessionPersistence(options ?? {}, localStorageData, cookies) +} + +export default defaultExport \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.test.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.test.ts new file mode 100644 index 00000000..d9b595cb --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.test.ts @@ -0,0 +1,65 @@ +import test from 'ava'; +import fs from 'fs'; +import { Protocol } from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; +import { FileSystemStorage } from './fileSystemStorage'; + +const localStorageDataFilePath = './testLocalStorageData.json'; +const cookiesFilePath = './testCookies.json'; + +test.afterEach(() => { + // Clean up files after each test + if (fs.existsSync(localStorageDataFilePath)) { + fs.unlinkSync(localStorageDataFilePath); + } + if (fs.existsSync(cookiesFilePath)) { + fs.unlinkSync(cookiesFilePath); + } +}); + +test('FileSystemStorage: load and save LocalStorageData', async t => { + const fileSystemStorage = new FileSystemStorage({ + localStorageDataFile: localStorageDataFilePath + }); + + const localStorageData = { + 'example.com': { + key1: 'value1', + key2: 'value2' + } + }; + + await fileSystemStorage.saveLocalStorageData(localStorageData); + const loadedData = await fileSystemStorage.loadLocalStorageData(); + + t.deepEqual(loadedData, localStorageData); +}); + +test('FileSystemStorage: load and save Cookies', async t => { + const fileSystemStorage = new FileSystemStorage({ + cookiesFile: cookiesFilePath + }); + + const cookies: Cookie[] = [ + { + name: 'cookie1', + value: 'value1', + domain: 'example.com', + path: '/', + expires: 1628770800, + size: 10, + httpOnly: false, + secure: false, + session: false, + priority: 'Medium', + sameParty: false, + sourceScheme: 'Secure', + sourcePort: 443, + }, + ]; + + await fileSystemStorage.saveCookies(cookies); + const loadedCookies = await fileSystemStorage.loadCookies(); + + t.deepEqual(loadedCookies, cookies); +}); diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.ts new file mode 100644 index 00000000..c5655cb3 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/fileSystemStorage.ts @@ -0,0 +1,49 @@ +import {FileSystemStorageOptions, LocalStorageData, Storage} from "../types"; +import fs from "fs"; +import {Protocol} from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; + + +export class FileSystemStorage implements Storage { + private localStorageDataFile: string; + private cookiesFile: string; + + get name() { + return 'filesystem'; + } + + constructor(fileSystemStorageOptions: FileSystemStorageOptions = {}) { + this.localStorageDataFile = fileSystemStorageOptions.localStorageDataFile || './localStorageData.json'; + this.cookiesFile = fileSystemStorageOptions.cookiesFile || './cookies.json'; + } + + async loadLocalStorageData(): Promise { + try { + const localStorageDataLoaded = fs.readFileSync(this.localStorageDataFile); + return JSON.parse(localStorageDataLoaded.toString()); + } catch (err) { + await this.saveLocalStorageData({}); + } + + return {}; + } + + async saveLocalStorageData(data: LocalStorageData): Promise { + fs.writeFileSync(this.localStorageDataFile, JSON.stringify(data)); + } + + async loadCookies(): Promise { + try { + const cookiesLoaded = fs.readFileSync(this.cookiesFile); + return JSON.parse(cookiesLoaded.toString()); + } catch (err) { + await this.saveCookies([]); + } + + return []; + } + + async saveCookies(cookies: Cookie[]): Promise { + fs.writeFileSync(this.cookiesFile, JSON.stringify(cookies)); + } +} \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.test.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.test.ts new file mode 100644 index 00000000..f002c193 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.test.ts @@ -0,0 +1,47 @@ +import test from 'ava'; +import { Protocol } from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; +import { InMemoryStorage } from './inMemoryStorage'; + +test('InMemoryStorage: load and save LocalStorageData', async t => { + const inMemoryStorage = new InMemoryStorage(); + + const localStorageData = { + 'example.com': { + key1: 'value1', + key2: 'value2' + } + }; + + await inMemoryStorage.saveLocalStorageData(localStorageData); + const loadedData = await inMemoryStorage.loadLocalStorageData(); + + t.deepEqual(loadedData, localStorageData); +}); + +test('InMemoryStorage: load and save Cookies', async t => { + const inMemoryStorage = new InMemoryStorage(); + + const cookies: Cookie[] = [ + { + name: 'cookie1', + value: 'value1', + domain: 'example.com', + path: '/', + expires: 1628770800, + size: 10, + httpOnly: false, + secure: false, + session: false, + priority: 'Medium', + sameParty: false, + sourceScheme: 'Secure', + sourcePort: 443, + }, + ]; + + await inMemoryStorage.saveCookies(cookies); + const loadedCookies = await inMemoryStorage.loadCookies(); + + t.deepEqual(loadedCookies, cookies); +}); diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.ts new file mode 100644 index 00000000..2b2cbaf9 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/inMemoryStorage.ts @@ -0,0 +1,36 @@ +import {LocalStorageData, Storage} from "../types"; +import {Protocol} from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; + +export class InMemoryStorage implements Storage { + private localStorageData: Map = new Map(); + private cookies: Cookie[] = []; + + get name() { + return 'in-memory'; + } + + async loadLocalStorageData(): Promise { + const result: { [domain: string]: { [key: string]: any } } = {}; + + this.localStorageData.forEach((value, key) => { + result[key] = {...value}; + }); + + return result; + } + + async saveLocalStorageData(data: LocalStorageData): Promise { + for (const domain in data) { + this.localStorageData.set(domain, {...data[domain]}); + } + } + + async loadCookies(): Promise { + return [...this.cookies]; + } + + async saveCookies(cookies: Cookie[]): Promise { + this.cookies = [...cookies]; + } +} \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.test.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.test.ts new file mode 100644 index 00000000..a7979b38 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.test.ts @@ -0,0 +1,74 @@ +import test from 'ava'; +import { Protocol } from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; +import { RedisStorage } from './redisStorage'; + +// Create a factory function to generate the mockRedisClient with a custom get method and a temporary storage +const createMockRedisClient = () => { + let tempStorage: Record = {}; + + return { + get: async (key: string) => { + return tempStorage[key]; + }, + set: async (key: string, value: string) => { + tempStorage[key] = value; + return 'OK'; + }, + // Method to access the temporary storage for testing purposes + getTempStorage: () => tempStorage, + flushTempStorage: () => { + tempStorage = {}; + } + }; +}; + +const mockRedisClient = createMockRedisClient(); + +test.afterEach(() => { + // Clean up temporary storage after each test + mockRedisClient.flushTempStorage(); +}); + +test('RedisStorage: load and save LocalStorageData', async t => { + const redisStorage = new RedisStorage({}, mockRedisClient); + + const localStorageData = { + 'example.com': { + key1: 'value1', + key2: 'value2' + } + }; + + await redisStorage.saveLocalStorageData(localStorageData); + const loadedData = await redisStorage.loadLocalStorageData(); + + t.deepEqual(loadedData, localStorageData); +}); + +test('RedisStorage: load and save Cookies', async t => { + const redisStorage = new RedisStorage({}, mockRedisClient); + + const cookies: Cookie[] = [ + { + name: 'cookie1', + value: 'value1', + domain: 'example.com', + path: '/', + expires: 1628770800, + size: 10, + httpOnly: false, + secure: false, + session: false, + priority: 'Medium', + sameParty: false, + sourceScheme: 'Secure', + sourcePort: 443, + }, + ]; + + await redisStorage.saveCookies(cookies); + const loadedCookies = await redisStorage.loadCookies(); + + t.deepEqual(loadedCookies, cookies); +}); diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.ts b/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.ts new file mode 100644 index 00000000..a1f50f4a --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/storage/redisStorage.ts @@ -0,0 +1,46 @@ +import {LocalStorageData, RedisClientTypeCustom, RedisStorageOptions, Storage} from "../types"; +import {createClient} from "redis"; +import {Protocol} from 'devtools-protocol'; +import Cookie = Protocol.Network.Cookie; + +export class RedisStorage implements Storage { + private client: RedisClientTypeCustom; + + constructor(options: RedisStorageOptions = {}, client?: RedisClientTypeCustom) { + this.client = client || createClient(options); + } + + get name() { + return 'redis'; + } + + async loadLocalStorageData(): Promise { + let localStorageData: LocalStorageData = {}; + + const localStorageDataStr = await this.client.get('localStorageData'); + if (!localStorageDataStr) { + return localStorageData; + } + + return JSON.parse(localStorageDataStr) as LocalStorageData; + } + + async saveLocalStorageData(data: LocalStorageData): Promise { + await this.client.set('localStorageData', JSON.stringify(data)); + } + + async loadCookies(): Promise { + let cookies: Cookie[] = []; + + const cookiesStr = await this.client.get('cookies'); + if (!cookiesStr) { + return cookies; + } + + return JSON.parse(cookiesStr) as Cookie[]; + } + + async saveCookies(cookies: Cookie[]): Promise { + await this.client.set('cookies', JSON.stringify(cookies)); + } +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/types.test.ts b/packages/puppeteer-extra-plugin-session-persistence/src/types.test.ts new file mode 100644 index 00000000..fa076045 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/types.test.ts @@ -0,0 +1,55 @@ +import test from 'ava'; +import { createStorage, StorageConfig } from './types'; +import { RedisStorage } from './storage/redisStorage'; +import { FileSystemStorage } from './storage/fileSystemStorage'; +import { InMemoryStorage } from './storage/inMemoryStorage'; + +test('createStorage should create RedisStorage instance', (t) => { + const config: StorageConfig = { + name: RedisStorage.name.toLowerCase(), + options: { + host: 'localhost', + port: 6379, + password: 'password', + }, + }; + + const storage = createStorage(config); + t.true(storage instanceof RedisStorage); +}); + +test('createStorage should create FileSystemStorage instance', (t) => { + const config: StorageConfig = { + name: FileSystemStorage.name.toLowerCase(), + options: { + localStorageDataFile: 'localStorageData.json', + cookiesFile: 'cookies.json', + }, + }; + + const storage = createStorage(config); + t.true(storage instanceof FileSystemStorage); +}); + +test('createStorage should create InMemoryStorage instance', (t) => { + const config: StorageConfig = { + name: InMemoryStorage.name.toLowerCase(), + options: undefined, + }; + + const storage = createStorage(config); + t.true(storage instanceof InMemoryStorage); +}); + +test('createStorage should throw an error for unknown storage name', (t) => { + const config: StorageConfig = { + name: 'unknown', + options: undefined, + }; + + const error = t.throws(() => { + createStorage(config); + }); + + t.is(error.message, 'Unknown storage name: unknown'); +}); diff --git a/packages/puppeteer-extra-plugin-session-persistence/src/types.ts b/packages/puppeteer-extra-plugin-session-persistence/src/types.ts new file mode 100644 index 00000000..127f57a9 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/src/types.ts @@ -0,0 +1,78 @@ +import {Protocol} from 'devtools-protocol'; +import NetworkCookie = Protocol.Network.Cookie; +import {SetOptions} from "redis"; +import {FileSystemStorage} from './storage/fileSystemStorage'; +import {RedisStorage} from './storage/redisStorage'; +import {InMemoryStorage} from './storage/inMemoryStorage'; + +export interface Storage { + loadLocalStorageData(): Promise; + + saveLocalStorageData(data: LocalStorageData): Promise; + + loadCookies(): Promise; + + saveCookies(cookies: Cookie[]): Promise; + + name: string; +} + +export interface RedisClientTypeCustom { + get: (key: string) => Promise; + set: (key: string, value: string, options?: SetOptions) => Promise; +} + +export interface RedisStorageOptions { + host?: string; + port?: number; + password?: string; +} + +export interface FileSystemStorageOptions { + localStorageDataFile?: string; + cookiesFile?: string; +} + +export type RedisStorageConfig = { + name: RedisStorage['name']; + options: RedisStorageOptions; +}; + +export type FileSystemStorageConfig = { + name: FileSystemStorage["name"]; + options: FileSystemStorageOptions; +}; + +export type InMemoryStorageConfig = { + name: InMemoryStorage["name"]; + options: undefined; +}; + +export interface PluginOptions { + persistCookiesEnabled?: boolean; + persistLocalStorageEnabled?: boolean; + storage: StorageConfig; + cookiesPollingEnabled?: boolean; + cookiesPollingInterval?: number; +} + +export interface LocalStorageData { + [domain: string]: { [key: string]: any }; +} + +export type StorageConfig = RedisStorageConfig | FileSystemStorageConfig | InMemoryStorageConfig; + +export function createStorage(opts: StorageConfig) { + switch (opts.name) { + case RedisStorage.name.toLowerCase(): + return new RedisStorage(opts.options as RedisStorageOptions); + case FileSystemStorage.name.toLowerCase(): + return new FileSystemStorage(opts.options as FileSystemStorageOptions); + case InMemoryStorage.name.toLowerCase(): + return new InMemoryStorage(); + default: + throw new Error(`Unknown storage name: ${opts.name}`); + } +} + +export type Cookie = NetworkCookie; \ No newline at end of file diff --git a/packages/puppeteer-extra-plugin-session-persistence/tsconfig.json b/packages/puppeteer-extra-plugin-session-persistence/tsconfig.json new file mode 100644 index 00000000..82e4389a --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "outDir": "./dist", + "target": "es2017", + "module": "es2015", + "moduleResolution": "node", + "typeRoots": ["./node_modules/@types", "./src/types"], + "lib": ["es2015", "es2016", "es2017", "dom"], + "sourceMap": true, + "declaration": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": false, + "noUnusedLocals": true, + "noUnusedParameters": false, + "pretty": true, + "stripInternal": true, + "types": ["node"] + }, + "include": [ + "./src/**/*.tsx", + "./src/**/*.ts", + "./src/**/*.test.ts", + "./test/**/*.ts" + ], + "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] +} diff --git a/packages/puppeteer-extra-plugin-session-persistence/tslint.json b/packages/puppeteer-extra-plugin-session-persistence/tslint.json new file mode 100644 index 00000000..c664ff25 --- /dev/null +++ b/packages/puppeteer-extra-plugin-session-persistence/tslint.json @@ -0,0 +1,6 @@ +{ + "extends": ["tslint-config-standard", "tslint-config-prettier"], + "rules": { + "ordered-imports": true + } +}