Support new concurrency mode: contextPerRenderKey

default.json

@@ -33,7 +33,9 @@
  "headed": false,
 
  "mode": "default",
+ "emulateNetworkConditions": false,
  "clustering": {
+ "monitor": false,
  "mode": "browser",
  "maxConcurrency": 5,
  "timeout": 30

dev.json

@@ -33,7 +33,16 @@
  "headed": false,
 
  "mode": "default",
+ "emulateNetworkConditions": false,
+ "networkConditions": {
+ "downloadThroughput": 4000000,
+ "uploadThroughput": 4000000,
+ "latency": 100,
+ "offline": false
+ },
+
  "clustering": {
+ "monitor": true,
  "mode": "browser",
  "maxConcurrency": 5,
  "timeout": 30

package.json

@@ -14,6 +14,7 @@
  "prettier:write": "prettier --list-different \"**/*.ts\" --write",
  "precommit": "npm run eslint & npm run typecheck",
  "watch": "tsc-watch --onSuccess \"node build/app.js server --config=dev.json\"",
+ "watch:debug": "tsc-watch --onSuccess \"cross-env DEBUG=puppeteer-cluster:* node build/app.js server --config=dev.json\"",
  "build": "tsc",
  "start": "node build/app.js --config=dev.json"
  },
@@ -30,8 +31,9 @@
  "morgan": "^1.9.0",
  "on-finished": "^2.3.0",
  "prom-client": "^11.5.3",
- "puppeteer": "^10.0.0",
+ "puppeteer": "13.1.3",
  "puppeteer-cluster": "^0.22.0",
+ "poolpeteer": "^0.22.0",
  "sharp": "0.29.3",
  "unique-filename": "^1.1.0",
  "winston": "^3.2.1"
@@ -40,6 +42,7 @@
  "@grafana/eslint-config": "^2.5.0",
  "@types/express": "^4.11.1",
  "@types/node": "^14.14.41",
+ "cross-env": "7.0.3",
  "@typescript-eslint/eslint-plugin": "^4.32.0",
  "@typescript-eslint/parser": "^4.32.0",
  "eslint": "^7.32.0",

src/browser/browser.ts

@@ -148,6 +148,11 @@ export class Browser {
  }
 
  async preparePage(page: puppeteer.Page, options: RenderOptions) {
+ if (this.config.emulateNetworkConditions && this.config.networkConditions) {
+ const client = await page.target().createCDPSession();
+ await client.send('Network.emulateNetworkConditions', this.config.networkConditions);
+ }
+
  if (options.renderKey) {
  if (this.config.verboseLogging) {
  this.log.debug('Setting cookie for page', 'renderKey', options.renderKey, 'domain', options.domain);

src/browser/clustered.ts

@@ -1,4 +1,5 @@
-import { Cluster } from 'puppeteer-cluster';
+import { Cluster as PoolpeteerCluster } from 'poolpeteer';
+import { Cluster as PuppeteerCluster } from 'puppeteer-cluster';
 import { ImageRenderOptions, RenderOptions } from '../types';
 import { Browser, RenderResponse, RenderCSVResponse, Metrics } from './browser';
 import { Logger } from '../logger';
@@ -10,12 +11,17 @@ enum RenderType {
 }
 
 interface ClusterOptions {
+ groupId?: string;
  options: RenderOptions | ImageRenderOptions;
  renderType: RenderType;
 }
 
 type ClusterResponse = RenderResponse | RenderCSVResponse;
 
+const contextPerRenderKey = 'contextPerRenderKey';
+
+type Cluster<JobData = any, ReturnData = any> = PuppeteerCluster<JobData, ReturnData> | PoolpeteerCluster<JobData, ReturnData>;
+
 export class ClusteredBrowser extends Browser {
  cluster: Cluster<ClusterOptions, ClusterResponse>;
  clusteringConfig: ClusteringConfig;
@@ -25,21 +31,46 @@ export class ClusteredBrowser extends Browser {
  super(config, log, metrics);
 
  this.clusteringConfig = config.clustering;
- this.concurrency = Cluster.CONCURRENCY_BROWSER;
+ this.concurrency = PuppeteerCluster.CONCURRENCY_BROWSER;
 
  if (this.clusteringConfig.mode === 'context') {
- this.concurrency = Cluster.CONCURRENCY_CONTEXT;
+ this.concurrency = PuppeteerCluster.CONCURRENCY_CONTEXT;
+ }
+
+ if (this.clusteringConfig.mode === contextPerRenderKey) {
+ this.concurrency = PoolpeteerCluster.CONCURRENCY_CONTEXT_PER_REQUEST_GROUP;
  }
  }
 
- async start(): Promise<void> {
+ shouldUsePoolpeteer(): boolean {
+ return this.clusteringConfig.mode === contextPerRenderKey;
+ }
+
+ async createCluster(): Promise<Cluster<ClusterOptions, ClusterResponse>> {
  const launcherOptions = this.getLauncherOptions({});
- this.cluster = await Cluster.launch({
+
+ const clusterOptions = {
  concurrency: this.concurrency,
+ workerShutdownTimeout: 5000,
+ monitor: this.clusteringConfig.monitor,
  maxConcurrency: this.clusteringConfig.maxConcurrency,
  timeout: this.clusteringConfig.timeout * 1000,
+ workerCreationDelay: 1000,
  puppeteerOptions: launcherOptions,
- });
+ };
+
+ // TODO use poolpeteer by default after initial release and testing (8.5?)
+ if (this.shouldUsePoolpeteer()) {
+ this.log.debug('Launching Browser cluster with poolpeteer');
+ return PoolpeteerCluster.launch(clusterOptions);
+ }
+
+ this.log.debug('Launching Browser cluster with puppeteer-cluster');
+ return PuppeteerCluster.launch(clusterOptions);
+ }
+
+ async start(): Promise<void> {
+ this.cluster = await this.createCluster();
  await this.cluster.task(async ({ page, data }) => {
  if (data.options.timezone) {
  // set timezone
@@ -61,13 +92,21 @@ export class ClusteredBrowser extends Browser {
  });
  }
 
+ private getGroupId = (options: ImageRenderOptions | RenderOptions) => {
+ if (this.clusteringConfig.mode === contextPerRenderKey) {
+ return `${options.domain}${options.renderKey}`;
+ }
+
+ return undefined;
+ };
+
  async render(options: ImageRenderOptions): Promise<RenderResponse> {
  this.validateImageOptions(options);
- return this.cluster.execute({ options, renderType: RenderType.PNG });
+ return this.cluster.execute({ groupId: this.getGroupId(options), options, renderType: RenderType.PNG });
  }
 
  async renderCSV(options: RenderOptions): Promise<RenderCSVResponse> {
  this.validateRenderOptions(options);
- return this.cluster.execute({ options, renderType: RenderType.CSV });
+ return this.cluster.execute({ groupId: this.getGroupId(options), options, renderType: RenderType.CSV });
  }
 }

src/config.ts

@@ -1,11 +1,20 @@
 import * as fs from 'fs';
 
 export interface ClusteringConfig {
+ monitor: boolean;
  mode: string;
  maxConcurrency: number;
  timeout: number;
 }
 
+// https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-emulateNetworkConditions
+type NetworkConditions = {
+ offline: boolean;
+ downloadThroughput: number;
+ uploadThroughput: number;
+ latency: number;
+};
+
 export interface RenderingConfig {
  chromeBin?: string;
  args: string[];
@@ -24,6 +33,8 @@ export interface RenderingConfig {
  dumpio: boolean;
  timingMetrics: boolean;
  headed?: boolean;
+ networkConditions?: NetworkConditions;
+ emulateNetworkConditions: boolean;
 }
 
 export interface MetricsConfig {
@@ -76,12 +87,14 @@ const defaultRenderingConfig: RenderingConfig = {
  maxWidth: 3000,
  maxHeight: 3000,
  maxDeviceScaleFactor: 4,
- mode: 'default',
+ mode: 'clustered',
  clustering: {
- mode: 'browser',
+ monitor: false,
+ mode: 'browserPerRenderKey',
  maxConcurrency: 5,
  timeout: 30,
  },
+ emulateNetworkConditions: false,
  verboseLogging: false,
  dumpio: false,
  timingMetrics: false,