- const npmPackage = require('./package.json');
-const crypto = require('crypto');
-const express = require('express');
-const proxy = require('express-http-proxy');
-const cors = require('cors');
-const bodyParser = require('body-parser');
-const urlUtil = require('url');
-const chalk = require('chalk');
-
-/** The caching proxy server. */
-class JsonCachingProxy {
- /**
- * @param {Object} options - Options passed into the ctor will override defaults if defined
- */
- constructor(options = {}) {
- this.defaultOptions = {
- remoteServerUrl: 'http://localhost:8080',
- proxyPort: 3001,
- harObject: null,
- commandPrefix: 'proxy',
- proxyHeaderIdentifier: 'caching-proxy-playback',
- middlewareList: [],
- excludedRouteMatchers: [],
- cacheBustingParams: [],
- cacheEverything: false,
- dataPlayback: true,
- dataRecord: true,
- showConsoleOutput: false,
- proxyTimeout: 3600000, // one hour
- deleteCookieDomain: false, // Removes the domain portion from all cookies
- overrideCors: false,
- useCorsCredentials: false
- };
-
- // Ignore undefined values and combine the options with defaults
- this.options = Object.assign({},
- this.defaultOptions,
- Object.keys(options).reduce(function (passedOpts, key) {
- if (typeof options[key] !== 'undefined' && options[key] !== null) passedOpts[key] = options[key];
- return passedOpts;
- }, {}));
-
- // Will be set when the app starts
- this.server = null;
-
- this.app = express();
- this.routeCache = {};
-
- this.excludedParamMap = this.options.cacheBustingParams.reduce((map, param) => { map[param] = true; return map }, {});
-
- if (this.options.overrideCors) {
- this.app.use(cors({credentials: this.options.useCorsCredentials, origin: this.options.overrideCors}));
- this.app.options('*', cors({credentials: this.options.useCorsCredentials, origin: this.options.overrideCors}));
- }
-
- if (this.options.showConsoleOutput) {
- this.log = console.log;
- this.err = console.error;
- } else {
- this.log = () => false;
- this.err = () => false;
- }
- }
-
- /**
- * Remove the domain portion of any cookies from the object. Remove the secure attribute so we can set cookies to https targets
- * @param {Object} cookies - Express cookies array
- * @returns {Object[]} - Cookies with domain portion removed
- */
- removeCookiesDomain(cookies) {
- return cookies.map(c => {
- let cookieParts = c.split(';');
- let newCookieParts = [];
-
- cookieParts.forEach(c => {
- if (c.indexOf('domain') === -1 && c.indexOf('secure') === -1) {
- newCookieParts.push(c);
- }
- });
-
- return newCookieParts.join(';');
- });
- }
-
- /**
- * Returns an Object's own properties into an array of name-value pair objects
- * @param {Object} obj
- * @returns {Object[]}
- */
- convertToNameValueList(obj) {
- return typeof obj === 'object' ? Object.keys(obj).map(key => { return { name: key, value: obj[key] }; }) : [];
- }
-
- /**
- * Generate a unique hash key from a har file entry's request object: TODO: Include headers?
- * @param {Object} harEntryReq - HAR request object
- * @returns {Object} A unique key, hash tuple that identifies the request
- */
- genKeyFromHarReq(harEntryReq) {
- let { method, url, queryString = [], postData = { text: '' } } = harEntryReq;
- let uri = urlUtil.parse(url).pathname;
- let postParams = postData.text;
-
- queryString = queryString.filter(param => !this.excludedParamMap[param.name]);
-
- let plainText = `${method} ${uri} ${JSON.stringify(queryString)} ${postParams}`;
- let hash = crypto.createHash('md5').update(plainText).digest("hex");
- let key = `${method} ${uri} ${hash}`;
-
- return { key, hash };
- }
-
- /**
- * Takes a generic express request and convert it into a HAR request so that a unique key can be generated
- * @param {Object} req - An express IncomingMessage request
- * @returns {string} A unique hash key that identifies the request
- */
- genKeyFromExpressReq(req) {
- return this.genKeyFromHarReq({
- method: req.method,
- url: req.url,
- queryString: this.convertToNameValueList(req.query),
- postData: { text: req.body && req.body.length > 0 ? req.body.toString('utf8') : '' }
- });
- }
-
- /**
- * Build a HAR entry object from an express Request and response
- * @param {string} startedDateTime - An ISO Datetime String
- * @param {Object} req - An express IncomingMessage request
- * @param {Object} res - An express ServerResponse response
- * @param {Object} data - An express response body (the content)
- * @returns {Object} A HAR entry object
- */
- createHarEntry(startedDateTime, req, res, data) {
- let reqMimeType = req.get('Content-Type');
- let resMimeType = res.get('Content-Type') || 'text/plain';
- let encoding = (/^text\/|^application\/(javascript|json)/).test(resMimeType) ? 'utf8' : 'base64';
-
- let entry = {
- request: {
- startedDateTime: startedDateTime,
- method: req.method.toUpperCase(),
- url: req.url,
- cookies: this.convertToNameValueList(req.cookies),
- headers: this.convertToNameValueList(req.headers),
- queryString: this.convertToNameValueList(req.query),
- headersSize: -1,
- bodySize: -1
- },
- response: {
- status: res.statusCode,
- statusText: res.statusMessage,
- cookies: this.convertToNameValueList(res.cookies),
- headers: this.convertToNameValueList(res._headers).filter(header => header.name.toLowerCase() !== 'content-encoding'), // Not compressed
- content: {
- size: -1,
- mimeType: resMimeType,
- text: data.toString(encoding),
- encoding: encoding
- },
- headersSize: -1,
- bodySize: -1
- }
- };
-
- // Add the request body if it exists
- if (req.body && req.body.length > 0) {
- entry.request.postData = {
- mimeType: reqMimeType,
- text: req.body.toString(encoding)
- };
- }
-
- return entry;
- }
-
- /**
- * Check to see if the pieces of a request are excluded. This checks only the method and the uri. It uses the list
- * of regExp matchers to test
- * @param {string} method - e.g. GET, POST, PUT, etc.
- * @param {string} uri - e.g. http://www.api.com/rest/accounts
- * @returns {boolean} Whether the test is true for some matcher
- */
- isRouteExcluded(method, uri) {
- return this.options.excludedRouteMatchers.some(regExp => regExp.test(`${method} ${uri}`))
- }
-
- /**
- * Add express routes for each entry in a harObject. The harObject would have been read in from a har file at some point
- * @param {Object} harObject - A standard HAR file object that contains a collection of entries
- * @returns {JsonCachingProxy}
- */
- addHarEntriesToCache(harObject) {
- if (harObject) {
- harObject.log.entries.forEach(entry => {
- let { key, hash } = this.genKeyFromHarReq(entry.request);
-
- if (this.isRouteExcluded(entry.request.method, entry.request.url)) {
- this.log(chalk.red('Excluded from Cache', chalk.bold(entry.request.method, entry.request.url)));
- return;
- }
-
- // Only cache things that have content. Some times HAR files generated elsewhere will be missing this parameter
- if (entry.response.content.text) {
- let mimeType = entry.response.content.mimeType;
-
- if (entry.response.headers && (this.options.cacheEverything || !this.options.cacheEverything && mimeType && mimeType.indexOf('application/json') >= 0)) {
- // Remove content-encoding. gzip compression won't be used
- entry.response.headers = entry.response.headers.filter(header => header.name.toLowerCase() !== 'content-encoding');
- this.routeCache[key] = entry;
-
- this.log(chalk.yellow('Saved to Cache', hash, chalk.bold(entry.request.method, entry.request.url)));
- }
- }
- });
- }
-
- return this;
- }
-
- /**
- * Add the admin express routes for controlling the proxy server through a browser. Allows one to make GET requests to clear
- * the cache, disable/enable playback/recording, and generate a har file of the cache to download for later use.
- * @returns {JsonCachingProxy}
- */
- addAdminRoutes() {
- // These are not really restful because the GET is changing state. But it's easier to use in a browser
- this.app.get(`/${this.options.commandPrefix}/playback`, (req, res) => {
- this.options.dataPlayback = typeof req.query.enabled !== 'undefined' ? req.query.enabled === 'true'.toLowerCase() : this.options.dataPlayback;
- this.log(chalk.blue(`Replay from cache: ${this.options.dataPlayback}`));
- res.send(`Replay from cache: ${this.options.dataPlayback}`);
- });
-
- this.app.get(`/${this.options.commandPrefix}/record`, (req, res) => {
- this.options.dataRecord = typeof req.query.enabled !== 'undefined' ? req.query.enabled === 'true'.toLowerCase() : this.options.dataRecord;
- this.log(chalk.blue('Saving to cache: ' + this.options.dataRecord));
- res.send(`Saving to cache: ${this.options.dataRecord}`);
- });
-
- this.app.get(`/${this.options.commandPrefix}/clear`, (req, res) => {
- this.routeCache = {};
- this.log(chalk.blue('Cleared cache'));
- res.send('Cleared cache');
- });
-
- this.app.get(`/${this.options.commandPrefix}/har`, (req, res) => {
- this.log(chalk.blue('Generating har file'));
-
- let har = {
- log: {
- version: "1.2",
- creator: {
- name: npmPackage.name,
- version: npmPackage.version
- },
- entries: []
- }
- };
-
- Object.keys(this.routeCache).forEach(key => har.log.entries.push(this.routeCache[key]));
-
- res.setHeader('Content-disposition', 'attachment; filename=json-caching-proxy.har');
- res.setHeader('Content-type', 'application/json');
- res.json(har);
- });
-
- return this;
- }
-
- /**
- * Add user supplied middleware routes to express in order to handle special cases (browser-sync middleware options)
- * @param {Object[]} middlewareList - A list of route/handler pairs
- * @returns {JsonCachingProxy}
- */
- addMiddleWareRoutes(middlewareList) {
- middlewareList.forEach(mw => {
- if (mw.route) {
- this.app.use(mw.route, mw.handle);
- } else {
- this.app.use(mw.handle);
- }
- });
-
- return this;
- }
-
- /**
- * Add Request body parsing into RAW if there is actual body content
- * @returns {JsonCachingProxy}
- */
- addBodyParser() {
- this.app.use(bodyParser.raw({ type: '*/*', limit: '100mb' }));
-
- // Remove the body if there is no body content. Some sites check for malformed requests
- this.app.use((req, res, next) => {
- if (!req.headers['content-length']) {
- delete req.body;
- }
-
- next();
- });
-
- return this;
- }
-
- /**
- * An express route that reads from the cache if possible for any routes persisted in cache memory
- * @returns {JsonCachingProxy}
- */
- addCachingRoute() {
- this.app.use('/', (req, res, next) => {
- if (!this.options.dataPlayback) {
- next();
- } else {
- let { key, hash } = this.genKeyFromExpressReq(req);
- let entry = this.routeCache[key];
-
- if (!(entry && entry.response && entry.response.content)) {
- next();
- } else {
- let response = entry.response;
- let headerList = response.headers || [];
- let text = response.content.text || '';
- let encoding = response.content.encoding || 'utf8';
-
- headerList.forEach(function (header) {
- res.set(header.name, header.value);
- });
-
- // Use our header identifier to signal that this response is cached
- res.set(this.options.proxyHeaderIdentifier, true);
-
- if (text.length === 0) {
- res.sendStatus(response.status);
- } else {
- if (encoding === 'base64') {
- let bin = new Buffer(text, 'base64');
- res.writeHead(response.status, {
- 'Content-Type': response.content.mimeType,
- 'Content-Length': bin.length
- });
- res.end(bin);
- } else {
- res.status(response.status);
- res.type(response.content.mimeType);
- res.send(text);
- }
- }
-
- this.log(chalk.green('Reading From Cache', hash, chalk.bold(entry.request.method, entry.request.url)));
- }
- }
- });
-
- return this;
- }
-
- /**
- * Add the proxy route that makes the actual request to the target server and cache the response when it comes back.
- * Modifies locations on redirects.
- * @returns {JsonCachingProxy}
- */
- addProxyRoute() {
- this.app.use('/', proxy(this.options.remoteServerUrl, {
- userResDecorator: (rsp, rspData, req, res) => {
- // Handle Redirects by modifying the location property of the response header
- let location = res.get('location');
- if (location) {
- res.location(urlUtil.parse(location).path);
- }
-
- if (this.options.overrideCors && this.options.overrideCors !== '*') {
- res.header('access-control-allow-origin', this.options.overrideCors);
- }
-
- if (this.options.deleteCookieDomain && res._headers['set-cookie']) {
- res.header('set-cookie', this.removeCookiesDomain(res._headers['set-cookie'] || []));
- }
-
- if (this.isRouteExcluded(req.method, req.url)) {
- this.log(chalk.red('Exclude Proxied Resource', chalk.bold(req.method, req.url)));
- } else {
- let mimeType = res._headers['content-type'];
-
- if (this.options.dataRecord && (this.options.cacheEverything || !this.options.cacheEverything && mimeType && mimeType.indexOf('application/json') >= 0)) {
- let { key, hash } = this.genKeyFromExpressReq(req);
- let entry = this.createHarEntry(new Date().toISOString(), req, res, rspData);
- this.routeCache[key] = entry;
- this.log(chalk.yellow('Saved to Cache', hash, chalk.bold(entry.request.method, entry.request.url)));
- } else {
- this.log(chalk.gray('Proxied Resource', chalk.bold(req.method, req.url)));
- }
- }
-
- return rspData;
- }
- }));
-
- return this;
- }
-
- /**
- * Start the server and generate any log output if needed
- * @param {Function} callback fn executed after the server has started
- * @returns {JsonCachingProxy}
- */
- start(onStarted) {
- this.server = this.app.listen(this.options.proxyPort);
- this.server.setTimeout(this.options.proxyTimeout);
-
- this.log(chalk.bold(`\JSON Caching Proxy Started:`));
- this.log(chalk.gray(`===========================\n`));
- this.log(`Remote server url: \t${chalk.bold(this.options.remoteServerUrl)}`);
- this.log(`Proxy running on port: \t${chalk.bold(this.options.proxyPort)}`);
- this.log(`Proxy Timeout: \t\t${chalk.bold(this.options.proxyTimeout)}`);
- this.log(`Replay cache: \t\t${chalk.bold(this.options.dataPlayback)}`);
- this.log(`Save to cache: \t\t${chalk.bold(this.options.dataRecord)}`);
- this.log(`Command prefix: \t${chalk.bold(this.options.commandPrefix)}`);
- this.log(`Proxy response header: \t${chalk.bold(this.options.proxyHeaderIdentifier)}`);
- this.log(`Cache all: \t\t${chalk.bold(this.options.cacheEverything)}`);
- this.log(`Delete cookies domain: \t${chalk.bold(this.options.deleteCookieDomain)}`);
- this.log(`Override CORS origin: \t${chalk.bold(this.options.overrideCors)}`);
- this.log(`Use CORS Credentials: \t${chalk.bold(this.options.useCorsCredentials)}`);
- this.log(`Cache busting params: \t${chalk.bold(this.options.cacheBustingParams)}`);
- this.log(`Excluded routes: `);
- this.options.excludedRouteMatchers.forEach((regExp) => {
- this.log(`\t\t\t${chalk.bold(regExp)}`)
- });
-
- this.log('\nListening...\n');
-
- // The order of these routes is important
- this.addHarEntriesToCache(this.options.harObject);
- this.addAdminRoutes();
- this.addMiddleWareRoutes(this.options.middlewareList);
- this.addBodyParser();
- this.addCachingRoute();
- this.addProxyRoute();
-
- onStarted && onStarted();
-
- return this;
- }
-
- /**
- * Stops the proxy server
- * @param {Function} callback fn executed after the server has stopped
- * @returns {JsonCachingProxy}
- */
- stop(onStopped) {
- if (this.server) {
- this.server.close(onStopped);
- this.log(chalk.bold('\nStopping Proxy Server'));
- }
-
- return this;
- }
-
- /**
- * Returns the options passed into the proxy
- * @returns {Object}
- */
- getOptions() { return this.options; }
-
- /**
- * Returns the default options that are used when no options are passed in
- * @returns {Object}
- */
- getDefaultOptions() { return this.defaultOptions; }
-
- /**
- * Returns the Express App
- * @returns {Object}
- */
- getApp() { return this.app; }
-
- /**
- * Returns the Node server object
- * @returns {Object}
- */
- getServer() { return this.server; }
-
- /**
- * Returns the key value map of all the excluded params
- * @returns {Object}
- */
- getExcludedParamMap() { return this.excludedParamMap; }
-
- /**
- * Count of total cached routes in memory
- * @returns {number}
- */
- getTotalCachedRoutes() { return Object.keys(this.routeCache).length; }
-
- /**
- * Determines if we have anything in the cache
- * @returns {boolean}
- */
- isRouteCacheEmpty() { return this.getTotalCachedRoutes() === 0; }
-
- /**
- * Is the server sending us cached responses
- * @returns {boolean}
- */
- isReplaying() { return this.options.dataPlayback; }
-
- /**
- * Is the server saving to the cache
- * @returns {JsonCachingProxy}
- */
- isRecording() { return this.options.dataRecord; }
-}
-
-module.exports = JsonCachingProxy;
-
-
-