From 0a38dce0c825888111ecf6d17d9c436bfcca64ec Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Tue, 20 Oct 2020 14:26:55 +0200 Subject: [PATCH 1/8] init --- src/server/app.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/server/app.ts b/src/server/app.ts index 43b59d5b4..be5f7d643 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -65,6 +65,9 @@ if (SERVER_SETTINGS.getStrictTransportSecurity() === "always") { })); } +// TODO: plugins go here + +// TODO: after plugins // development HMR if (app.get("env") === "dev-hmr") { // add hot module replacement @@ -93,12 +96,13 @@ if (app.get("env") === "dev-hmr") { attachRouter("/", express.static(path.join(__dirname, "../../build/public"))); attachRouter("/", express.static(path.join(__dirname, "../../assets"))); +// TODO: before plugins app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); const settingsGetter: SettingsGetter = opts => SETTINGS_MANAGER.getSettings(opts); -// Auth +// TODO: remove if (AUTH) { app.use(AUTH); } @@ -112,7 +116,7 @@ attachRouter("/plyql", plyqlRouter(settingsGetter)); attachRouter("/mkurl", mkurlRouter(settingsGetter)); attachRouter("/shorten", shortenRouter(settingsGetter, isTrustedProxy)); -// View routes +// TODO: Try to move before routes and/or plugins if (SERVER_SETTINGS.getIframe() === "deny") { app.use((req: Request, res: Response, next: Function) => { res.setHeader("X-Frame-Options", "DENY"); From d936fe3ca40733c4052bb89b360c6eecc595dd5e Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 22 Oct 2020 12:10:02 +0200 Subject: [PATCH 2/8] Remove AUTH module --- src/server/app.ts | 7 +---- src/server/config.ts | 26 ------------------- .../models/server-settings/server-settings.ts | 3 --- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/src/server/app.ts b/src/server/app.ts index be5f7d643..06a26e41b 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -22,7 +22,7 @@ import { Handler, Request, Response, Router } from "express"; import { hsts } from "helmet"; import * as path from "path"; import { LOGGER } from "../common/logger/logger"; -import { AUTH, SERVER_SETTINGS, SETTINGS_MANAGER, VERSION } from "./config"; +import { SERVER_SETTINGS, SETTINGS_MANAGER, VERSION } from "./config"; import { livenessRouter } from "./routes/liveness/liveness"; import { mkurlRouter } from "./routes/mkurl/mkurl"; import { plyqlRouter } from "./routes/plyql/plyql"; @@ -102,11 +102,6 @@ app.use(bodyParser.urlencoded({ extended: true })); const settingsGetter: SettingsGetter = opts => SETTINGS_MANAGER.getSettings(opts); -// TODO: remove -if (AUTH) { - app.use(AUTH); -} - attachRouter(SERVER_SETTINGS.getReadinessEndpoint(), readinessRouter(settingsGetter)); attachRouter(SERVER_SETTINGS.getLivenessEndpoint(), livenessRouter); diff --git a/src/server/config.ts b/src/server/config.ts index c00653a33..66da66c6d 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -203,32 +203,6 @@ if (parsedArgs["auth"]) { export const VERBOSE = Boolean(parsedArgs["verbose"] || serverSettingsJS.verbose); export const SERVER_SETTINGS = ServerSettings.fromJS(serverSettingsJS); -// --- Auth ------------------------------- - -var auth = SERVER_SETTINGS.auth; -var authMiddleware: any = null; -if (auth && auth !== "none") { - auth = path.resolve(anchorPath, auth); - logger.log(`Using auth ${auth}`); - try { - var authModule = require(auth); - } catch (e) { - exitWithError(`error loading auth module: ${e.message}`); - } - - if (authModule.version !== AUTH_MODULE_VERSION) { - exitWithError(`incorrect auth module version ${authModule.version} needed ${AUTH_MODULE_VERSION}`); - } - if (typeof authModule.auth !== "function") exitWithError("Invalid auth module: must export 'auth' function"); - authMiddleware = authModule.auth({ - logger, - verbose: VERBOSE, - version: VERSION, - serverSettings: SERVER_SETTINGS - }); -} -export const AUTH = authMiddleware; - // --- Sign of Life ------------------------------- if (START_SERVER) { logger.log(`Starting Turnilo v${VERSION}`); diff --git a/src/server/models/server-settings/server-settings.ts b/src/server/models/server-settings/server-settings.ts index eb986fee9..c8ecf790e 100644 --- a/src/server/models/server-settings/server-settings.ts +++ b/src/server/models/server-settings/server-settings.ts @@ -33,7 +33,6 @@ export interface ServerSettingsValue { iframe?: Iframe; trustProxy?: TrustProxy; strictTransportSecurity?: StrictTransportSecurity; - auth?: string; settingsLocation?: SettingsLocation; } @@ -76,7 +75,6 @@ export class ServerSettings extends BaseImmutable Date: Thu, 22 Oct 2020 14:06:43 +0200 Subject: [PATCH 3/8] Reorder npm build scripts a little --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 5eeb52723..0d42b8400 100644 --- a/package.json +++ b/package.json @@ -47,16 +47,16 @@ "test-watcher": "mocha --require ./test/setup/mocha.js --watch --watch-extensions ts,tsx", "start": "./bin/turnilo", "start:dev": "NODE_ENV=dev-hmr ./bin/turnilo", - "build": "npm run clean && npm run tsc -- -p src/server/tsconfig.json && webpack --config config/webpack.prod.js", - "build:dev": "npm run clean && npm run tsc -- -p src/server/tsconfig.json", - "dev": "npm run clean && npm run tsc -- -p tsconfig.json -w --pretty", + "start:examples": "npm run start -- --examples", + "start:dev:examples": "npm run start:dev -- --examples", + "build:client": "webpack --config config/webpack.prod.js", + "build:server": "npm run tsc -- -p src/server/tsconfig.json ", + "build": "npm-run-all -s clean build:server build:client", "clean": "rimraf build/*", "tsc": "tsc", "lint": "npm-run-all -p lint:*", "lint:ts": "tslint -p . --format verbose", "lint:sass": "sass-lint -v", - "start:examples": "npm run start -- --examples", - "start:dev:examples": "npm run start:dev -- --examples", "e2e": "start-server-and-test start:examples http://localhost:9090/health/ready 'cypress run'", "e2e:dev": "cypress open", "check": "npm-run-all -c -p lint test build e2e" From 3a0bf8c7939d6b7fc619defaa79a17e393f4d0b1 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 22 Oct 2020 14:07:16 +0200 Subject: [PATCH 4/8] AppSettings are always loaded from the same file as ServerSettings --- src/server/config.ts | 63 ++++++------ .../server-settings/server-settings.mocha.ts | 14 --- .../models/server-settings/server-settings.ts | 10 +- .../settings-location.mocha.ts | 53 ---------- .../settings-location/settings-location.ts | 96 ------------------- .../utils/settings-store/settings-store.ts | 41 +------- 6 files changed, 37 insertions(+), 240 deletions(-) delete mode 100644 src/server/models/settings-location/settings-location.mocha.ts delete mode 100644 src/server/models/settings-location/settings-location.ts diff --git a/src/server/config.ts b/src/server/config.ts index 66da66c6d..5f134d166 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -23,7 +23,7 @@ import { Cluster } from "../common/models/cluster/cluster"; import { DataCube } from "../common/models/data-cube/data-cube"; import { arraySum } from "../common/utils/general/general"; import { appSettingsToYAML } from "../common/utils/yaml-helper/yaml-helper"; -import { ServerSettings } from "./models/server-settings/server-settings"; +import { ServerSettings, ServerSettingsJS } from "./models/server-settings/server-settings"; import { loadFileSync } from "./utils/file/file"; import { SettingsManager } from "./utils/settings-manager/settings-manager"; import { SettingsStore } from "./utils/settings-store/settings-store"; @@ -166,24 +166,26 @@ export const START_SERVER = !PRINT_CONFIG; const logger = START_SERVER ? LOGGER : NULL_LOGGER; // Load server settings -var serverSettingsFilePath = parsedArgs["config"]; +let configPath = parsedArgs["config"]; if (parsedArgs["examples"]) { - serverSettingsFilePath = path.join(__dirname, "../../config-examples.yaml"); + configPath = path.join(__dirname, "../../config-examples.yaml"); } -var anchorPath: string; -var serverSettingsJS: any; -if (serverSettingsFilePath) { - anchorPath = path.dirname(serverSettingsFilePath); +let serverSettingsJS: ServerSettingsJS; +let configDirPath; +let configContent; +if (configPath) { + configDirPath = path.dirname(configPath); try { - serverSettingsJS = loadFileSync(serverSettingsFilePath, "yaml"); - logger.log(`Using config ${serverSettingsFilePath}`); + configContent = loadFileSync(configPath, "yaml"); + serverSettingsJS = configContent; + logger.log(`Using config ${configPath}`); } catch (e) { - exitWithError(`Could not load config from '${serverSettingsFilePath}': ${e.message}`); + exitWithError(`Could not load config from '${configPath}': ${e.message}`); } } else { - anchorPath = process.cwd(); + configDirPath = process.cwd(); serverSettingsJS = {}; } @@ -196,11 +198,12 @@ if (parsedArgs["server-host"]) { if (parsedArgs["server-root"]) { serverSettingsJS.serverRoot = parsedArgs["server-root"]; } -if (parsedArgs["auth"]) { - serverSettingsJS.auth = parsedArgs["auth"]; +if (parsedArgs["verbose"]) { + serverSettingsJS.verbose = parsedArgs["verbose"]; } -export const VERBOSE = Boolean(parsedArgs["verbose"] || serverSettingsJS.verbose); +// TODO: Remove this export +export const VERBOSE = Boolean(serverSettingsJS.verbose); export const SERVER_SETTINGS = ServerSettings.fromJS(serverSettingsJS); // --- Sign of Life ------------------------------- @@ -210,28 +213,18 @@ if (START_SERVER) { // --- Location ------------------------------- -var settingsStore: SettingsStore = null; - -if (serverSettingsFilePath) { - var settingsLocation = SERVER_SETTINGS.getSettingsLocation(); - if (settingsLocation) { - switch (settingsLocation.getLocation()) { - case "file": - var settingsFilePath = path.resolve(anchorPath, settingsLocation.uri); - settingsStore = SettingsStore.fromReadOnlyFile(settingsFilePath, settingsLocation.getFormat()); - break; - default: - exitWithError(`unknown location '${settingsLocation.location}'`); - } - - } else { - settingsStore = SettingsStore.fromReadOnlyFile(serverSettingsFilePath, "yaml"); - } +let settingsStore: SettingsStore; + +if (configContent) { + const appSettings = AppSettings.fromJS(configContent, {}); + // TODO: this validation should be done via #365 + appSettings.validate(); + settingsStore = SettingsStore.create(appSettings); } else { - var initAppSettings = AppSettings.BLANK; + let initAppSettings = AppSettings.BLANK; // If a file is specified add it as a dataCube - var fileToLoad = parsedArgs["file"]; + const fileToLoad = parsedArgs["file"]; if (fileToLoad) { initAppSettings = initAppSettings.addDataCube(new DataCube({ name: path.basename(fileToLoad, path.extname(fileToLoad)), @@ -253,13 +246,13 @@ if (serverSettingsFilePath) { })); } - settingsStore = SettingsStore.fromTransient(initAppSettings); + settingsStore = SettingsStore.create(initAppSettings); } export const SETTINGS_MANAGER = new SettingsManager(settingsStore, { logger, verbose: VERBOSE, - anchorPath, + anchorPath: configDirPath, initialLoadTimeout: SERVER_SETTINGS.getPageMustLoadTimeout() }); diff --git a/src/server/models/server-settings/server-settings.mocha.ts b/src/server/models/server-settings/server-settings.mocha.ts index bbae32480..096f85371 100644 --- a/src/server/models/server-settings/server-settings.mocha.ts +++ b/src/server/models/server-settings/server-settings.mocha.ts @@ -23,9 +23,6 @@ describe("ServerSettings", () => { it("is an immutable class", () => { testImmutableClass(ServerSettings, [ {}, - { - port: 9090 - }, { port: 9091 }, @@ -50,17 +47,6 @@ describe("ServerSettings", () => { serverRoot: "/swivs", readinessEndpoint: "/status/readiness", pageMustLoadTimeout: 901 - }, - { - port: 9091, - auth: "my_auth.js" - }, - { - port: 9091, - settingsLocation: { - location: "file", - uri: "path/to/my/file.yaml" - } } ]); }); diff --git a/src/server/models/server-settings/server-settings.ts b/src/server/models/server-settings/server-settings.ts index c8ecf790e..be5bbb104 100644 --- a/src/server/models/server-settings/server-settings.ts +++ b/src/server/models/server-settings/server-settings.ts @@ -16,7 +16,6 @@ */ import { BackCompat, BaseImmutable } from "immutable-class"; -import { SettingsLocation } from "../settings-location/settings-location"; export type Iframe = "allow" | "deny"; export type TrustProxy = "none" | "always"; @@ -30,10 +29,10 @@ export interface ServerSettingsValue { livenessEndpoint?: string; requestLogFormat?: string; pageMustLoadTimeout?: number; + verbose?: boolean; iframe?: Iframe; trustProxy?: TrustProxy; strictTransportSecurity?: StrictTransportSecurity; - settingsLocation?: SettingsLocation; } export interface ServerSettingsJS extends ServerSettingsValue { @@ -69,13 +68,13 @@ export class ServerSettings extends BaseImmutable Iframe; public getTrustProxy: () => TrustProxy; public getStrictTransportSecurity: () => StrictTransportSecurity; - public getSettingsLocation: () => SettingsLocation; } BaseImmutable.finalize(ServerSettings); diff --git a/src/server/models/settings-location/settings-location.mocha.ts b/src/server/models/settings-location/settings-location.mocha.ts deleted file mode 100644 index 2fd5f363c..000000000 --- a/src/server/models/settings-location/settings-location.mocha.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2015-2016 Imply Data, Inc. - * Copyright 2017-2019 Allegro.pl - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { expect } from "chai"; -import { testImmutableClass } from "immutable-class-tester"; -import { SettingsLocation } from "./settings-location"; - -describe("SettingsLocation", () => { - it("is an immutable class", () => { - testImmutableClass(SettingsLocation, [ - { - location: "file", - uri: "../private/lol.yaml" - }, - { - location: "mysql", - uri: "mysql://root:@192.168.99.100:3306/datazoo" - }, - { - location: "mysql", - uri: "mysql://root:@192.168.99.100:3306/datazoo", - table: "swiv_state" - } - ]); - }); - - describe("gets the right format", () => { - it("gets yaml", () => { - var settingsLocation = SettingsLocation.fromJS({ - location: "file", - uri: "../private/lol.yaml" - }); - - expect(settingsLocation.getFormat()).to.equal("yaml"); - }); - - }); - -}); diff --git a/src/server/models/settings-location/settings-location.ts b/src/server/models/settings-location/settings-location.ts deleted file mode 100644 index 7efe17945..000000000 --- a/src/server/models/settings-location/settings-location.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2015-2016 Imply Data, Inc. - * Copyright 2017-2019 Allegro.pl - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { BaseImmutable, Property } from "immutable-class"; - -export type Location = "file" | "mysql" | "postgres"; -export type Format = "json" | "yaml"; - -export interface SettingsLocationValue { - location: Location; - uri: string; - table?: string; - format?: Format; - readOnly?: boolean; -} - -export interface SettingsLocationJS { - location: Location; - uri: string; - table?: string; - format?: Format; - readOnly?: boolean; -} - -export class SettingsLocation extends BaseImmutable { - static LOCATION_VALUES: Location[] = ["file", "mysql", "postgres"]; - static DEFAULT_FORMAT: Format = "json"; - static FORMAT_VALUES: Format[] = ["json", "yaml"]; - - static isSettingsLocation(candidate: any): candidate is SettingsLocation { - return candidate instanceof SettingsLocation; - } - - static fromJS(parameters: SettingsLocationJS): SettingsLocation { - return new SettingsLocation(BaseImmutable.jsToValue(SettingsLocation.PROPERTIES, parameters)); - } - - static PROPERTIES: Property[] = [ - { name: "location", possibleValues: SettingsLocation.LOCATION_VALUES }, - { name: "uri" }, - { name: "table", defaultValue: null }, - { name: "format", defaultValue: SettingsLocation.DEFAULT_FORMAT, possibleValues: SettingsLocation.FORMAT_VALUES }, - { name: "readOnly", defaultValue: false } - ]; - - public location: Location; - public uri: string; - public table: string; - public format: Format; - public readOnly: boolean; - - constructor(parameters: SettingsLocationValue) { - super(parameters); - - // remove table if file - if (this.location === "file" && this.table) this.table = null; - } - - public getLocation: () => Location; - public getUri: () => string; - public getTable: () => string; - - public getFormat(): Format { - if (this.format) return this.format; - - // derive format from extension if not set, and possible - if (this.location === "file") { - if (/\.json$/.test(this.uri)) { - return "json"; - } else if (/\.yaml$/.test(this.uri)) { - return "yaml"; - } - } - - return SettingsLocation.DEFAULT_FORMAT; - } - - public getReadOnly: () => boolean; - -} - -BaseImmutable.finalize(SettingsLocation); diff --git a/src/server/utils/settings-store/settings-store.ts b/src/server/utils/settings-store/settings-store.ts index 31c5e84ce..b9c5143ad 100644 --- a/src/server/utils/settings-store/settings-store.ts +++ b/src/server/utils/settings-store/settings-store.ts @@ -15,47 +15,16 @@ * limitations under the License. */ -import * as fs from "fs-promise"; -import * as yaml from "js-yaml"; import { AppSettings } from "../../../common/models/app-settings/app-settings"; -import { inlineVars } from "../../../common/utils/general/general"; -import { Format } from "../../models/settings-location/settings-location"; +import { Nullary } from "../../../common/utils/functional/functional"; -function readSettingsFactory(filepath: string, format: Format, inline = false): () => Promise { - return () => fs.readFile(filepath, "utf-8") - .then(fileData => { - switch (format) { - case "json": - return JSON.parse(fileData); - case "yaml": - return yaml.safeLoad(fileData); - default: - throw new Error(`unsupported format '${format}'`); - } - }) - .then(appSettingsJS => { - if (inline) appSettingsJS = inlineVars(appSettingsJS, process.env); - const appSettings = AppSettings.fromJS(appSettingsJS, { }); - appSettings.validate(); - return appSettings; - }); -} +type SettingsPromise = Nullary>; export class SettingsStore { - static fromTransient(initAppSettings: AppSettings): SettingsStore { - let settingsStore = new SettingsStore(); - settingsStore.readSettings = () => Promise.resolve(initAppSettings); - return settingsStore; - } - - static fromReadOnlyFile(filepath: string, format: Format): SettingsStore { - let settingsStore = new SettingsStore(); - settingsStore.readSettings = readSettingsFactory(filepath, format, true); - return settingsStore; + static create(appSettings: AppSettings): SettingsStore { + return new SettingsStore(() => Promise.resolve(appSettings)); } - public readSettings: () => Promise; - - constructor() { + constructor(public readSettings: SettingsPromise) { } } From a8b75037f61d5207724a5e8d0bde656416ea45e4 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Fri, 6 Nov 2020 16:22:44 +0100 Subject: [PATCH 5/8] We no longer support `dataCubeOfInterest` option --- src/server/routes/mkurl/mkurl.ts | 2 +- src/server/routes/plyql/plyql.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/routes/mkurl/mkurl.ts b/src/server/routes/mkurl/mkurl.ts index 719d7a988..def7df404 100644 --- a/src/server/routes/mkurl/mkurl.ts +++ b/src/server/routes/mkurl/mkurl.ts @@ -55,7 +55,7 @@ export function mkurlRouter(settingsGetter: SettingsGetter) { let settings: AppSettings; try { - settings = await settingsGetter( { dataCubeOfInterest: dataCubeName }); + settings = await settingsGetter(); } catch (e) { res.status(400).send({ error: "Couldn't load settings" }); return; diff --git a/src/server/routes/plyql/plyql.ts b/src/server/routes/plyql/plyql.ts index cab40710f..35b1e60f0 100644 --- a/src/server/routes/plyql/plyql.ts +++ b/src/server/routes/plyql/plyql.ts @@ -79,7 +79,7 @@ export function plyqlRouter(settingsGetter: SettingsGetter) { }); try { - const settings = await settingsGetter( { dataCubeOfInterest: dataCube }); + const settings = await settingsGetter(); const myDataCube = settings.getDataCube(dataCube); if (!myDataCube) { From f82faf03dd89a683aad9aa7f35278aca7a86a60c Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Fri, 6 Nov 2020 16:38:22 +0100 Subject: [PATCH 6/8] Split settings getter to simple getter and one with settings refresh. Only root endpoint refreshes settings (if specific option is set in config) --- src/server/app.ts | 3 +- src/server/config.ts | 2 +- .../settings-manager/settings-manager.ts | 46 ++++++++++--------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/server/app.ts b/src/server/app.ts index 06a26e41b..9c447053f 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -120,7 +120,8 @@ if (SERVER_SETTINGS.getIframe() === "deny") { }); } -attachRouter("/", turniloRouter(settingsGetter, VERSION)); +const freshSettingsGetter: SettingsGetter = opts => SETTINGS_MANAGER.getFreshSettings(opts); +attachRouter("/", turniloRouter(freshSettingsGetter, VERSION)); // Catch 404 and redirect to / app.use((req: Request, res: Response) => { diff --git a/src/server/config.ts b/src/server/config.ts index 5f134d166..805459ad8 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -261,7 +261,7 @@ export const SETTINGS_MANAGER = new SettingsManager(settingsStore, { if (PRINT_CONFIG) { var withComments = Boolean(parsedArgs["with-comments"]); - SETTINGS_MANAGER.getSettings({ + SETTINGS_MANAGER.getFreshSettings({ timeout: 10000 }).then(appSettings => { const config = appSettingsToYAML(appSettings, withComments, { diff --git a/src/server/utils/settings-manager/settings-manager.ts b/src/server/utils/settings-manager/settings-manager.ts index ddca2bcae..5b44310fe 100644 --- a/src/server/utils/settings-manager/settings-manager.ts +++ b/src/server/utils/settings-manager/settings-manager.ts @@ -51,7 +51,7 @@ export class SettingsManager { public timeMonitor: TimeMonitor; public fileManagers: FileManager[]; public clusterManagers: ClusterManager[]; - public currentWork: Promise; + public settingsLoaded: Promise; public initialLoadTimeout: number; constructor(settingsStore: SettingsStore, options: SettingsManagerOptions) { @@ -68,7 +68,7 @@ export class SettingsManager { this.initialLoadTimeout = options.initialLoadTimeout || 30000; this.appSettings = AppSettings.BLANK; - this.currentWork = settingsStore.readSettings() + this.settingsLoaded = settingsStore.readSettings() .then(appSettings => this.reviseSettings(appSettings)) .catch(e => { logger.error(`Fatal settings load error: ${e.message}`); @@ -126,23 +126,27 @@ export class SettingsManager { return this.timeMonitor.timekeeper; } - getSettings(opts: GetSettingsOptions = {}): Promise { - let currentWork = this.currentWork; + handleSettingsTask(task: Promise, opts: GetSettingsOptions = {}): Promise { + const timeoutMs = opts.timeout || this.initialLoadTimeout; + if (timeoutMs === 0) { + return task.then(() => this.appSettings); + } + return Promise.race([task, timeout(timeoutMs)]) + .catch(() => { + this.logger.error("Settings load timeout hit, continuing"); + }) + .then(() => this.appSettings); + } - // Refresh all clusters - currentWork = currentWork.then(() => { + getFreshSettings(opts: GetSettingsOptions = {}): Promise { + const task = this.settingsLoaded.then(() => { return Promise.all(this.clusterManagers.map(clusterManager => clusterManager.refresh())) as any; }); + return this.handleSettingsTask(task, opts); + } - const timeoutPeriod = opts.timeout || this.initialLoadTimeout; - if (timeoutPeriod !== 0) { - currentWork = Promise.race([currentWork, timeout(timeoutPeriod)]) - .catch(e => { - this.logger.error("Settings load timeout hit, continuing"); - }); - } - - return currentWork.then(() => this.appSettings); + getSettings(opts: GetSettingsOptions = {}): Promise { + return this.handleSettingsTask(this.settingsLoaded, opts); } reviseSettings(newSettings: AppSettings): Promise { @@ -178,7 +182,7 @@ export class SettingsManager { candidateName = source + i; } return candidateName; - } + }; onDatasetChange = (dataCubeName: string, changedDataset: Dataset): void => { const { logger } = this; @@ -196,9 +200,9 @@ export class SettingsManager { } this.appSettings = this.appSettings.addOrUpdateDataCube(dataCube); - } + }; - onExternalChange = (cluster: Cluster, dataCubeName: string, changedExternal: External): Promise => { + onExternalChange = (cluster: Cluster, dataCubeName: string, changedExternal: External): Promise => { if (!changedExternal.attributes || !changedExternal.requester) return Promise.resolve(null); const { logger } = this; @@ -218,9 +222,9 @@ export class SettingsManager { this.appSettings = this.appSettings.addOrUpdateDataCube(dataCube); return Promise.resolve(null); - } + }; - onExternalRemoved = (cluster: Cluster, dataCubeName: string, changedExternal: External): Promise => { + onExternalRemoved = (cluster: Cluster, dataCubeName: string, changedExternal: External): Promise => { if (!changedExternal.attributes || !changedExternal.requester) return Promise.resolve(null); const { logger } = this; @@ -232,5 +236,5 @@ export class SettingsManager { this.timeMonitor.removeCheck(dataCube.name); } return Promise.resolve(null); - } + }; } From 840241e289bfbdf8193184f313e487ba6b1f133d Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 9 Nov 2020 11:13:30 +0100 Subject: [PATCH 7/8] Move "middlewares" inside app.ts --- package-lock.json | 54 +++++++++++++++++++++++------------------------ src/server/app.ts | 24 ++++++++++----------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6880f5b52..55f02ba2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1748,7 +1748,7 @@ }, "acorn": { "version": "2.7.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-2.7.0.tgz", + "resolved": "http://registry.npmjs.org/acorn/-/acorn-2.7.0.tgz", "integrity": "sha1-q259nYhqrKiwhbwzEreaGYQz8Oc=", "dev": true }, @@ -1772,7 +1772,7 @@ "dependencies": { "acorn": { "version": "3.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "resolved": "http://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=", "dev": true } @@ -2055,7 +2055,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -3092,7 +3092,7 @@ "dependencies": { "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -4880,7 +4880,7 @@ "dependencies": { "core-js": { "version": "1.2.7", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", + "resolved": "http://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY=", "dev": true } @@ -5941,7 +5941,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -6091,7 +6091,7 @@ "dependencies": { "minimist": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.1.3.tgz", "integrity": "sha1-O+39kaktOQFvz6ocaB6Pqhoe/ag=", "dev": true } @@ -6375,7 +6375,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "requires": { "depd": "~1.1.2", @@ -6651,7 +6651,7 @@ }, "is-accessor-descriptor": { "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "resolved": "http://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { @@ -6731,7 +6731,7 @@ }, "is-data-descriptor": { "version": "0.1.4", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "resolved": "http://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { @@ -7149,7 +7149,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "requires": { "graceful-fs": "^4.1.6" @@ -7392,7 +7392,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -7673,7 +7673,7 @@ "dependencies": { "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -7767,7 +7767,7 @@ }, "mime": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/mime/-/mime-2.3.1.tgz", "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==", "dev": true }, @@ -8235,7 +8235,7 @@ }, "semver": { "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -8813,7 +8813,7 @@ }, "os-locale": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9181,7 +9181,7 @@ "dependencies": { "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -10083,7 +10083,7 @@ }, "fast-deep-equal": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" }, "har-validator": { @@ -10112,7 +10112,7 @@ }, "tough-cookie": { "version": "2.3.4", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "resolved": "http://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", "requires": { "punycode": "^1.4.1" @@ -10460,7 +10460,7 @@ "dependencies": { "source-map": { "version": "0.4.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", "dev": true, "requires": { @@ -10614,7 +10614,7 @@ }, "kind-of": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", "integrity": "sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU=", "dev": true, "requires": { @@ -11012,7 +11012,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { @@ -11272,7 +11272,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -11322,7 +11322,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { @@ -11430,7 +11430,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -12803,7 +12803,7 @@ }, "load-json-file": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", "requires": { "graceful-fs": "^4.1.2", @@ -13043,7 +13043,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/server/app.ts b/src/server/app.ts index 9c447053f..05715ae50 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -65,6 +65,17 @@ if (SERVER_SETTINGS.getStrictTransportSecurity() === "always") { })); } +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: true })); + +if (SERVER_SETTINGS.getIframe() === "deny") { + app.use((req: Request, res: Response, next: Function) => { + res.setHeader("X-Frame-Options", "DENY"); + res.setHeader("Content-Security-Policy", "frame-ancestors 'none'"); + next(); + }); +} + // TODO: plugins go here // TODO: after plugins @@ -96,10 +107,6 @@ if (app.get("env") === "dev-hmr") { attachRouter("/", express.static(path.join(__dirname, "../../build/public"))); attachRouter("/", express.static(path.join(__dirname, "../../assets"))); -// TODO: before plugins -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: true })); - const settingsGetter: SettingsGetter = opts => SETTINGS_MANAGER.getSettings(opts); attachRouter(SERVER_SETTINGS.getReadinessEndpoint(), readinessRouter(settingsGetter)); @@ -111,15 +118,6 @@ attachRouter("/plyql", plyqlRouter(settingsGetter)); attachRouter("/mkurl", mkurlRouter(settingsGetter)); attachRouter("/shorten", shortenRouter(settingsGetter, isTrustedProxy)); -// TODO: Try to move before routes and/or plugins -if (SERVER_SETTINGS.getIframe() === "deny") { - app.use((req: Request, res: Response, next: Function) => { - res.setHeader("X-Frame-Options", "DENY"); - res.setHeader("Content-Security-Policy", "frame-ancestors 'none'"); - next(); - }); -} - const freshSettingsGetter: SettingsGetter = opts => SETTINGS_MANAGER.getFreshSettings(opts); attachRouter("/", turniloRouter(freshSettingsGetter, VERSION)); From c899f8519e154751e5ebd3cdf0bf05ac06dfd22f Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 9 Nov 2020 12:00:00 +0100 Subject: [PATCH 8/8] Better warning for settings load timeout --- src/server/utils/settings-manager/settings-manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/utils/settings-manager/settings-manager.ts b/src/server/utils/settings-manager/settings-manager.ts index 5b44310fe..4b1a8e640 100644 --- a/src/server/utils/settings-manager/settings-manager.ts +++ b/src/server/utils/settings-manager/settings-manager.ts @@ -133,7 +133,7 @@ export class SettingsManager { } return Promise.race([task, timeout(timeoutMs)]) .catch(() => { - this.logger.error("Settings load timeout hit, continuing"); + this.logger.warn(`Settings load timeout (${timeoutMs}ms) hit, continuing`); }) .then(() => this.appSettings); }