From 0b1c27e16b7d66cdeb8f959ca6ba178a24929102 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 27 Oct 2022 17:50:41 +0200 Subject: [PATCH 01/28] main menu for cli --- bin/turnilo | 2 +- src/server/cli.ts | 169 ++++++++++++++++++++++++++++++++++++++++++ src/server/version.ts | 29 ++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 src/server/cli.ts create mode 100644 src/server/version.ts diff --git a/bin/turnilo b/bin/turnilo index 7e048520f..60bc71281 100755 --- a/bin/turnilo +++ b/bin/turnilo @@ -19,6 +19,6 @@ var path = require('path'); var fs = require('fs'); -var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../build/server/www.js'); +var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../build/server/cli.js'); require(lib); diff --git a/src/server/cli.ts b/src/server/cli.ts new file mode 100644 index 000000000..88a7cb26e --- /dev/null +++ b/src/server/cli.ts @@ -0,0 +1,169 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InvalidArgumentError, Option, program } from "commander"; +import { VERSION } from "./version"; + +function assertIntegerOption(value: string): number { + const parsed = parseInt(value, 10); + if (isNaN(parsed)) throw new InvalidArgumentError("Must be an integer"); + return parsed; +} + +const port = new Option("-p, --port ", "port number").argParser(assertIntegerOption).default(900); // TODO: DEFAULT_PORT +const serverRoot = new Option("--server-root ", "server root").default(""); // TODO: DEFAULT_ROOT +const serverHost = new Option("--server-host ", "server host").default(null); // TODO: DEFAULT_HOST +const verbose = new Option("--verbose", "verbose mode").default(false); +const username = new Option("--username ", "username"); +const password = new Option("--password ", "password"); + +function assertCredentials(username: string | undefined, password: string | undefined) { + if (password && !username || username && !password) { + program.error("You need to pass both username and password"); + } +} + +program + .name("turnilo") + .description("Turnilo is a data exploration tool that connects to Druid database") + .version(VERSION, "--version"); + +program + .command("run-config") + .argument("", "path of config file") + .addOption(port) + .addOption(serverRoot) + .addOption(serverHost) + .addOption(username) + .addOption(password) + .addOption(verbose) + .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { + assertCredentials(username, password); + /* + 1. load yml file as config + 2. override config with all existing options + 3. create ServerSettings from 2 + 4. create AppSettings and Sources from 2 + 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + 6. start ./app.ts with ServerSettings and code from ./www.ts + */ + console.log("run config with", configPath, "and options", { port, verbose }); + }); + +program + .command("run-examples") + .addOption(port) + .addOption(serverRoot) + .addOption(serverHost) + .addOption(verbose) + .action(({ port, verbose, serverRoot, serverHost }) => { + /* + 1. load config-examples.yaml file as config + 2. override config with all existing options + 3. create ServerSettings from 2 + 4. create AppSettings and Sources from 2 + 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + 6. start ./app.ts with ServerSettings and code from ./www.ts + */ + console.log("run examples with options", { port, verbose }); + }); + +program + .command("connect-druid") + .argument("", "druid url") + .addOption(port) + .addOption(serverRoot) + .addOption(serverHost) + .addOption(verbose) + .option("--username ", "username") + .option("--password ", "password") + .action((url, { port, verbose, username, password }) => { + assertCredentials(username, password); + /* + 1. create cluster from name: druid and passed url and empty dataCubes list + 2. create appSettings from EMPTY_APP_SETTINGS + 3. create ServerSettings from options + 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + 5. start ./app.ts with ServerSettings and code from ./www.ts + */ + console.log("connect to", url, "with options", { port, verbose, username, password }); + }); + +program + .command("load-file") + .argument("", "json file") + .addOption(port) + .addOption(serverRoot) + .addOption(serverHost) + .addOption(verbose) + .action((file, { port, verbose, serverHost, serverRoot }) => { + /* + 1. create empty cluster list and dataCube: + dataCubeFromConfig({ + name: path.basename(file, path.extname(file)), + clusterName: "native", + source: file + }, undefined + 2. create appSettings from EMPTY_APP_SETTINGS + 3. create ServerSettings from options + 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + 5. start ./app.ts with ServerSettings and code from ./www.ts + */ + console.log("load file", file, "with options", { port, verbose, serverHost, serverRoot }); + }); + +program + .command("verify-config") + .argument("", "path to config file") + .addOption(verbose) + .action(file => { + /* + 1. load yaml + 2. create server settings + 3. create app settings + 4. create sources + 5. catch and log all errors + 6. return status code + */ + console.log("verify config from", file); + }); + +program + .command("introspect-druid") + .argument("", "druid url") + .addOption(port) + .addOption(serverRoot) + .addOption(serverHost) + .addOption(verbose) + .addOption(username) + .addOption(password) + .option("--with-comments", "print comments") + .action((url, { port, verbose, username, password, withComments }) => { + assertCredentials(username, password); + /* + 1. create cluster from name: druid and passed url and empty dataCubes list + 2. create appSettings from EMPTY_APP_SETTINGS + 3. create ServerSettings from options + 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + 5. call SettingsManager.getFreshSources with timeout 1000 + 6. using yaml-helper print out appSettings, fetched sources and "Extra" + */ + console.log("introspect ", url, "with options", { port, verbose, username, password, withComments }); + }); + +program.showHelpAfterError(); + +program.parse(); diff --git a/src/server/version.ts b/src/server/version.ts new file mode 100644 index 000000000..0f6662ff6 --- /dev/null +++ b/src/server/version.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from "path"; +import { loadFileSync } from "./utils/file/file"; + +const PACKAGE_FILE = path.join(__dirname, "../../package.json"); + +let packageObj: any = null; +try { + packageObj = loadFileSync(PACKAGE_FILE, "json"); +} catch (e) { + console.error(`Could not read package.json: ${e.message}`); + process.exit(1); // TODO: common tool for exit? +} + +export const VERSION = packageObj.version; From ad2ff34c7fc6111e581703349a80799e15ceb3d7 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:55:20 +0100 Subject: [PATCH 02/28] utils for commander --- src/server/cli/utils.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/server/cli/utils.ts diff --git a/src/server/cli/utils.ts b/src/server/cli/utils.ts new file mode 100644 index 000000000..330cf46b2 --- /dev/null +++ b/src/server/cli/utils.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InvalidArgumentError } from "commander"; +import { isNil } from "../../common/utils/general/general"; + +export function parseInteger(value: string): number { + const parsed = parseInt(value, 10); + invariant(!isNaN(parsed), "Must be an integer"); + return parsed; +} + +export function assertCredentials(username: string | undefined, password: string | undefined) { + invariant(isNil(password) && isNil(username) || !isNil(username) && !isNil(password), "You need to pass both username and password"); +} + +function invariant(condition: boolean, message: string) { + if (!condition) throw new InvalidArgumentError(message); +} From 37b537259f16206c263c036982af2d984b5a5b77 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:55:48 +0100 Subject: [PATCH 03/28] create server as a function with explicit dependencies --- src/server/{www.ts => cli/create-server.ts} | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) rename src/server/{www.ts => cli/create-server.ts} (62%) mode change 100755 => 100644 diff --git a/src/server/www.ts b/src/server/cli/create-server.ts old mode 100755 new mode 100644 similarity index 62% rename from src/server/www.ts rename to src/server/cli/create-server.ts index 3d712abc3..acba12ff7 --- a/src/server/www.ts +++ b/src/server/cli/create-server.ts @@ -1,6 +1,5 @@ /* - * Copyright 2015-2016 Imply Data, Inc. - * Copyright 2017-2019 Allegro.pl + * Copyright 2017-2022 Allegro.pl * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import * as http from "http"; - +import { Express } from "express"; +import http from "http"; import { AddressInfo } from "net"; -import app from "./app"; -import { SERVER_SETTINGS, START_SERVER } from "./config"; +import { ServerSettings } from "../models/server-settings/server-settings"; + +export default function createServer(serverSettings: ServerSettings, app: Express) { -if (START_SERVER) { - var server = http.createServer(app); + const server = http.createServer(app); server.on("error", (error: any) => { if (error.syscall !== "listen") { @@ -32,12 +30,14 @@ if (START_SERVER) { // handle specific listen errors with friendly messages switch (error.code) { case "EACCES": - console.error(`Port ${SERVER_SETTINGS.port} requires elevated privileges`); + // TODO: this should do something that commander understands + console.error(`Port ${serverSettings.port} requires elevated privileges`); process.exit(1); break; case "EADDRINUSE": - console.error(`Port ${SERVER_SETTINGS.port} is already in use`); + // TODO: this should do something that commander understands + console.error(`Port ${serverSettings.port} is already in use`); process.exit(1); break; @@ -51,6 +51,6 @@ if (START_SERVER) { console.log(`Turnilo is listening on address ${address.address} port ${address.port}`); }); - app.set("port", SERVER_SETTINGS.port); - server.listen(SERVER_SETTINGS.port, SERVER_SETTINGS.serverHost); + app.set("port", serverSettings.port); + server.listen(serverSettings.port, serverSettings.serverHost); } From daee927f3fe23c3e3d814b99f0e76b214580881e Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:56:08 +0100 Subject: [PATCH 04/28] util for loading file, soon will talk to commander --- src/server/cli/load-config-file.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/server/cli/load-config-file.ts diff --git a/src/server/cli/load-config-file.ts b/src/server/cli/load-config-file.ts new file mode 100644 index 000000000..aa0487ff8 --- /dev/null +++ b/src/server/cli/load-config-file.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { loadFileSync } from "../utils/file/file"; + +export function loadConfigFile(configPath: string): object { + try { + return loadFileSync(configPath, "yaml"); + } catch (e) { + throw new Error("SOMETHING"); // something that commander can handle + } +} From 6762dede1f73eb6c66b07d9688648516419eafc1 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:56:20 +0100 Subject: [PATCH 05/28] explicit default value --- src/server/models/server-settings/server-settings.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/models/server-settings/server-settings.ts b/src/server/models/server-settings/server-settings.ts index 6f0dec00e..944c77aeb 100644 --- a/src/server/models/server-settings/server-settings.ts +++ b/src/server/models/server-settings/server-settings.ts @@ -46,6 +46,7 @@ export type ServerSettingsJS = ServerSettingsValue & { export const DEFAULT_PORT = 9090; export const DEFAULT_SERVER_ROOT = ""; +export const DEFAULT_SERVER_HOST: string = null; const DEFAULT_READINESS_ENDPOINT = "/health/ready"; const DEFAULT_LIVENESS_ENDPOINT = "/health/alive"; const DEFAULT_SERVER_TIMEOUT = 0; @@ -66,7 +67,7 @@ const defaultServerSettings: ServerSettingsValue = { port: DEFAULT_PORT, readinessEndpoint: DEFAULT_READINESS_ENDPOINT, requestLogFormat: DEFAULT_REQUEST_LOG_FORMAT, - serverHost: null, + serverHost: DEFAULT_SERVER_HOST, serverRoot: DEFAULT_SERVER_ROOT, serverTimeout: DEFAULT_SERVER_TIMEOUT, strictTransportSecurity: DEFAULT_STRICT_TRANSPORT_SECURITY, From 071eb9eb76bc9905bf5c39447371a568bc3f9581 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:56:36 +0100 Subject: [PATCH 06/28] explicit dependencie on timekeeper getter --- src/server/routes/turnilo/turnilo.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/server/routes/turnilo/turnilo.ts b/src/server/routes/turnilo/turnilo.ts index 6ebd35b44..e6b02d811 100644 --- a/src/server/routes/turnilo/turnilo.ts +++ b/src/server/routes/turnilo/turnilo.ts @@ -18,10 +18,11 @@ import { Request, Response, Router } from "express"; import { AppSettings } from "../../../common/models/app-settings/app-settings"; import { getTitle } from "../../../common/models/customization/customization"; -import { SETTINGS_MANAGER } from "../../config"; +import { Timekeeper } from "../../../common/models/timekeeper/timekeeper"; +import { Nullary } from "../../../common/utils/functional/functional"; import { mainLayout } from "../../views"; -export function turniloRouter(appSettings: AppSettings, version: string) { +export function turniloRouter(appSettings: AppSettings, getTimekeeper: Nullary, version: string) { const router = Router(); @@ -31,7 +32,7 @@ export function turniloRouter(appSettings: AppSettings, version: string) { version, title: getTitle(appSettings.customization, version), appSettings, - timekeeper: SETTINGS_MANAGER.getTimekeeper() + timekeeper: getTimekeeper() })); } catch (e) { res.status(400).send({ error: "Couldn't load Turnilo Application" }); From 595f97d4bc698a64b9a086b79ca39b5f501a7cc6 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:56:55 +0100 Subject: [PATCH 07/28] we always post process loaded files so enforce that intypes --- src/server/utils/file/file.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/server/utils/file/file.ts b/src/server/utils/file/file.ts index 37827ea7a..91a0306ad 100644 --- a/src/server/utils/file/file.ts +++ b/src/server/utils/file/file.ts @@ -18,13 +18,12 @@ import * as fs from "fs"; import * as yaml from "js-yaml"; -export function loadFileSync(filepath: string, postProcess: string = null): any { - var fileData = fs.readFileSync(filepath, "utf-8"); - if (postProcess === "json") { - fileData = JSON.parse(fileData); - } else if (postProcess === "yaml") { - fileData = yaml.safeLoad(fileData); +export function loadFileSync(filepath: string, postProcess: "json" | "yaml"): object { + const fileData = fs.readFileSync(filepath, "utf-8"); + switch (postProcess) { + case "json": + return JSON.parse(fileData); + case "yaml": + return yaml.safeLoad(fileData); } - - return fileData; } From 728b892214429900a321f104c9318919ce6ca4d4 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:57:18 +0100 Subject: [PATCH 08/28] create express app in function with explicit dependencies --- src/server/app.ts | 215 +++++++++++++++++++++++----------------------- 1 file changed, 109 insertions(+), 106 deletions(-) diff --git a/src/server/app.ts b/src/server/app.ts index a54a03950..47071378a 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -17,13 +17,12 @@ import * as bodyParser from "body-parser"; import compress from "compression"; -import express from "express"; +import express, { Express } from "express"; import { Handler, Request, Response, Router } from "express"; import { hsts } from "helmet"; import { join } from "path"; -import { LOGGER } from "../common/logger/logger"; -import { SERVER_SETTINGS, SETTINGS_MANAGER, VERSION } from "./config"; import { PluginSettings } from "./models/plugin-settings/plugin-settings"; +import { ServerSettings } from "./models/server-settings/server-settings"; import { livenessRouter } from "./routes/liveness/liveness"; import { mkurlRouter } from "./routes/mkurl/mkurl"; import { plyqlRouter } from "./routes/plyql/plyql"; @@ -33,6 +32,7 @@ import { shortenRouter } from "./routes/shorten/shorten"; import { sourcesRouter } from "./routes/sources/sources"; import { turniloRouter } from "./routes/turnilo/turnilo"; import { loadPlugin } from "./utils/plugin-loader/load-plugin"; +import { SettingsManager } from "./utils/settings-manager/settings-manager"; import { errorLayout } from "./views"; declare module "express" { @@ -41,128 +41,131 @@ declare module "express" { } } -let app = express(); -app.disable("x-powered-by"); +export default function createApp(serverSettings: ServerSettings, settingsManager: SettingsManager, version: string): Express { -const isDev = app.get("env") === "development"; -const isTrustedProxy = SERVER_SETTINGS.trustProxy === "always"; + let app = express(); + app.disable("x-powered-by"); -if (isTrustedProxy) { - app.set("trust proxy", true); // trust X-Forwarded-*, use left-most entry as the client -} - -const timeout = SERVER_SETTINGS.serverTimeout; -app.use((req, res, next) => { - res.setTimeout(timeout); - next(); -}); + const isDev = app.get("env") === "development"; + const isTrustedProxy = serverSettings.trustProxy === "always"; -function getRoutePath(route: string): string { - const serverRoot = SERVER_SETTINGS.serverRoot; - const prefix = serverRoot.length > 0 ? `/${serverRoot}` : ""; - return `${prefix}${route}`; -} + if (isTrustedProxy) { + app.set("trust proxy", true); // trust X-Forwarded-*, use left-most entry as the client + } -function attachRouter(route: string, router: Router | Handler): void { - app.use(getRoutePath(route), router); -} + const timeout = serverSettings.serverTimeout; + app.use((req, res, next) => { + res.setTimeout(timeout); + next(); + }); -// Add compression -app.use(compress()); + function getRoutePath(route: string): string { + const serverRoot = serverSettings.serverRoot; + const prefix = serverRoot.length > 0 ? `/${serverRoot}` : ""; + return `${prefix}${route}`; + } -// Add Strict Transport Security -if (SERVER_SETTINGS.strictTransportSecurity === "always") { - app.use(hsts({ - maxAge: 10886400000, // Must be at least 18 weeks to be approved by Google - includeSubdomains: true, // Must be enabled to be approved by Google - preload: true - })); -} + function attachRouter(route: string, router: Router | Handler): void { + app.use(getRoutePath(route), router); + } -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: true })); + // Add compression + app.use(compress()); -if (SERVER_SETTINGS.iframe === "deny") { - app.use((req: Request, res: Response, next: Function) => { - res.setHeader("X-Frame-Options", "DENY"); - res.setHeader("Content-Security-Policy", "frame-ancestors 'none'"); - next(); - }); -} - -app.use((req: Request, res: Response, next: Function) => { - req.turniloMetadata = {}; - next(); -}); - -SERVER_SETTINGS.plugins.forEach(({ path, name, settings }: PluginSettings) => { - try { - LOGGER.log(`Loading plugin ${name} module`); - const module = loadPlugin(path, SETTINGS_MANAGER.anchorPath); - LOGGER.log(`Invoking plugin ${name}`); - module.plugin(app, - settings, - SERVER_SETTINGS, - SETTINGS_MANAGER.appSettings, - SETTINGS_MANAGER.sourcesGetter, - LOGGER.addPrefix(name)); - } catch (e) { - LOGGER.warn(`Plugin ${name} threw an error: ${e.message}`); + // Add Strict Transport Security + if (serverSettings.strictTransportSecurity === "always") { + app.use(hsts({ + maxAge: 10886400000, // Must be at least 18 weeks to be approved by Google + includeSubdomains: true, // Must be enabled to be approved by Google + preload: true + })); } -}); -// development HMR -if (app.get("env") === "dev-hmr") { - // add hot module replacement + app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ extended: true })); - const webpack = require("webpack"); - const webpackConfig = require("../../config/webpack.dev"); - const webpackDevMiddleware = require("webpack-dev-middleware"); - const webpackHotMiddleware = require("webpack-hot-middleware"); + if (serverSettings.iframe === "deny") { + app.use((req: Request, res: Response, next: Function) => { + res.setHeader("X-Frame-Options", "DENY"); + res.setHeader("Content-Security-Policy", "frame-ancestors 'none'"); + next(); + }); + } - if (webpack && webpackDevMiddleware && webpackHotMiddleware) { - const webpackCompiler = webpack(webpackConfig); + app.use((req: Request, res: Response, next: Function) => { + req.turniloMetadata = {}; + next(); + }); - app.use(webpackDevMiddleware(webpackCompiler, { - hot: true, - noInfo: true, - publicPath: webpackConfig.output.publicPath - })); + serverSettings.plugins.forEach(({ path, name, settings }: PluginSettings) => { + try { + settingsManager.logger.log(`Loading plugin ${name} module`); + const module = loadPlugin(path, settingsManager.anchorPath); + settingsManager.logger.log(`Invoking plugin ${name}`); + module.plugin(app, + settings, + serverSettings, + settingsManager.appSettings, + settingsManager.sourcesGetter, + settingsManager.logger.addPrefix(name)); + } catch (e) { + settingsManager.logger.warn(`Plugin ${name} threw an error: ${e.message}`); + } + }); - app.use(webpackHotMiddleware(webpackCompiler, { - log: console.log, - path: "/__webpack_hmr" - })); + // development HMR + if (app.get("env") === "dev-hmr") { + // add hot module replacement + + const webpack = require("webpack"); + const webpackConfig = require("../../config/webpack.dev"); + const webpackDevMiddleware = require("webpack-dev-middleware"); + const webpackHotMiddleware = require("webpack-hot-middleware"); + + if (webpack && webpackDevMiddleware && webpackHotMiddleware) { + const webpackCompiler = webpack(webpackConfig); + + app.use(webpackDevMiddleware(webpackCompiler, { + hot: true, + noInfo: true, + publicPath: webpackConfig.output.publicPath + })); + + app.use(webpackHotMiddleware(webpackCompiler, { + log: console.log, + path: "/__webpack_hmr" + })); + } } -} -attachRouter("/", express.static(join(__dirname, "../../build/public"))); -attachRouter("/", express.static(join(__dirname, "../../assets"))); + attachRouter("/", express.static(join(__dirname, "../../build/public"))); + attachRouter("/", express.static(join(__dirname, "../../assets"))); -attachRouter(SERVER_SETTINGS.readinessEndpoint, readinessRouter(SETTINGS_MANAGER.sourcesGetter)); -attachRouter(SERVER_SETTINGS.livenessEndpoint, livenessRouter); + attachRouter(serverSettings.readinessEndpoint, readinessRouter(settingsManager.sourcesGetter)); + attachRouter(serverSettings.livenessEndpoint, livenessRouter); -// Data routes -attachRouter("/sources", sourcesRouter(SETTINGS_MANAGER.sourcesGetter)); -attachRouter("/plywood", plywoodRouter(SETTINGS_MANAGER)); -attachRouter("/plyql", plyqlRouter(SETTINGS_MANAGER.sourcesGetter)); -attachRouter("/mkurl", mkurlRouter(SETTINGS_MANAGER.sourcesGetter)); -attachRouter("/shorten", shortenRouter(SETTINGS_MANAGER.appSettings, isTrustedProxy)); + // Data routes + attachRouter("/sources", sourcesRouter(settingsManager.sourcesGetter)); + attachRouter("/plywood", plywoodRouter(settingsManager)); + attachRouter("/plyql", plyqlRouter(settingsManager.sourcesGetter)); + attachRouter("/mkurl", mkurlRouter(settingsManager.sourcesGetter)); + attachRouter("/shorten", shortenRouter(settingsManager.appSettings, isTrustedProxy)); -attachRouter("/", turniloRouter(SETTINGS_MANAGER.appSettings, VERSION)); + attachRouter("/", turniloRouter(settingsManager.appSettings, settingsManager.getTimekeeper, version)); -// Catch 404 and redirect to / -app.use((req: Request, res: Response) => { - res.redirect(getRoutePath("/")); -}); + // Catch 404 and redirect to / + app.use((req: Request, res: Response) => { + res.redirect(getRoutePath("/")); + }); -app.use((err: any, req: Request, res: Response, next: Function) => { - LOGGER.error(`Server Error: ${err.message}`); - LOGGER.error(err.stack); - res.status(err.status || 500); - // no stacktraces leaked to user - const error = isDev ? err : null; - res.send(errorLayout({ version: VERSION, title: "Error" }, err.message, error)); -}); + app.use((err: any, req: Request, res: Response, next: Function) => { + settingsManager.logger.error(`Server Error: ${err.message}`); + settingsManager.logger.error(err.stack); + res.status(err.status || 500); + // no stacktraces leaked to user + const error = isDev ? err : null; + res.send(errorLayout({ version, title: "Error" }, err.message, error)); + }); -export default app; + return app; +} From e15686e6c9f16c64c1fbd60690f8b1354919900f Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Thu, 3 Nov 2022 17:57:39 +0100 Subject: [PATCH 09/28] implement commands --- src/server/cli.ts | 301 ++++++++++++++++++++++++++++--------------- src/server/config.ts | 9 +- 2 files changed, 203 insertions(+), 107 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 88a7cb26e..b9bc87a36 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -14,154 +14,251 @@ * limitations under the License. */ -import { InvalidArgumentError, Option, program } from "commander"; +import { Option, program } from "commander"; +import path from "path"; +import { LOGGER } from "../common/logger/logger"; +import { EMPTY_APP_SETTINGS, fromConfig as appSettingsFromConfig } from "../common/models/app-settings/app-settings"; +import { fromConfig as clusterFromConfig } from "../common/models/cluster/cluster"; +import { fromConfig as dataCubeFromConfig } from "../common/models/data-cube/data-cube"; +import { fromConfig as sourcesFromConfig, Sources } from "../common/models/sources/sources"; +import { appSettingsToYaml, printExtra, sourcesToYaml } from "../common/utils/yaml-helper/yaml-helper"; +import createApp from "./app"; +import createServer from "./cli/create-server"; +import { loadConfigFile } from "./cli/load-config-file"; +import { assertCredentials, parseInteger } from "./cli/utils"; +import { ServerSettings, ServerSettingsJS } from "./models/server-settings/server-settings"; +import { SettingsManager } from "./utils/settings-manager/settings-manager"; import { VERSION } from "./version"; -function assertIntegerOption(value: string): number { - const parsed = parseInt(value, 10); - if (isNaN(parsed)) throw new InvalidArgumentError("Must be an integer"); - return parsed; -} +const portOption = new Option("-p, --port ", "port number").argParser(parseInteger); +const serverRootOption = new Option("--server-root ", "server root"); +const serverHostOption = new Option("--server-host ", "server host"); +const verboseOption = new Option("--verbose", "verbose mode"); +const usernameOption = new Option("--username ", "username"); +const passwordOption = new Option("--password ", "password"); -const port = new Option("-p, --port ", "port number").argParser(assertIntegerOption).default(900); // TODO: DEFAULT_PORT -const serverRoot = new Option("--server-root ", "server root").default(""); // TODO: DEFAULT_ROOT -const serverHost = new Option("--server-host ", "server host").default(null); // TODO: DEFAULT_HOST -const verbose = new Option("--verbose", "verbose mode").default(false); -const username = new Option("--username ", "username"); -const password = new Option("--password ", "password"); - -function assertCredentials(username: string | undefined, password: string | undefined) { - if (password && !username || username && !password) { - program.error("You need to pass both username and password"); - } -} +const version = VERSION; program .name("turnilo") .description("Turnilo is a data exploration tool that connects to Druid database") - .version(VERSION, "--version"); + .version(version, "--version"); program .command("run-config") .argument("", "path of config file") - .addOption(port) - .addOption(serverRoot) - .addOption(serverHost) - .addOption(username) - .addOption(password) - .addOption(verbose) + .addOption(portOption) + .addOption(serverRootOption) + .addOption(serverHostOption) + .addOption(usernameOption) + .addOption(passwordOption) + .addOption(verboseOption) .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { assertCredentials(username, password); - /* - 1. load yml file as config - 2. override config with all existing options - 3. create ServerSettings from 2 - 4. create AppSettings and Sources from 2 - 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - 6. start ./app.ts with ServerSettings and code from ./www.ts - */ - console.log("run config with", configPath, "and options", { port, verbose }); + // 1. load yml file as config + const config = loadConfigFile(configPath); + // 2. override config with all existing options + const serverSettingsJS: ServerSettingsJS = { + ...config, + port, + verbose, + serverHost, + serverRoot + }; + // 3. create ServerSettings from 2 + const serverSettings = ServerSettings.fromJS(serverSettingsJS); + // 4. create AppSettings and Sources from 2 + // TODO: pass credentials somewhere + const appSettings = appSettingsFromConfig(config); + const sources = sourcesFromConfig(config); + // 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: path.dirname(configPath), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + // 6. start ./app.ts with ServerSettings and code from ./www.ts + createServer(serverSettings, createApp(serverSettings, settingsManager, version)); + console.log("run config with", configPath, "and options"); }); program .command("run-examples") - .addOption(port) - .addOption(serverRoot) - .addOption(serverHost) - .addOption(verbose) + .addOption(portOption) + .addOption(serverRootOption) + .addOption(serverHostOption) + .addOption(verboseOption) .action(({ port, verbose, serverRoot, serverHost }) => { - /* - 1. load config-examples.yaml file as config - 2. override config with all existing options - 3. create ServerSettings from 2 - 4. create AppSettings and Sources from 2 - 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - 6. start ./app.ts with ServerSettings and code from ./www.ts - */ - console.log("run examples with options", { port, verbose }); + // 1. load config-examples.yaml file as config + const configPath = path.join(__dirname, "../../config-examples.yaml"); + const config = loadConfigFile(configPath); + // 2. override config with all existing options + const serverSettingsJS: ServerSettingsJS = { + ...config, + port, + verbose, + serverHost, + serverRoot + }; + // 3. create ServerSettings from 2 + const serverSettings = ServerSettings.fromJS(serverSettingsJS); + // 4. create AppSettings and Sources from 2 + const appSettings = appSettingsFromConfig(config); + const sources = sourcesFromConfig(config); + // 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: path.dirname(configPath), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + // 6. start ./app.ts with ServerSettings and code from ./www.ts + createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); program .command("connect-druid") .argument("", "druid url") - .addOption(port) - .addOption(serverRoot) - .addOption(serverHost) - .addOption(verbose) - .option("--username ", "username") - .option("--password ", "password") - .action((url, { port, verbose, username, password }) => { + .addOption(portOption) + .addOption(serverRootOption) + .addOption(serverHostOption) + .addOption(verboseOption) + .addOption(usernameOption) + .addOption(passwordOption) + .action((url, { port, verbose, username, password, serverRoot, serverHost }) => { assertCredentials(username, password); - /* - 1. create cluster from name: druid and passed url and empty dataCubes list - 2. create appSettings from EMPTY_APP_SETTINGS - 3. create ServerSettings from options - 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - 5. start ./app.ts with ServerSettings and code from ./www.ts - */ - console.log("connect to", url, "with options", { port, verbose, username, password }); + // 1. create cluster from name: druid and passed url and empty dataCubes list + // TODO: pass credentials somewhere + const sources: Sources = { + dataCubes: [], + clusters: [clusterFromConfig({ + name: "druid", + url + })] + }; + // 2. create appSettings from EMPTY_APP_SETTINGS + const appSettings = EMPTY_APP_SETTINGS; + // 3. create ServerSettings from options + const serverSettings = ServerSettings.fromJS({ serverRoot, serverHost, port, verbose }); + // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: process.cwd(), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + // 5. start ./app.ts with ServerSettings and code from ./www.ts + createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); program .command("load-file") .argument("", "json file") - .addOption(port) - .addOption(serverRoot) - .addOption(serverHost) - .addOption(verbose) + .addOption(portOption) + .addOption(serverRootOption) + .addOption(serverHostOption) + .addOption(verboseOption) .action((file, { port, verbose, serverHost, serverRoot }) => { - /* - 1. create empty cluster list and dataCube: - dataCubeFromConfig({ + // 1. create empty cluster list and dataCube: + // dataCubeFromConfig({ + // name: path.basename(file, path.extname(file)), + // clusterName: "native", + // source: file + // }, undefined + const sources: Sources = { + dataCubes: [dataCubeFromConfig({ name: path.basename(file, path.extname(file)), clusterName: "native", source: file - }, undefined - 2. create appSettings from EMPTY_APP_SETTINGS - 3. create ServerSettings from options - 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - 5. start ./app.ts with ServerSettings and code from ./www.ts - */ - console.log("load file", file, "with options", { port, verbose, serverHost, serverRoot }); + }, undefined)], + clusters: [] + }; + // 2. create appSettings from EMPTY_APP_SETTINGS + const appSettings = EMPTY_APP_SETTINGS; + // 3. create ServerSettings from options + const serverSettings = ServerSettings.fromJS({ serverRoot, serverHost, port, verbose }); + // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: process.cwd(), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + // 5. start ./app.ts with ServerSettings and code from ./www.ts + createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); program .command("verify-config") .argument("", "path to config file") - .addOption(verbose) + .addOption(verboseOption) .action(file => { - /* - 1. load yaml - 2. create server settings - 3. create app settings - 4. create sources - 5. catch and log all errors - 6. return status code - */ - console.log("verify config from", file); + // 1. load yaml + const config = loadConfigFile(file); + try { + // 2. create server settings + const serverSettings = ServerSettings.fromJS(config); + // 3. create app settings + const appSettings = appSettingsFromConfig(config); + // 4. create sources + const sources = sourcesFromConfig(config); + // 5. catch and log all errors + } catch (e) { + program.error("Config verification error: ", e.message); + } }); program .command("introspect-druid") .argument("", "druid url") - .addOption(port) - .addOption(serverRoot) - .addOption(serverHost) - .addOption(verbose) - .addOption(username) - .addOption(password) + .addOption(verboseOption) + .addOption(usernameOption) + .addOption(passwordOption) .option("--with-comments", "print comments") - .action((url, { port, verbose, username, password, withComments }) => { + .action((url, { verbose, username, password, withComments }) => { assertCredentials(username, password); - /* - 1. create cluster from name: druid and passed url and empty dataCubes list - 2. create appSettings from EMPTY_APP_SETTINGS - 3. create ServerSettings from options - 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - 5. call SettingsManager.getFreshSources with timeout 1000 - 6. using yaml-helper print out appSettings, fetched sources and "Extra" - */ - console.log("introspect ", url, "with options", { port, verbose, username, password, withComments }); + // 1. create cluster from name: druid and passed url and empty dataCubes list + // TODO: pass credentials somewhere + const sources: Sources = { + dataCubes: [], + clusters: [clusterFromConfig({ + name: "druid", + url + })] + }; + // 2. create appSettings from EMPTY_APP_SETTINGS + const appSettings = EMPTY_APP_SETTINGS; + // 3. create ServerSettings from options + const serverSettings = ServerSettings.fromJS({ verbose }); + // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: process.cwd(), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + // 5. call SettingsManager.getFreshSources with timeout 1000 + // 6. using yaml-helper print out appSettings, fetched sources and "Extra" + settingsManager.getFreshSources({ + timeout: 10000 + }).then(sources => { + const extra = { + header: true, + version: VERSION, + verbose + // Why port here? We don't start server so port is meaningless + // port: SERVER_SETTINGS.port + }; + const config = [ + printExtra(extra, withComments), + appSettingsToYaml(appSettings, withComments), + sourcesToYaml(sources, withComments) + ].join("\n"); + process.stdout.write(config, () => process.exit()); + }).catch((e: Error) => { + program.error("There was an error generating a config: " + e.message); + }); }); program.showHelpAfterError(); diff --git a/src/server/config.ts b/src/server/config.ts index 87e1b480d..fbe889627 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -171,7 +171,7 @@ if (numSettingsInputs > 1) { process.exit(1); } -export const PRINT_CONFIG = Boolean(parsedArgs["print-config"]); +const PRINT_CONFIG = Boolean(parsedArgs["print-config"]); export const START_SERVER = !PRINT_CONFIG; const logger = START_SERVER ? LOGGER : NULL_LOGGER; @@ -212,8 +212,7 @@ if (parsedArgs["verbose"]) { serverSettingsJS.verbose = parsedArgs["verbose"]; } -// TODO: Remove this export -export const VERBOSE = Boolean(serverSettingsJS.verbose); +const verbose = Boolean(serverSettingsJS.verbose); export const SERVER_SETTINGS = ServerSettings.fromJS(serverSettingsJS); // --- Sign of Life ------------------------------- @@ -254,7 +253,7 @@ const { appSettings, sources } = configContent export const SETTINGS_MANAGER = new SettingsManager(appSettings, sources, { logger, - verbose: VERBOSE, + verbose, anchorPath: configDirPath, initialLoadTimeout: SERVER_SETTINGS.pageMustLoadTimeout }); @@ -270,7 +269,7 @@ if (PRINT_CONFIG) { const extra = { header: true, version: VERSION, - verbose: VERBOSE, + verbose, port: SERVER_SETTINGS.port }; const config = [ From 42dcf79b2a1cce7b57da06f021952a1975fbefd5 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 12:15:00 +0100 Subject: [PATCH 10/28] be careful to what "this" is bound --- src/server/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/app.ts b/src/server/app.ts index 47071378a..fabed025c 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -151,7 +151,7 @@ export default function createApp(serverSettings: ServerSettings, settingsManage attachRouter("/mkurl", mkurlRouter(settingsManager.sourcesGetter)); attachRouter("/shorten", shortenRouter(settingsManager.appSettings, isTrustedProxy)); - attachRouter("/", turniloRouter(settingsManager.appSettings, settingsManager.getTimekeeper, version)); + attachRouter("/", turniloRouter(settingsManager.appSettings, () => settingsManager.getTimekeeper(), version)); // Catch 404 and redirect to / app.use((req: Request, res: Response) => { From 6bf4d0798e9f508dedc8690ce0a272a6616f78f4 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 12:24:30 +0100 Subject: [PATCH 11/28] use relative path because sources endpoint is placed on server-root directory not a file system root --- src/client/utils/ajax/ajax.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/utils/ajax/ajax.ts b/src/client/utils/ajax/ajax.ts index 610d0625a..96728c111 100644 --- a/src/client/utils/ajax/ajax.ts +++ b/src/client/utils/ajax/ajax.ts @@ -75,7 +75,7 @@ export class Ajax { static sources(appSettings: ClientAppSettings): Promise { const headers = Ajax.headers(appSettings.oauth); - return axios.get("/sources", { headers }) + return axios.get("sources", { headers }) .then(resp => resp.data) .catch(error => { throw mapOauthError(appSettings.oauth, error); From 03aadceff261b7641a7ad69489eebe2c50f1484d Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 12:44:31 +0100 Subject: [PATCH 12/28] add required option for time attribute for json files --- src/server/cli.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index b9bc87a36..7178bccc1 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -155,11 +155,12 @@ program program .command("load-file") .argument("", "json file") + .requiredOption("-t, --time-attribute ", "time attribute") .addOption(portOption) .addOption(serverRootOption) .addOption(serverHostOption) .addOption(verboseOption) - .action((file, { port, verbose, serverHost, serverRoot }) => { + .action((file, { timeAttribute, port, verbose, serverHost, serverRoot }) => { // 1. create empty cluster list and dataCube: // dataCubeFromConfig({ // name: path.basename(file, path.extname(file)), @@ -170,7 +171,8 @@ program dataCubes: [dataCubeFromConfig({ name: path.basename(file, path.extname(file)), clusterName: "native", - source: file + source: file, + timeAttribute }, undefined)], clusters: [] }; From 5a7ee5b3fda0759cb1abb3cc0591d003d1b0e723 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 12:45:00 +0100 Subject: [PATCH 13/28] columns from json files are loaded later so we can't check for time column at the start --- src/common/models/data-cube/data-cube.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/common/models/data-cube/data-cube.ts b/src/common/models/data-cube/data-cube.ts index a6ab40f77..8a46c6b4d 100644 --- a/src/common/models/data-cube/data-cube.ts +++ b/src/common/models/data-cube/data-cube.ts @@ -302,10 +302,6 @@ function readTimeAttribute(config: DataCubeJS, cluster: Cluster | undefined, dim throw new Error(`DataCube "${config.name}" must have defined timeAttribute property`); } const timeAttribute = $(config.timeAttribute); - const timeDimension = findDimensionByExpression(dimensions, timeAttribute); - if (!isTruthy(timeDimension)) { - throw new Error(`In DataCube "${config.name}" could not find dimension for supplied timeAttribute "${config.timeAttribute}" (must match)`); - } return { timeAttribute, dimensions From d9cb767ad56504d141b1cd70e95eb9862e1387a8 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 12:53:35 +0100 Subject: [PATCH 14/28] add verbose option for config verification --- src/server/cli.ts | 12 ++++++------ src/server/cli/load-config-file.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 7178bccc1..e4dd7dac3 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -195,16 +195,16 @@ program .command("verify-config") .argument("", "path to config file") .addOption(verboseOption) - .action(file => { + .action((file, { verbose }) => { // 1. load yaml const config = loadConfigFile(file); try { // 2. create server settings - const serverSettings = ServerSettings.fromJS(config); - // 3. create app settings - const appSettings = appSettingsFromConfig(config); - // 4. create sources - const sources = sourcesFromConfig(config); + const serverSettings = ServerSettings.fromJS({ ...config, verbose }); + // 3. create app settings + const appSettings = appSettingsFromConfig(config); + // 4. create sources + const sources = sourcesFromConfig(config); // 5. catch and log all errors } catch (e) { program.error("Config verification error: ", e.message); diff --git a/src/server/cli/load-config-file.ts b/src/server/cli/load-config-file.ts index aa0487ff8..330b8df55 100644 --- a/src/server/cli/load-config-file.ts +++ b/src/server/cli/load-config-file.ts @@ -19,6 +19,6 @@ export function loadConfigFile(configPath: string): object { try { return loadFileSync(configPath, "yaml"); } catch (e) { - throw new Error("SOMETHING"); // something that commander can handle + throw new Error("SOMETHING"); // TODO: something that commander can handle } } From 8a260049088934bd453407680d34c9a1f7af7136 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 14:32:43 +0100 Subject: [PATCH 15/28] refactor a little --- src/server/cli.ts | 145 ++++----------------------- src/server/cli/build-settings.ts | 97 ++++++++++++++++++ src/server/cli/introspect-cluster.ts | 55 ++++++++++ 3 files changed, 172 insertions(+), 125 deletions(-) create mode 100644 src/server/cli/build-settings.ts create mode 100644 src/server/cli/introspect-cluster.ts diff --git a/src/server/cli.ts b/src/server/cli.ts index e4dd7dac3..38b559acb 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -17,16 +17,12 @@ import { Option, program } from "commander"; import path from "path"; import { LOGGER } from "../common/logger/logger"; -import { EMPTY_APP_SETTINGS, fromConfig as appSettingsFromConfig } from "../common/models/app-settings/app-settings"; -import { fromConfig as clusterFromConfig } from "../common/models/cluster/cluster"; -import { fromConfig as dataCubeFromConfig } from "../common/models/data-cube/data-cube"; -import { fromConfig as sourcesFromConfig, Sources } from "../common/models/sources/sources"; -import { appSettingsToYaml, printExtra, sourcesToYaml } from "../common/utils/yaml-helper/yaml-helper"; import createApp from "./app"; +import buildSettings, { settingsForDatasetFile, settingsForDruidConnection } from "./cli/build-settings"; import createServer from "./cli/create-server"; +import printIntrospectedSettings from "./cli/introspect-cluster"; import { loadConfigFile } from "./cli/load-config-file"; import { assertCredentials, parseInteger } from "./cli/utils"; -import { ServerSettings, ServerSettingsJS } from "./models/server-settings/server-settings"; import { SettingsManager } from "./utils/settings-manager/settings-manager"; import { VERSION } from "./version"; @@ -54,33 +50,18 @@ program .addOption(passwordOption) .addOption(verboseOption) .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { + const anchorPath = path.dirname(configPath); assertCredentials(username, password); - // 1. load yml file as config const config = loadConfigFile(configPath); - // 2. override config with all existing options - const serverSettingsJS: ServerSettingsJS = { - ...config, - port, - verbose, - serverHost, - serverRoot - }; - // 3. create ServerSettings from 2 - const serverSettings = ServerSettings.fromJS(serverSettingsJS); - // 4. create AppSettings and Sources from 2 // TODO: pass credentials somewhere - const appSettings = appSettingsFromConfig(config); - const sources = sourcesFromConfig(config); - // 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }); const settingsManager = new SettingsManager(appSettings, sources, { - anchorPath: path.dirname(configPath), + anchorPath, initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, logger: LOGGER }); - // 6. start ./app.ts with ServerSettings and code from ./www.ts createServer(serverSettings, createApp(serverSettings, settingsManager, version)); - console.log("run config with", configPath, "and options"); }); program @@ -90,30 +71,16 @@ program .addOption(serverHostOption) .addOption(verboseOption) .action(({ port, verbose, serverRoot, serverHost }) => { - // 1. load config-examples.yaml file as config const configPath = path.join(__dirname, "../../config-examples.yaml"); + const anchorPath = path.dirname(configPath); const config = loadConfigFile(configPath); - // 2. override config with all existing options - const serverSettingsJS: ServerSettingsJS = { - ...config, - port, - verbose, - serverHost, - serverRoot - }; - // 3. create ServerSettings from 2 - const serverSettings = ServerSettings.fromJS(serverSettingsJS); - // 4. create AppSettings and Sources from 2 - const appSettings = appSettingsFromConfig(config); - const sources = sourcesFromConfig(config); - // 5. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const { sources, serverSettings, appSettings } = buildSettings(config, { port, verbose, serverHost, serverRoot }); const settingsManager = new SettingsManager(appSettings, sources, { - anchorPath: path.dirname(configPath), + anchorPath, initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, logger: LOGGER }); - // 6. start ./app.ts with ServerSettings and code from ./www.ts createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); @@ -128,27 +95,14 @@ program .addOption(passwordOption) .action((url, { port, verbose, username, password, serverRoot, serverHost }) => { assertCredentials(username, password); - // 1. create cluster from name: druid and passed url and empty dataCubes list // TODO: pass credentials somewhere - const sources: Sources = { - dataCubes: [], - clusters: [clusterFromConfig({ - name: "druid", - url - })] - }; - // 2. create appSettings from EMPTY_APP_SETTINGS - const appSettings = EMPTY_APP_SETTINGS; - // 3. create ServerSettings from options - const serverSettings = ServerSettings.fromJS({ serverRoot, serverHost, port, verbose }); - // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { port, verbose, serverHost, serverRoot }); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath: process.cwd(), initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, logger: LOGGER }); - // 5. start ./app.ts with ServerSettings and code from ./www.ts createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); @@ -161,33 +115,13 @@ program .addOption(serverHostOption) .addOption(verboseOption) .action((file, { timeAttribute, port, verbose, serverHost, serverRoot }) => { - // 1. create empty cluster list and dataCube: - // dataCubeFromConfig({ - // name: path.basename(file, path.extname(file)), - // clusterName: "native", - // source: file - // }, undefined - const sources: Sources = { - dataCubes: [dataCubeFromConfig({ - name: path.basename(file, path.extname(file)), - clusterName: "native", - source: file, - timeAttribute - }, undefined)], - clusters: [] - }; - // 2. create appSettings from EMPTY_APP_SETTINGS - const appSettings = EMPTY_APP_SETTINGS; - // 3. create ServerSettings from options - const serverSettings = ServerSettings.fromJS({ serverRoot, serverHost, port, verbose }); - // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) + const { appSettings, sources, serverSettings } = settingsForDatasetFile(file, timeAttribute, { serverRoot, serverHost, verbose, port }); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath: process.cwd(), initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, logger: LOGGER }); - // 5. start ./app.ts with ServerSettings and code from ./www.ts createServer(serverSettings, createApp(serverSettings, settingsManager, version)); }); @@ -196,16 +130,9 @@ program .argument("", "path to config file") .addOption(verboseOption) .action((file, { verbose }) => { - // 1. load yaml - const config = loadConfigFile(file); try { - // 2. create server settings - const serverSettings = ServerSettings.fromJS({ ...config, verbose }); - // 3. create app settings - const appSettings = appSettingsFromConfig(config); - // 4. create sources - const sources = sourcesFromConfig(config); - // 5. catch and log all errors + const config = loadConfigFile(file); + buildSettings(config, { verbose }); } catch (e) { program.error("Config verification error: ", e.message); } @@ -217,48 +144,16 @@ program .addOption(verboseOption) .addOption(usernameOption) .addOption(passwordOption) - .option("--with-comments", "print comments") - .action((url, { verbose, username, password, withComments }) => { + .action((url, { verbose, username, password }) => { assertCredentials(username, password); - // 1. create cluster from name: druid and passed url and empty dataCubes list // TODO: pass credentials somewhere - const sources: Sources = { - dataCubes: [], - clusters: [clusterFromConfig({ - name: "druid", - url - })] - }; - // 2. create appSettings from EMPTY_APP_SETTINGS - const appSettings = EMPTY_APP_SETTINGS; - // 3. create ServerSettings from options - const serverSettings = ServerSettings.fromJS({ verbose }); - // 4. create SettingsManager with 4, logger, verbose option, anchorPath (config path) and initialLoadTimeout (ServerSettings.pageMustLoadTimeout) - const settingsManager = new SettingsManager(appSettings, sources, { - anchorPath: process.cwd(), - initialLoadTimeout: serverSettings.pageMustLoadTimeout, - verbose, - logger: LOGGER - }); - // 5. call SettingsManager.getFreshSources with timeout 1000 - // 6. using yaml-helper print out appSettings, fetched sources and "Extra" - settingsManager.getFreshSources({ - timeout: 10000 - }).then(sources => { - const extra = { - header: true, - version: VERSION, - verbose - // Why port here? We don't start server so port is meaningless - // port: SERVER_SETTINGS.port - }; - const config = [ - printExtra(extra, withComments), - appSettingsToYaml(appSettings, withComments), - sourcesToYaml(sources, withComments) - ].join("\n"); - process.stdout.write(config, () => process.exit()); - }).catch((e: Error) => { + const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { verbose }); + printIntrospectedSettings( + serverSettings, + appSettings, + sources, + verbose + ).catch((e: Error) => { program.error("There was an error generating a config: " + e.message); }); }); diff --git a/src/server/cli/build-settings.ts b/src/server/cli/build-settings.ts new file mode 100644 index 000000000..9685d3eff --- /dev/null +++ b/src/server/cli/build-settings.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AppSettings, + EMPTY_APP_SETTINGS, + fromConfig as appSettingsFromConfig +} from "../../common/models/app-settings/app-settings"; +import { fromConfig as sourcesFromConfig, Sources } from "../../common/models/sources/sources"; +import { ServerSettings, ServerSettingsJS } from "../models/server-settings/server-settings"; +import { fromConfig as clusterFromConfig } from "../../common/models/cluster/cluster"; +import { fromConfig as dataCubeFromConfig } from "../../common/models/data-cube/data-cube"; +import path from "path"; + +interface Settings { + serverSettings: ServerSettings; + appSettings: AppSettings; + sources: Sources; +} + +interface Options { + port?: number; + verbose?: boolean; + serverHost?: string; + serverRoot?: string; +} + +export default function buildSettings(config: object, options: Options = {}): Settings { + const serverSettingsJS: ServerSettingsJS = { + ...config, + ...options + }; + // 3. create ServerSettings from 2 + const serverSettings = ServerSettings.fromJS(serverSettingsJS); + // 4. create AppSettings and Sources from 2 + const appSettings = appSettingsFromConfig(config); + const sources = sourcesFromConfig(config); + + return { + serverSettings, + appSettings, + sources + }; +} + +export function settingsForDruidConnection(url: string, options: Options = {}): Settings { + // TODO: pass credentials somewhere + const sources: Sources = { + dataCubes: [], + clusters: [clusterFromConfig({ + name: "druid", + url + })] + }; + const appSettings = EMPTY_APP_SETTINGS; + const serverSettings = ServerSettings.fromJS(options); + + return { + sources, + appSettings, + serverSettings + }; +} + +export function settingsForDatasetFile(datasetPath: string, timeAttribute: string, options: Options = {}): Settings { + // TODO: pass credentials somewhere + const sources: Sources = { + dataCubes: [dataCubeFromConfig({ + name: path.basename(datasetPath, path.extname(datasetPath)), + clusterName: "native", + source: datasetPath, + timeAttribute + }, undefined)], + clusters: [] + }; + const appSettings = EMPTY_APP_SETTINGS; + const serverSettings = ServerSettings.fromJS(options); + + return { + sources, + appSettings, + serverSettings + }; +} diff --git a/src/server/cli/introspect-cluster.ts b/src/server/cli/introspect-cluster.ts new file mode 100644 index 000000000..a29729224 --- /dev/null +++ b/src/server/cli/introspect-cluster.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LOGGER } from "../../common/logger/logger"; +import { AppSettings } from "../../common/models/app-settings/app-settings"; +import { Sources } from "../../common/models/sources/sources"; +import { appSettingsToYaml, printExtra, sourcesToYaml } from "../../common/utils/yaml-helper/yaml-helper"; +import { ServerSettings } from "../models/server-settings/server-settings"; +import { SettingsManager } from "../utils/settings-manager/settings-manager"; +import { VERSION } from "../version"; + +export default function printIntrospectedSettings( + serverSettings: ServerSettings, + appSettings: AppSettings, + sources: Sources, + verbose: boolean +) { + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath: process.cwd(), + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + + return settingsManager.getFreshSources({ + timeout: 10000 + }).then(sources => { + const extra = { + header: true, + version: VERSION, + verbose + // Why port here? We don't start server so port is meaningless + // port: SERVER_SETTINGS.port + }; + const config = [ + printExtra(extra, verbose), + appSettingsToYaml(appSettings, verbose), + sourcesToYaml(sources, verbose) + ].join("\n"); + process.stdout.write(config); + }); +} From 187af65c1f71088021bc0835fbec125a4d3712cb Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 16:41:33 +0100 Subject: [PATCH 16/28] handle version read failure --- src/server/cli.ts | 12 +++++++++--- src/server/cli/introspect-cluster.ts | 6 +++--- src/server/version.ts | 16 +++++++--------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 38b559acb..c353a6149 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -24,7 +24,7 @@ import printIntrospectedSettings from "./cli/introspect-cluster"; import { loadConfigFile } from "./cli/load-config-file"; import { assertCredentials, parseInteger } from "./cli/utils"; import { SettingsManager } from "./utils/settings-manager/settings-manager"; -import { VERSION } from "./version"; +import { readVersion } from "./version"; const portOption = new Option("-p, --port ", "port number").argParser(parseInteger); const serverRootOption = new Option("--server-root ", "server root"); @@ -33,7 +33,12 @@ const verboseOption = new Option("--verbose", "verbose mode"); const usernameOption = new Option("--username ", "username"); const passwordOption = new Option("--password ", "password"); -const version = VERSION; +let version: string; +try { + version = readVersion(); +} catch (e) { + program.error(`?Failed to read turnilo version. Error: ${e.message}`); +} program .name("turnilo") @@ -152,7 +157,8 @@ program serverSettings, appSettings, sources, - verbose + verbose, + version ).catch((e: Error) => { program.error("There was an error generating a config: " + e.message); }); diff --git a/src/server/cli/introspect-cluster.ts b/src/server/cli/introspect-cluster.ts index a29729224..a6551e3e1 100644 --- a/src/server/cli/introspect-cluster.ts +++ b/src/server/cli/introspect-cluster.ts @@ -20,13 +20,13 @@ import { Sources } from "../../common/models/sources/sources"; import { appSettingsToYaml, printExtra, sourcesToYaml } from "../../common/utils/yaml-helper/yaml-helper"; import { ServerSettings } from "../models/server-settings/server-settings"; import { SettingsManager } from "../utils/settings-manager/settings-manager"; -import { VERSION } from "../version"; export default function printIntrospectedSettings( serverSettings: ServerSettings, appSettings: AppSettings, sources: Sources, - verbose: boolean + verbose: boolean, + version: string ) { const settingsManager = new SettingsManager(appSettings, sources, { anchorPath: process.cwd(), @@ -40,7 +40,7 @@ export default function printIntrospectedSettings( }).then(sources => { const extra = { header: true, - version: VERSION, + version, verbose // Why port here? We don't start server so port is meaningless // port: SERVER_SETTINGS.port diff --git a/src/server/version.ts b/src/server/version.ts index 0f6662ff6..fc753b81f 100644 --- a/src/server/version.ts +++ b/src/server/version.ts @@ -18,12 +18,10 @@ import { loadFileSync } from "./utils/file/file"; const PACKAGE_FILE = path.join(__dirname, "../../package.json"); -let packageObj: any = null; -try { - packageObj = loadFileSync(PACKAGE_FILE, "json"); -} catch (e) { - console.error(`Could not read package.json: ${e.message}`); - process.exit(1); // TODO: common tool for exit? -} - -export const VERSION = packageObj.version; +export const readVersion = (): string => { + const packageObj = loadFileSync(PACKAGE_FILE, "json"); + if (!("version" in packageObj)) { + throw new Error("Couldn't read version from package.json"); + } + return (packageObj as any).version; +}; From 46242972e3b6d585a5f8163b6a2a6efbb8afa972 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 7 Nov 2022 16:43:41 +0100 Subject: [PATCH 17/28] pass command to server so we can recover from errors --- src/server/cli.ts | 8 ++++---- src/server/cli/create-server.ts | 11 ++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index c353a6149..83cafb3e4 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -66,7 +66,7 @@ program verbose, logger: LOGGER }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version)); + createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); }); program @@ -86,7 +86,7 @@ program verbose, logger: LOGGER }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version)); + createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); }); program @@ -108,7 +108,7 @@ program verbose, logger: LOGGER }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version)); + createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); }); program @@ -127,7 +127,7 @@ program verbose, logger: LOGGER }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version)); + createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); }); program diff --git a/src/server/cli/create-server.ts b/src/server/cli/create-server.ts index acba12ff7..3bfa8752d 100644 --- a/src/server/cli/create-server.ts +++ b/src/server/cli/create-server.ts @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Command } from "commander"; import { Express } from "express"; import http from "http"; import { AddressInfo } from "net"; import { ServerSettings } from "../models/server-settings/server-settings"; -export default function createServer(serverSettings: ServerSettings, app: Express) { +export default function createServer(serverSettings: ServerSettings, app: Express, program: Command) { const server = http.createServer(app); @@ -30,15 +31,11 @@ export default function createServer(serverSettings: ServerSettings, app: Expres // handle specific listen errors with friendly messages switch (error.code) { case "EACCES": - // TODO: this should do something that commander understands - console.error(`Port ${serverSettings.port} requires elevated privileges`); - process.exit(1); + program.error(`Port ${serverSettings.port} requires elevated privileges`); break; case "EADDRINUSE": - // TODO: this should do something that commander understands - console.error(`Port ${serverSettings.port} is already in use`); - process.exit(1); + program.error(`Port ${serverSettings.port} is already in use`); break; default: From 62d3f9d5dc7f29387b5f32e9f71164327b9c4fdd Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Tue, 8 Nov 2022 09:31:22 +0100 Subject: [PATCH 18/28] lint fixes --- src/server/cli/build-settings.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/cli/build-settings.ts b/src/server/cli/build-settings.ts index 9685d3eff..e41801745 100644 --- a/src/server/cli/build-settings.ts +++ b/src/server/cli/build-settings.ts @@ -14,16 +14,16 @@ * limitations under the License. */ +import path from "path"; import { AppSettings, EMPTY_APP_SETTINGS, fromConfig as appSettingsFromConfig } from "../../common/models/app-settings/app-settings"; -import { fromConfig as sourcesFromConfig, Sources } from "../../common/models/sources/sources"; -import { ServerSettings, ServerSettingsJS } from "../models/server-settings/server-settings"; import { fromConfig as clusterFromConfig } from "../../common/models/cluster/cluster"; import { fromConfig as dataCubeFromConfig } from "../../common/models/data-cube/data-cube"; -import path from "path"; +import { fromConfig as sourcesFromConfig, Sources } from "../../common/models/sources/sources"; +import { ServerSettings, ServerSettingsJS } from "../models/server-settings/server-settings"; interface Settings { serverSettings: ServerSettings; From 797031e5b713b1f05b550e8b5f0446f9a95ac716 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Tue, 8 Nov 2022 13:51:13 +0100 Subject: [PATCH 19/28] native data cubes dimensions are loaded after introspection so we can't check for dimension on cube creation --- src/common/models/data-cube/data-cube.mocha.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/common/models/data-cube/data-cube.mocha.ts b/src/common/models/data-cube/data-cube.mocha.ts index 1227fd112..e0354f560 100644 --- a/src/common/models/data-cube/data-cube.mocha.ts +++ b/src/common/models/data-cube/data-cube.mocha.ts @@ -783,10 +783,6 @@ describe("DataCube", () => { expect(() => fromConfig({ ...baseCube })).to.throw("DataCube \"medals\" must have defined timeAttribute property"); }); - it("should throw with timeAttribute property pointing to non-existing dimension", () => { - expect(() => fromConfig({ ...baseCube, timeAttribute: "foobar", dimensions: [] })).to.throw("In DataCube \"medals\" could not find dimension for supplied timeAttribute \"foobar\""); - }); - it("should pass well defined dimensions and timeAttribute", () => { const cube = fromConfig({ ...baseCube, timeAttribute: "time_column", dimensions: [timeDimensionJS] }); const timeAttribute = $("time_column"); From b53f60c9a4eb13d106d3f7a36cba6d761730124d Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 15:22:00 +0100 Subject: [PATCH 20/28] auth is optional --- src/common/models/cluster/cluster.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/models/cluster/cluster.ts b/src/common/models/cluster/cluster.ts index abdcebd41..785d33592 100644 --- a/src/common/models/cluster/cluster.ts +++ b/src/common/models/cluster/cluster.ts @@ -44,7 +44,7 @@ export interface Cluster { introspectionStrategy?: string; requestDecorator?: RequestDecorator; retry?: RetryOptions; - auth: ClusterAuth; + auth?: ClusterAuth; } export interface ClusterJS { From 38248046d99d433b2547318eeab663aa6af3ed31 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 15:22:28 +0100 Subject: [PATCH 21/28] pass parsed cluster credentials to sources config --- src/server/cli.ts | 17 +++++++---------- src/server/cli/build-settings.ts | 29 +++++++++++++++++++++-------- src/server/cli/utils.ts | 24 +++++++++++++++++------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 83cafb3e4..1c59e0f83 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -22,7 +22,7 @@ import buildSettings, { settingsForDatasetFile, settingsForDruidConnection } fro import createServer from "./cli/create-server"; import printIntrospectedSettings from "./cli/introspect-cluster"; import { loadConfigFile } from "./cli/load-config-file"; -import { assertCredentials, parseInteger } from "./cli/utils"; +import { parseCredentials, parseInteger } from "./cli/utils"; import { SettingsManager } from "./utils/settings-manager/settings-manager"; import { readVersion } from "./version"; @@ -56,10 +56,9 @@ program .addOption(verboseOption) .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { const anchorPath = path.dirname(configPath); - assertCredentials(username, password); + const auth = parseCredentials(username, password); const config = loadConfigFile(configPath); - // TODO: pass credentials somewhere - const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }); + const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }, auth); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath, initialLoadTimeout: serverSettings.pageMustLoadTimeout, @@ -99,9 +98,8 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { port, verbose, username, password, serverRoot, serverHost }) => { - assertCredentials(username, password); - // TODO: pass credentials somewhere - const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { port, verbose, serverHost, serverRoot }); + const auth = parseCredentials(username, password); + const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { port, verbose, serverHost, serverRoot }, auth); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath: process.cwd(), initialLoadTimeout: serverSettings.pageMustLoadTimeout, @@ -150,9 +148,8 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { verbose, username, password }) => { - assertCredentials(username, password); - // TODO: pass credentials somewhere - const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { verbose }); + const auth = parseCredentials(username, password); + const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { verbose }, auth); printIntrospectedSettings( serverSettings, appSettings, diff --git a/src/server/cli/build-settings.ts b/src/server/cli/build-settings.ts index e41801745..af3219416 100644 --- a/src/server/cli/build-settings.ts +++ b/src/server/cli/build-settings.ts @@ -20,9 +20,11 @@ import { EMPTY_APP_SETTINGS, fromConfig as appSettingsFromConfig } from "../../common/models/app-settings/app-settings"; +import { ClusterAuthJS } from "../../common/models/cluster-auth/cluster-auth"; import { fromConfig as clusterFromConfig } from "../../common/models/cluster/cluster"; import { fromConfig as dataCubeFromConfig } from "../../common/models/data-cube/data-cube"; -import { fromConfig as sourcesFromConfig, Sources } from "../../common/models/sources/sources"; +import { fromConfig as sourcesFromConfig, Sources, SourcesJS } from "../../common/models/sources/sources"; +import { isNil } from "../../common/utils/general/general"; import { ServerSettings, ServerSettingsJS } from "../models/server-settings/server-settings"; interface Settings { @@ -38,7 +40,18 @@ interface Options { serverRoot?: string; } -export default function buildSettings(config: object, options: Options = {}): Settings { +function overrideClustersAuth(config: SourcesJS, auth: ClusterAuthJS): SourcesJS { + if (!config.clusters) return config; + return { + ...config, + clusters: config.clusters.map(cluster => ({ + ...cluster, + auth + })) + }; +} + +export default function buildSettings(config: object, options: Options, auth?: ClusterAuthJS): Settings { const serverSettingsJS: ServerSettingsJS = { ...config, ...options @@ -47,7 +60,8 @@ export default function buildSettings(config: object, options: Options = {}): Se const serverSettings = ServerSettings.fromJS(serverSettingsJS); // 4. create AppSettings and Sources from 2 const appSettings = appSettingsFromConfig(config); - const sources = sourcesFromConfig(config); + const sourcesJS = isNil(auth) ? config : overrideClustersAuth(config, auth); + const sources = sourcesFromConfig(sourcesJS); return { serverSettings, @@ -56,13 +70,13 @@ export default function buildSettings(config: object, options: Options = {}): Se }; } -export function settingsForDruidConnection(url: string, options: Options = {}): Settings { - // TODO: pass credentials somewhere +export function settingsForDruidConnection(url: string, options: Options, auth?: ClusterAuthJS): Settings { const sources: Sources = { dataCubes: [], clusters: [clusterFromConfig({ name: "druid", - url + url, + auth })] }; const appSettings = EMPTY_APP_SETTINGS; @@ -75,8 +89,7 @@ export function settingsForDruidConnection(url: string, options: Options = {}): }; } -export function settingsForDatasetFile(datasetPath: string, timeAttribute: string, options: Options = {}): Settings { - // TODO: pass credentials somewhere +export function settingsForDatasetFile(datasetPath: string, timeAttribute: string, options: Options): Settings { const sources: Sources = { dataCubes: [dataCubeFromConfig({ name: path.basename(datasetPath, path.extname(datasetPath)), diff --git a/src/server/cli/utils.ts b/src/server/cli/utils.ts index 330cf46b2..e2c987fe0 100644 --- a/src/server/cli/utils.ts +++ b/src/server/cli/utils.ts @@ -15,18 +15,28 @@ */ import { InvalidArgumentError } from "commander"; +import { ClusterAuthJS } from "../../common/models/cluster-auth/cluster-auth"; import { isNil } from "../../common/utils/general/general"; export function parseInteger(value: string): number { const parsed = parseInt(value, 10); - invariant(!isNaN(parsed), "Must be an integer"); + if (isNaN(parsed)) { + throw new InvalidArgumentError("Must be an integer"); + } return parsed; } -export function assertCredentials(username: string | undefined, password: string | undefined) { - invariant(isNil(password) && isNil(username) || !isNil(username) && !isNil(password), "You need to pass both username and password"); -} - -function invariant(condition: boolean, message: string) { - if (!condition) throw new InvalidArgumentError(message); +export function parseCredentials(username: string | undefined, password: string | undefined): ClusterAuthJS | undefined { + if (isNil(password) && isNil(username)) return undefined; + if (isNil(username)) { + throw new InvalidArgumentError("You need to pass username if you pass password"); + } + if (isNil(password)) { + throw new InvalidArgumentError("You need to pass password if you pass username"); + } + return { + type: "http-basic", + username, + password + }; } From a5074a17bca77cdd5d665e96b309a87d0d82009c Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 15:22:42 +0100 Subject: [PATCH 22/28] add "Basic" prefix to encoded credentials header --- src/server/utils/cluster-manager/cluster-manager.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/utils/cluster-manager/cluster-manager.ts b/src/server/utils/cluster-manager/cluster-manager.ts index 8123af682..eb0cae733 100644 --- a/src/server/utils/cluster-manager/cluster-manager.ts +++ b/src/server/utils/cluster-manager/cluster-manager.ts @@ -165,7 +165,9 @@ export class ClusterManager { if (isNil(auth)) return undefined; switch (auth.type) { case "http-basic": { - return { Authorization: Buffer.from(`${auth.username}:${auth.password}`).toString("base64") }; + const credentials = `${auth.username}:${auth.password}`; + const Authorization = `Basic ${Buffer.from(credentials).toString("base64")}`; + return { Authorization }; } } } @@ -177,7 +179,7 @@ export class ClusterManager { if (isNil(authHeaders)) { return undefined; } else { - return constant({ headers: authHeaders }); + return constant( { headers: authHeaders }); } } From 5c06a047fc03bc163bcbd98260731eb196262b1f Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 16:10:11 +0100 Subject: [PATCH 23/28] handle errors when loading config file --- src/server/cli.ts | 8 ++++---- src/server/cli/load-config-file.ts | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 1c59e0f83..f57d9b884 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -37,7 +37,7 @@ let version: string; try { version = readVersion(); } catch (e) { - program.error(`?Failed to read turnilo version. Error: ${e.message}`); + program.error(`Failed to read turnilo version. Error: ${e.message}`); } program @@ -57,7 +57,7 @@ program .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { const anchorPath = path.dirname(configPath); const auth = parseCredentials(username, password); - const config = loadConfigFile(configPath); + const config = loadConfigFile(configPath, program); const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }, auth); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath, @@ -77,7 +77,7 @@ program .action(({ port, verbose, serverRoot, serverHost }) => { const configPath = path.join(__dirname, "../../config-examples.yaml"); const anchorPath = path.dirname(configPath); - const config = loadConfigFile(configPath); + const config = loadConfigFile(configPath, program); const { sources, serverSettings, appSettings } = buildSettings(config, { port, verbose, serverHost, serverRoot }); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath, @@ -134,7 +134,7 @@ program .addOption(verboseOption) .action((file, { verbose }) => { try { - const config = loadConfigFile(file); + const config = loadConfigFile(file, program); buildSettings(config, { verbose }); } catch (e) { program.error("Config verification error: ", e.message); diff --git a/src/server/cli/load-config-file.ts b/src/server/cli/load-config-file.ts index 330b8df55..8e3bc6a27 100644 --- a/src/server/cli/load-config-file.ts +++ b/src/server/cli/load-config-file.ts @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Command } from "commander"; import { loadFileSync } from "../utils/file/file"; -export function loadConfigFile(configPath: string): object { +export function loadConfigFile(configPath: string, program: Command): object { try { return loadFileSync(configPath, "yaml"); } catch (e) { - throw new Error("SOMETHING"); // TODO: something that commander can handle + program.error(`Loading config file (${configPath}) failed: ${e.message}`); + return {}; } } From db3f0df89816bf2c97d696894f6fa77f9b4750c9 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 16:11:53 +0100 Subject: [PATCH 24/28] remove config.ts file --- src/server/config.ts | 284 ------------------------------------------- 1 file changed, 284 deletions(-) delete mode 100644 src/server/config.ts diff --git a/src/server/config.ts b/src/server/config.ts deleted file mode 100644 index fbe889627..000000000 --- a/src/server/config.ts +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2015-2016 Imply Data, Inc. - * Copyright 2017-2019 Allegro.pl - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import nopt from "nopt"; -import * as path from "path"; -import { LOGGER, NULL_LOGGER } from "../common/logger/logger"; -import { - AppSettingsJS, - EMPTY_APP_SETTINGS, - fromConfig as appSettingsFromConfig -} from "../common/models/app-settings/app-settings"; -import { - fromConfig as clusterFromConfig -} from "../common/models/cluster/cluster"; -import { fromConfig as dataCubeFromConfig } from "../common/models/data-cube/data-cube"; -import { fromConfig as sourcesFromConfig, SourcesJS } from "../common/models/sources/sources"; -import { arraySum, isTruthy } from "../common/utils/general/general"; -import { appSettingsToYaml, printExtra, sourcesToYaml } from "../common/utils/yaml-helper/yaml-helper"; -import { - DEFAULT_PORT, - DEFAULT_SERVER_ROOT, - ServerSettings, - ServerSettingsJS -} from "./models/server-settings/server-settings"; -import { loadFileSync } from "./utils/file/file"; -import { SettingsManager } from "./utils/settings-manager/settings-manager"; - -const PACKAGE_FILE = path.join(__dirname, "../../package.json"); - -function exitWithMessage(message: string): void { - console.log(message); - - // Hack: load the package file for no reason other than to make some time for console.log to flush - try { - loadFileSync(PACKAGE_FILE, "json"); - } catch (e) { - } - - process.exit(); -} - -function exitWithError(message: string): void { - console.error(message); - process.exit(1); -} - -function zeroOne(thing: any): number { - return Number(Boolean(thing)); -} - -var packageObj: any = null; -try { - packageObj = loadFileSync(PACKAGE_FILE, "json"); -} catch (e) { - exitWithError(`Could not read package.json: ${e.message}`); -} -export const VERSION = packageObj.version; - -const USAGE = ` -Usage: turnilo [options] - -Possible usage: - - turnilo --examples - turnilo --druid http://your.broker.host:8082 - -General arguments: - - --help Print this help message - --version Display the version number - -v, --verbose Display the DB queries that are being made - -Server arguments: - - -p, --port The port turnilo will run on (default: ${DEFAULT_PORT}) - --server-host The host on which to listen on (default: all hosts) - --server-root A custom server root to listen on (default ${DEFAULT_SERVER_ROOT}) - -Data connection options: - - Exactly one data connection option must be provided. - - -c, --config Use this local configuration (YAML) file - --examples Start Turnilo with some example data for testing / demo - -f, --file Start Turnilo on top of this file based data cube (must be JSON, CSV, or TSV) - -d, --druid The url address (http[s]://hostname[:port]) of the druid broker. If no port, 80 is assumed for plain http, and 443 for secure https. - -Configuration printing utilities: - - --print-config Prints out the auto generated config and exits - --with-comments Adds comments when printing the auto generated config -`; - -function parseArgs() { - return nopt( - { - "help": Boolean, - "version": Boolean, - "verbose": Boolean, - - "port": Number, - "server-host": String, - "server-root": String, - - "examples": Boolean, - "example": String, // deprecated - "config": String, - "auth": String, - - "print-config": Boolean, - "with-comments": Boolean, - - "file": String, - "druid": String - }, - { - v: ["--verbose"], - p: ["--port"], - c: ["--config"], - f: ["--file"], - d: ["--druid"] - }, - process.argv - ); -} - -var parsedArgs = parseArgs(); - -if (parsedArgs["help"]) { - exitWithMessage(USAGE); -} - -if (parsedArgs["version"]) { - exitWithMessage(VERSION); -} - -if (parsedArgs["example"]) { - delete parsedArgs["example"]; - parsedArgs["examples"] = true; -} - -const SETTINGS_INPUTS = ["config", "examples", "file", "druid", "postgres", "mysql"]; - -var numSettingsInputs = arraySum(SETTINGS_INPUTS.map(input => zeroOne(parsedArgs[input]))); - -if (numSettingsInputs === 0) { - exitWithMessage(USAGE); -} - -if (numSettingsInputs > 1) { - console.error(`only one of --${SETTINGS_INPUTS.join(", --")} can be given on the command line`); - if (parsedArgs["druid"] && parsedArgs["config"]) { - console.error("Looks like you are using --config and --druid in conjunction with each other"); - console.error("This usage is no longer supported. If you are migrating from Swiv < 0.9.x"); - console.error("Please visit: (https://github.com/yahoo/swiv/blob/master/docs/swiv-0.9.x-migration.md)"); - } - process.exit(1); -} - -const PRINT_CONFIG = Boolean(parsedArgs["print-config"]); -export const START_SERVER = !PRINT_CONFIG; -const logger = START_SERVER ? LOGGER : NULL_LOGGER; - -// Load server settings -let configPath = parsedArgs["config"]; - -if (parsedArgs["examples"]) { - configPath = path.join(__dirname, "../../config-examples.yaml"); -} - -let serverSettingsJS: ServerSettingsJS; -let configDirPath; -let configContent; -if (configPath) { - configDirPath = path.dirname(configPath); - try { - configContent = loadFileSync(configPath, "yaml"); - serverSettingsJS = configContent; - logger.log(`Using config ${configPath}`); - } catch (e) { - exitWithError(`Could not load config from '${configPath}': ${e.message}`); - } -} else { - configDirPath = process.cwd(); - serverSettingsJS = {}; -} - -if (parsedArgs["port"]) { - serverSettingsJS.port = parsedArgs["port"]; -} -if (parsedArgs["server-host"]) { - serverSettingsJS.serverHost = parsedArgs["server-host"]; -} -if (parsedArgs["server-root"]) { - serverSettingsJS.serverRoot = parsedArgs["server-root"]; -} -if (parsedArgs["verbose"]) { - serverSettingsJS.verbose = parsedArgs["verbose"]; -} - -const verbose = Boolean(serverSettingsJS.verbose); -export const SERVER_SETTINGS = ServerSettings.fromJS(serverSettingsJS); - -// --- Sign of Life ------------------------------- -if (START_SERVER) { - logger.log(`Starting Turnilo v${VERSION}`); -} - -function readConfig(config: AppSettingsJS & SourcesJS) { - return { - appSettings: appSettingsFromConfig(config), - sources: sourcesFromConfig(config) - }; -} - -function readArgs(file: string | undefined, url: string | undefined) { - const sources = { - clusters: !isTruthy(url) ? [] : [clusterFromConfig({ - name: "druid", - url - })], - dataCubes: !isTruthy(file) ? [] : [dataCubeFromConfig({ - name: path.basename(file, path.extname(file)), - clusterName: "native", - source: file - }, undefined) - ] - }; - - return { - appSettings: EMPTY_APP_SETTINGS, - sources - }; -} - -const { appSettings, sources } = configContent - ? readConfig(configContent) - : readArgs(parsedArgs.file, parsedArgs.druid); - -export const SETTINGS_MANAGER = new SettingsManager(appSettings, sources, { - logger, - verbose, - anchorPath: configDirPath, - initialLoadTimeout: SERVER_SETTINGS.pageMustLoadTimeout -}); - -// --- Printing ------------------------------- - -if (PRINT_CONFIG) { - const withComments = Boolean(parsedArgs["with-comments"]); - - SETTINGS_MANAGER.getFreshSources({ - timeout: 10000 - }).then(sources => { - const extra = { - header: true, - version: VERSION, - verbose, - port: SERVER_SETTINGS.port - }; - const config = [ - printExtra(extra, withComments), - appSettingsToYaml(appSettings, withComments), - sourcesToYaml(sources, withComments) - ].join("\n"); - process.stdout.write(config, () => process.exit()); - }).catch((e: Error) => { - exitWithError("There was an error generating a config: " + e.message); - }); -} From 2086543c0eac1eec5ce509f4adb4dda5cc6f4d45 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Wed, 9 Nov 2022 16:14:20 +0100 Subject: [PATCH 25/28] remove old comments --- src/server/cli/build-settings.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/cli/build-settings.ts b/src/server/cli/build-settings.ts index af3219416..287295f9d 100644 --- a/src/server/cli/build-settings.ts +++ b/src/server/cli/build-settings.ts @@ -56,9 +56,7 @@ export default function buildSettings(config: object, options: Options, auth?: C ...config, ...options }; - // 3. create ServerSettings from 2 const serverSettings = ServerSettings.fromJS(serverSettingsJS); - // 4. create AppSettings and Sources from 2 const appSettings = appSettingsFromConfig(config); const sourcesJS = isNil(auth) ? config : overrideClustersAuth(config, auth); const sources = sourcesFromConfig(sourcesJS); From 968c487d2e43e5b92623e052af00c78cfc70ade0 Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 14 Nov 2022 11:39:30 +0100 Subject: [PATCH 26/28] "generic" parse credentials --- src/common/models/cluster-auth/cluster-auth.ts | 2 +- src/server/cli.ts | 5 ++--- src/server/cli/options.ts | 15 +++++++++++++++ src/server/cli/run-turnilo.ts | 15 +++++++++++++++ src/server/cli/utils.ts | 6 +++--- 5 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/server/cli/options.ts create mode 100644 src/server/cli/run-turnilo.ts diff --git a/src/common/models/cluster-auth/cluster-auth.ts b/src/common/models/cluster-auth/cluster-auth.ts index fbf3327c3..bb8b6476a 100644 --- a/src/common/models/cluster-auth/cluster-auth.ts +++ b/src/common/models/cluster-auth/cluster-auth.ts @@ -16,7 +16,7 @@ import { isNil } from "../../utils/general/general"; -type ClusterAuthType = "http-basic"; +export type ClusterAuthType = "http-basic"; interface BasicHttpClusterAuth { type: "http-basic"; diff --git a/src/server/cli.ts b/src/server/cli.ts index f57d9b884..3476838ee 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -56,7 +56,7 @@ program .addOption(verboseOption) .action((configPath, { username, password, serverRoot, serverHost, port, verbose }) => { const anchorPath = path.dirname(configPath); - const auth = parseCredentials(username, password); + const auth = parseCredentials(username, password, "http-basic"); const config = loadConfigFile(configPath, program); const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }, auth); const settingsManager = new SettingsManager(appSettings, sources, { @@ -98,11 +98,11 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { port, verbose, username, password, serverRoot, serverHost }) => { - const auth = parseCredentials(username, password); const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { port, verbose, serverHost, serverRoot }, auth); const settingsManager = new SettingsManager(appSettings, sources, { anchorPath: process.cwd(), initialLoadTimeout: serverSettings.pageMustLoadTimeout, + const auth = parseCredentials(username, password, "http-basic"); verbose, logger: LOGGER }); @@ -148,7 +148,6 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { verbose, username, password }) => { - const auth = parseCredentials(username, password); const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { verbose }, auth); printIntrospectedSettings( serverSettings, diff --git a/src/server/cli/options.ts b/src/server/cli/options.ts new file mode 100644 index 000000000..33ea1d030 --- /dev/null +++ b/src/server/cli/options.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/src/server/cli/run-turnilo.ts b/src/server/cli/run-turnilo.ts new file mode 100644 index 000000000..33ea1d030 --- /dev/null +++ b/src/server/cli/run-turnilo.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/src/server/cli/utils.ts b/src/server/cli/utils.ts index e2c987fe0..6d767f7f0 100644 --- a/src/server/cli/utils.ts +++ b/src/server/cli/utils.ts @@ -15,7 +15,7 @@ */ import { InvalidArgumentError } from "commander"; -import { ClusterAuthJS } from "../../common/models/cluster-auth/cluster-auth"; +import { ClusterAuthJS, ClusterAuthType } from "../../common/models/cluster-auth/cluster-auth"; import { isNil } from "../../common/utils/general/general"; export function parseInteger(value: string): number { @@ -26,7 +26,7 @@ export function parseInteger(value: string): number { return parsed; } -export function parseCredentials(username: string | undefined, password: string | undefined): ClusterAuthJS | undefined { +export function parseCredentials(username: string | undefined, password: string | undefined, type: ClusterAuthType): ClusterAuthJS | undefined { if (isNil(password) && isNil(username)) return undefined; if (isNil(username)) { throw new InvalidArgumentError("You need to pass username if you pass password"); @@ -35,7 +35,7 @@ export function parseCredentials(username: string | undefined, password: string throw new InvalidArgumentError("You need to pass password if you pass username"); } return { - type: "http-basic", + type, username, password }; From 317bbc11312af0de5b38046f65a524495408ac4c Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 14 Nov 2022 11:39:49 +0100 Subject: [PATCH 27/28] remove port from introspection --- src/server/cli/introspect-cluster.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/cli/introspect-cluster.ts b/src/server/cli/introspect-cluster.ts index a6551e3e1..4056c0da9 100644 --- a/src/server/cli/introspect-cluster.ts +++ b/src/server/cli/introspect-cluster.ts @@ -42,8 +42,6 @@ export default function printIntrospectedSettings( header: true, version, verbose - // Why port here? We don't start server so port is meaningless - // port: SERVER_SETTINGS.port }; const config = [ printExtra(extra, verbose), From 2ad61dfd0b97283b4b4b29761c856acde615a3ad Mon Sep 17 00:00:00 2001 From: Adrian Mroz Date: Mon, 14 Nov 2022 11:40:04 +0100 Subject: [PATCH 28/28] runTurnilo util --- src/server/cli.ts | 89 ++++++++++++++++++-------------- src/server/cli/build-settings.ts | 21 +++----- src/server/cli/options.ts | 10 ++++ src/server/cli/run-turnilo.ts | 32 ++++++++++++ 4 files changed, 98 insertions(+), 54 deletions(-) diff --git a/src/server/cli.ts b/src/server/cli.ts index 3476838ee..1e64f8691 100644 --- a/src/server/cli.ts +++ b/src/server/cli.ts @@ -14,25 +14,23 @@ * limitations under the License. */ -import { Option, program } from "commander"; +import { program } from "commander"; import path from "path"; -import { LOGGER } from "../common/logger/logger"; -import createApp from "./app"; import buildSettings, { settingsForDatasetFile, settingsForDruidConnection } from "./cli/build-settings"; -import createServer from "./cli/create-server"; import printIntrospectedSettings from "./cli/introspect-cluster"; import { loadConfigFile } from "./cli/load-config-file"; -import { parseCredentials, parseInteger } from "./cli/utils"; -import { SettingsManager } from "./utils/settings-manager/settings-manager"; +import { + passwordOption, + portOption, + serverHostOption, + serverRootOption, + usernameOption, + verboseOption +} from "./cli/options"; +import runTurnilo from "./cli/run-turnilo"; +import { parseCredentials } from "./cli/utils"; import { readVersion } from "./version"; -const portOption = new Option("-p, --port ", "port number").argParser(parseInteger); -const serverRootOption = new Option("--server-root ", "server root"); -const serverHostOption = new Option("--server-host ", "server host"); -const verboseOption = new Option("--verbose", "verbose mode"); -const usernameOption = new Option("--username ", "username"); -const passwordOption = new Option("--password ", "password"); - let version: string; try { version = readVersion(); @@ -58,14 +56,20 @@ program const anchorPath = path.dirname(configPath); const auth = parseCredentials(username, password, "http-basic"); const config = loadConfigFile(configPath, program); - const { appSettings, sources, serverSettings } = buildSettings(config, { serverRoot, serverHost, verbose, port }, auth); - const settingsManager = new SettingsManager(appSettings, sources, { + const options = { + serverRoot, + serverHost, + verbose, + port + }; + + runTurnilo( + buildSettings(config, options, auth), anchorPath, - initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, - logger: LOGGER - }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); + version, + program + ); }); program @@ -78,14 +82,15 @@ program const configPath = path.join(__dirname, "../../config-examples.yaml"); const anchorPath = path.dirname(configPath); const config = loadConfigFile(configPath, program); - const { sources, serverSettings, appSettings } = buildSettings(config, { port, verbose, serverHost, serverRoot }); - const settingsManager = new SettingsManager(appSettings, sources, { + const options = { port, verbose, serverHost, serverRoot }; + + runTurnilo( + buildSettings(config, options), anchorPath, - initialLoadTimeout: serverSettings.pageMustLoadTimeout, verbose, - logger: LOGGER - }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); + version, + program + ); }); program @@ -98,15 +103,15 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { port, verbose, username, password, serverRoot, serverHost }) => { - const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { port, verbose, serverHost, serverRoot }, auth); - const settingsManager = new SettingsManager(appSettings, sources, { - anchorPath: process.cwd(), - initialLoadTimeout: serverSettings.pageMustLoadTimeout, const auth = parseCredentials(username, password, "http-basic"); + const options = { port, verbose, serverHost, serverRoot }; + runTurnilo( + settingsForDruidConnection(url, options, auth), + process.cwd(), verbose, - logger: LOGGER - }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); + version, + program + ); }); program @@ -118,14 +123,19 @@ program .addOption(serverHostOption) .addOption(verboseOption) .action((file, { timeAttribute, port, verbose, serverHost, serverRoot }) => { - const { appSettings, sources, serverSettings } = settingsForDatasetFile(file, timeAttribute, { serverRoot, serverHost, verbose, port }); - const settingsManager = new SettingsManager(appSettings, sources, { - anchorPath: process.cwd(), - initialLoadTimeout: serverSettings.pageMustLoadTimeout, + const options = { + serverRoot, + serverHost, verbose, - logger: LOGGER - }); - createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); + port + }; + runTurnilo( + settingsForDatasetFile(file, timeAttribute, options), + process.cwd(), + verbose, + version, + program + ); }); program @@ -148,6 +158,7 @@ program .addOption(usernameOption) .addOption(passwordOption) .action((url, { verbose, username, password }) => { + const auth = parseCredentials(username, password, "http-basic"); const { appSettings, serverSettings, sources } = settingsForDruidConnection(url, { verbose }, auth); printIntrospectedSettings( serverSettings, diff --git a/src/server/cli/build-settings.ts b/src/server/cli/build-settings.ts index 287295f9d..a104a46fe 100644 --- a/src/server/cli/build-settings.ts +++ b/src/server/cli/build-settings.ts @@ -15,25 +15,16 @@ */ import path from "path"; -import { - AppSettings, - EMPTY_APP_SETTINGS, - fromConfig as appSettingsFromConfig -} from "../../common/models/app-settings/app-settings"; +import { EMPTY_APP_SETTINGS, fromConfig as appSettingsFromConfig } from "../../common/models/app-settings/app-settings"; import { ClusterAuthJS } from "../../common/models/cluster-auth/cluster-auth"; import { fromConfig as clusterFromConfig } from "../../common/models/cluster/cluster"; import { fromConfig as dataCubeFromConfig } from "../../common/models/data-cube/data-cube"; import { fromConfig as sourcesFromConfig, Sources, SourcesJS } from "../../common/models/sources/sources"; import { isNil } from "../../common/utils/general/general"; import { ServerSettings, ServerSettingsJS } from "../models/server-settings/server-settings"; +import { TurniloSettings } from "./run-turnilo"; -interface Settings { - serverSettings: ServerSettings; - appSettings: AppSettings; - sources: Sources; -} - -interface Options { +export interface ServerOptions { port?: number; verbose?: boolean; serverHost?: string; @@ -51,7 +42,7 @@ function overrideClustersAuth(config: SourcesJS, auth: ClusterAuthJS): SourcesJS }; } -export default function buildSettings(config: object, options: Options, auth?: ClusterAuthJS): Settings { +export default function buildSettings(config: object, options: ServerOptions, auth?: ClusterAuthJS): TurniloSettings { const serverSettingsJS: ServerSettingsJS = { ...config, ...options @@ -68,7 +59,7 @@ export default function buildSettings(config: object, options: Options, auth?: C }; } -export function settingsForDruidConnection(url: string, options: Options, auth?: ClusterAuthJS): Settings { +export function settingsForDruidConnection(url: string, options: ServerOptions, auth?: ClusterAuthJS): TurniloSettings { const sources: Sources = { dataCubes: [], clusters: [clusterFromConfig({ @@ -87,7 +78,7 @@ export function settingsForDruidConnection(url: string, options: Options, auth?: }; } -export function settingsForDatasetFile(datasetPath: string, timeAttribute: string, options: Options): Settings { +export function settingsForDatasetFile(datasetPath: string, timeAttribute: string, options: ServerOptions): TurniloSettings { const sources: Sources = { dataCubes: [dataCubeFromConfig({ name: path.basename(datasetPath, path.extname(datasetPath)), diff --git a/src/server/cli/options.ts b/src/server/cli/options.ts index 33ea1d030..51f9c1f68 100644 --- a/src/server/cli/options.ts +++ b/src/server/cli/options.ts @@ -13,3 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { Option } from "commander"; +import { parseInteger } from "./utils"; + +export const portOption = new Option("-p, --port ", "port number").argParser(parseInteger); +export const serverRootOption = new Option("--server-root ", "server root"); +export const serverHostOption = new Option("--server-host ", "server host"); +export const verboseOption = new Option("--verbose", "verbose mode"); +export const usernameOption = new Option("--username ", "username"); +export const passwordOption = new Option("--password ", "password"); diff --git a/src/server/cli/run-turnilo.ts b/src/server/cli/run-turnilo.ts index 33ea1d030..9beca95e5 100644 --- a/src/server/cli/run-turnilo.ts +++ b/src/server/cli/run-turnilo.ts @@ -13,3 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { Command } from "commander"; +import { LOGGER } from "../../common/logger/logger"; +import { AppSettings } from "../../common/models/app-settings/app-settings"; +import { Sources } from "../../common/models/sources/sources"; +import createApp from "../app"; +import { ServerSettings } from "../models/server-settings/server-settings"; +import { SettingsManager } from "../utils/settings-manager/settings-manager"; +import createServer from "./create-server"; + +export interface TurniloSettings { + serverSettings: ServerSettings; + appSettings: AppSettings; + sources: Sources; +} + +export default function runTurnilo( + { serverSettings, sources, appSettings }: TurniloSettings, + anchorPath: string, + verbose: boolean, + version: string, + program: Command +) { + + const settingsManager = new SettingsManager(appSettings, sources, { + anchorPath, + initialLoadTimeout: serverSettings.pageMustLoadTimeout, + verbose, + logger: LOGGER + }); + createServer(serverSettings, createApp(serverSettings, settingsManager, version), program); +}