From a78295adcee3a44527dcf262f3a6752845f92c1d Mon Sep 17 00:00:00 2001 From: adrianmroz <78143552+adrianmroz-allegro@users.noreply.github.com> Date: Thu, 17 Mar 2022 18:29:37 +0100 Subject: [PATCH] Use POJsOs to model Cluster. Same pattern as DataCube type. (#886) * Use POJsOs to model Cluster. Same pattern as DataCube type. * Cluster tests * Renames for brevity --- src/client/deserializers/app-settings.ts | 8 +- src/client/deserializers/cluster.ts | 21 ++ src/client/deserializers/customization.ts | 4 +- src/client/deserializers/data-cube.ts | 16 +- src/client/deserializers/dimensions.ts | 4 +- src/client/deserializers/measures.ts | 4 +- src/client/deserializers/sources.ts | 8 +- .../models/app-settings/app-settings.ts | 8 +- src/common/models/cluster/cluster.fixtures.ts | 8 +- src/common/models/cluster/cluster.mocha.ts | 168 +++++++++++---- src/common/models/cluster/cluster.ts | 197 +++++++++--------- .../models/cluster/find-cluster.mocha.ts | 6 +- .../models/customization/customization.ts | 8 +- .../models/data-cube/data-cube.mocha.ts | 10 +- src/common/models/data-cube/data-cube.ts | 9 +- src/common/models/dimension/dimensions.ts | 4 +- src/common/models/measure/measures.ts | 4 +- src/common/models/sources/sources.ts | 29 ++- src/server/config.ts | 25 +-- .../utils/cluster-manager/cluster-manager.ts | 8 +- 20 files changed, 326 insertions(+), 223 deletions(-) create mode 100644 src/client/deserializers/cluster.ts diff --git a/src/client/deserializers/app-settings.ts b/src/client/deserializers/app-settings.ts index 35d80f28d..b4b29e350 100644 --- a/src/client/deserializers/app-settings.ts +++ b/src/client/deserializers/app-settings.ts @@ -15,14 +15,14 @@ */ import { ClientAppSettings, SerializedAppSettings } from "../../common/models/app-settings/app-settings"; -import { deserialize as customizationDeserialize } from "./customization"; -import { deserialize as oauthDeserialize } from "./oauth"; +import { deserialize as deserializeCustomization } from "./customization"; +import { deserialize as deserializeOauth } from "./oauth"; export function deserialize({ oauth, clientTimeout, customization, version }: SerializedAppSettings): ClientAppSettings { return { clientTimeout, version, - customization: customizationDeserialize(customization), - oauth: oauthDeserialize(oauth) + customization: deserializeCustomization(customization), + oauth: deserializeOauth(oauth) }; } diff --git a/src/client/deserializers/cluster.ts b/src/client/deserializers/cluster.ts new file mode 100644 index 000000000..1750549cc --- /dev/null +++ b/src/client/deserializers/cluster.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientCluster, SerializedCluster } from "../../common/models/cluster/cluster"; + +export function deserialize(cluster: SerializedCluster): ClientCluster { + return cluster; +} diff --git a/src/client/deserializers/customization.ts b/src/client/deserializers/customization.ts index 01d0fe9c3..20cd6f2b0 100644 --- a/src/client/deserializers/customization.ts +++ b/src/client/deserializers/customization.ts @@ -16,7 +16,7 @@ import { Timezone } from "chronoshift"; import { ClientCustomization, SerializedCustomization } from "../../common/models/customization/customization"; -import { deserialize as localeDeserialize } from "../../common/models/locale/locale"; +import { deserialize as deserializeLocale } from "../../common/models/locale/locale"; export function deserialize(customization: SerializedCustomization): ClientCustomization { const { headerBackground, locale, customLogoSvg, timezones, externalViews, hasUrlShortener, sentryDSN } = customization; @@ -26,7 +26,7 @@ export function deserialize(customization: SerializedCustomization): ClientCusto externalViews, hasUrlShortener, sentryDSN, - locale: localeDeserialize(locale), + locale: deserializeLocale(locale), timezones: timezones.map(Timezone.fromJS) }; } diff --git a/src/client/deserializers/data-cube.ts b/src/client/deserializers/data-cube.ts index d826d47dd..23693e6c8 100644 --- a/src/client/deserializers/data-cube.ts +++ b/src/client/deserializers/data-cube.ts @@ -17,12 +17,12 @@ import { Duration, Timezone } from "chronoshift"; import { AttributeInfo, Executor } from "plywood"; import { ClientDataCube, SerializedDataCube } from "../../common/models/data-cube/data-cube"; -import { serialize as dimensionsSerialize } from "../../common/models/dimension/dimensions"; +import { serialize as serializeDimensions } from "../../common/models/dimension/dimensions"; import { Filter } from "../../common/models/filter/filter"; -import { serialize as measuresSerialize } from "../../common/models/measure/measures"; +import { serialize as serializeMeasures } from "../../common/models/measure/measures"; import { RefreshRule } from "../../common/models/refresh-rule/refresh-rule"; -import { deserialize as dimensionsDeserialize } from "./dimensions"; -import { deserialize as measuresDeserialize } from "./measures"; +import { deserialize as deserializeDimensions } from "./dimensions"; +import { deserialize as deserializeMeasures } from "./measures"; export function deserialize(dataCube: SerializedDataCube, executor: Executor): ClientDataCube { const { @@ -60,12 +60,12 @@ export function deserialize(dataCube: SerializedDataCube, executor: Executor): C defaultSplitDimensions, defaultTimezone: Timezone.fromJS(defaultTimezone), description, - dimensions: dimensionsDeserialize(dimensions), + dimensions: deserializeDimensions(dimensions), executor, extendedDescription, group, maxSplits, - measures: measuresDeserialize(measures), + measures: deserializeMeasures(measures), name, options, refreshRule: RefreshRule.fromJS(refreshRule), @@ -116,11 +116,11 @@ export function serialize(dataCube: ClientDataCube): SerializedDataCube { defaultSplitDimensions, defaultTimezone: defaultTimezone.toJS(), description, - dimensions: dimensionsSerialize(dimensions), + dimensions: serializeDimensions(dimensions), extendedDescription, group, maxSplits, - measures: measuresSerialize(measures), + measures: serializeMeasures(measures), name, options, refreshRule: RefreshRule.fromJS(refreshRule), diff --git a/src/client/deserializers/dimensions.ts b/src/client/deserializers/dimensions.ts index ee08f04ee..4f3f50cd9 100644 --- a/src/client/deserializers/dimensions.ts +++ b/src/client/deserializers/dimensions.ts @@ -16,11 +16,11 @@ import { ClientDimensions, SerializedDimensions } from "../../common/models/dimension/dimensions"; import { mapValues } from "../../common/utils/object/object"; -import { deserialize as dimensionDeserialize } from "./dimension"; +import { deserialize as deserializeDimension } from "./dimension"; export function deserialize({ tree, byName }: SerializedDimensions): ClientDimensions { return { tree, - byName: mapValues(byName, dimensionDeserialize) + byName: mapValues(byName, deserializeDimension) }; } diff --git a/src/client/deserializers/measures.ts b/src/client/deserializers/measures.ts index c0bbdc891..0410d9e88 100644 --- a/src/client/deserializers/measures.ts +++ b/src/client/deserializers/measures.ts @@ -16,11 +16,11 @@ import { ClientMeasures, SerializedMeasures } from "../../common/models/measure/measures"; import { mapValues } from "../../common/utils/object/object"; -import { deserialize as measureDeserialize } from "./measure"; +import { deserialize as deserializeMeasure } from "./measure"; export function deserialize({ tree, byName }: SerializedMeasures): ClientMeasures { return { tree, - byName: mapValues(byName, measureDeserialize) + byName: mapValues(byName, deserializeMeasure) }; } diff --git a/src/client/deserializers/sources.ts b/src/client/deserializers/sources.ts index 0293e855e..cdd78c938 100644 --- a/src/client/deserializers/sources.ts +++ b/src/client/deserializers/sources.ts @@ -15,18 +15,18 @@ */ import { ClientAppSettings } from "../../common/models/app-settings/app-settings"; -import { Cluster } from "../../common/models/cluster/cluster"; import { SerializedDataCube } from "../../common/models/data-cube/data-cube"; import { ClientSources, SerializedSources } from "../../common/models/sources/sources"; import { Ajax } from "../utils/ajax/ajax"; -import { deserialize as dataCubeDeserialize } from "./data-cube"; +import { deserialize as deserializeCluster } from "./cluster"; +import { deserialize as deserializeDataCube } from "./data-cube"; export function deserialize(settings: SerializedSources, appSettings: ClientAppSettings): ClientSources { - const clusters = settings.clusters.map(cluster => Cluster.fromJS(cluster)); + const clusters = settings.clusters.map(deserializeCluster); const dataCubes = settings.dataCubes.map((dataCube: SerializedDataCube) => { const executor = Ajax.queryUrlExecutorFactory(dataCube.name, appSettings); - return dataCubeDeserialize(dataCube, executor); + return deserializeDataCube(dataCube, executor); }); return { diff --git a/src/common/models/app-settings/app-settings.ts b/src/common/models/app-settings/app-settings.ts index f85edb29b..870cffc25 100644 --- a/src/common/models/app-settings/app-settings.ts +++ b/src/common/models/app-settings/app-settings.ts @@ -21,14 +21,14 @@ import { Customization, CustomizationJS, fromConfig as customizationFromConfig, - serialize as customizationSerialize, + serialize as serializeCustomization, SerializedCustomization } from "../customization/customization"; import { fromConfig as oauthFromConfig, Oauth, OauthJS, - serialize as oauthSerialize, + serialize as serializeOauth, SerializedOauth } from "../oauth/oauth"; @@ -82,8 +82,8 @@ export function serialize({ oauth, clientTimeout, customization, version }: AppS return { clientTimeout, version, - customization: customizationSerialize(customization), - oauth: oauthSerialize(oauth) + customization: serializeCustomization(customization), + oauth: serializeOauth(oauth) }; } diff --git a/src/common/models/cluster/cluster.fixtures.ts b/src/common/models/cluster/cluster.fixtures.ts index 916645f11..abc669838 100644 --- a/src/common/models/cluster/cluster.fixtures.ts +++ b/src/common/models/cluster/cluster.fixtures.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Cluster, ClusterJS } from "./cluster"; +import { Cluster, ClusterJS, fromConfig } from "./cluster"; export class ClusterFixtures { static druidWikiClusterJS(): ClusterJS { @@ -33,7 +33,7 @@ export class ClusterFixtures { } static druidWikiCluster(): Cluster { - return Cluster.fromJS(ClusterFixtures.druidWikiClusterJS()); + return fromConfig(ClusterFixtures.druidWikiClusterJS()); } static druidTwitterClusterJS(): ClusterJS { @@ -52,11 +52,11 @@ export class ClusterFixtures { } static druidTwitterCluster(): Cluster { - return Cluster.fromJS(ClusterFixtures.druidTwitterClusterJS()); + return fromConfig(ClusterFixtures.druidTwitterClusterJS()); } static druidTwitterClusterJSWithGuard(guardDataCubes = true): Cluster { - return Cluster.fromJS({ + return fromConfig({ name: "druid-custom", url: "http://192.168.99.101", version: "0.9.1", diff --git a/src/common/models/cluster/cluster.mocha.ts b/src/common/models/cluster/cluster.mocha.ts index 6c33ced85..44a1e2e62 100644 --- a/src/common/models/cluster/cluster.mocha.ts +++ b/src/common/models/cluster/cluster.mocha.ts @@ -15,56 +15,140 @@ * limitations under the License. */ -import { expect } from "chai"; -import { testImmutableClass } from "immutable-class-tester"; -import { Cluster, ClusterJS } from "./cluster"; +import { expect, use } from "chai"; +import equivalent from "../../../client/utils/test-utils/equivalent"; +import { RequestDecorator } from "../../../server/utils/request-decorator/request-decorator"; +import { RetryOptions } from "../../../server/utils/retry-options/retry-options"; +import { ClusterJS, fromConfig } from "./cluster"; + +use(equivalent); describe("Cluster", () => { - // TODO: reimplement this test as simpler cases without immutable-class-tester - it checks too much - it.skip("is an immutable class", () => { - testImmutableClass(Cluster, [ - { - name: "my-druid-cluster" - }, - { - name: "my-druid-cluster", - url: "https://192.168.99.100", - version: "0.9.1", - timeout: 30000, - healthCheckTimeout: 50, - sourceListScan: "auto", - sourceListRefreshOnLoad: true, - sourceListRefreshInterval: 10000, - sourceReintrospectInterval: 10000, - - introspectionStrategy: "segment-metadata-fallback" - }, - { - name: "my-mysql-cluster", - url: "http://192.168.99.100", - timeout: 30000, - sourceListScan: "auto" - }, - { - name: "my-mysql-cluster", - url: "https://192.168.99.100", - timeout: 30000, - sourceListScan: "auto", + describe("fromConfig", () => { + it("should load default values", () => { + const cluster = fromConfig({ + name: "foobar", + url: "http://bazz" + }); + + expect(cluster).to.be.deep.equal({ + name: "foobar", + guardDataCubes: false, + healthCheckTimeout: 1000, + introspectionStrategy: "segment-metadata-fallback", + requestDecorator: null, + retry: new RetryOptions(), sourceListRefreshInterval: 0, - sourceReintrospectInterval: 0 - } - ]); - }); + sourceListRefreshOnLoad: false, + sourceListScan: "auto", + sourceReintrospectInterval: 0, + sourceReintrospectOnLoad: false, + timeout: undefined, + title: "", + type: "druid", + url: "http://bazz", + version: null + }); + }); + + it("should throw with incorrect name type", () => { + expect(() => fromConfig({ name: 1 } as unknown as ClusterJS)).to.throw("must be a string"); + }); + + it("should throw with incorrect empty name", () => { + expect(() => fromConfig({ name: "", url: "http://foobar" })).to.throw("empty name"); + }); + + it("should throw with not url safe name", () => { + expect(() => fromConfig({ name: "foobar%bazz#", url: "http://foobar" })).to.throw("is not a URL safe name"); + }); + + it("should throw with name equal to native", () => { + expect(() => fromConfig({ name: "native", url: "http://foobar" })).to.throw("name can not be 'native'"); + }); + + it("should read retry options", () => { + const cluster = fromConfig({ + name: "foobar", + url: "http://foobar", + retry: { + maxAttempts: 1, + delay: 42 + } + }); + + expect(cluster.retry).to.be.equivalent(new RetryOptions({ maxAttempts: 1, delay: 42 })); + }); + + it("should read request decorator", () => { + const cluster = fromConfig({ + name: "foobar", + url: "http://foobar", + requestDecorator: { + path: "foobar", + options: { bazz: true } + } + }); + + expect(cluster.requestDecorator).to.be.equivalent(new RequestDecorator("foobar", { bazz: true })); + }); + + it("should read request decorator old format", () => { + const cluster = fromConfig({ + name: "foobar", + url: "http://foobar", + requestDecorator: "foobar", + decoratorOptions: { bazz: true } + } as unknown as ClusterJS); + + expect(cluster.requestDecorator).to.be.equivalent(new RequestDecorator("foobar", { bazz: true })); + }); - describe("backward compatibility", () => { it("should read old host and assume http protocol", () => { - const cluster = Cluster.fromJS({ + const cluster = fromConfig({ name: "old-host", host: "broker-host.com" - } as ClusterJS); + } as unknown as ClusterJS); expect(cluster.url).to.be.eq("http://broker-host.com"); }); - }); -}); + it("should override default values", () => { + const cluster = fromConfig({ + guardDataCubes: true, + healthCheckTimeout: 42, + introspectionStrategy: "introspection-introspection", + name: "cluster-name", + sourceListRefreshInterval: 1123, + sourceListRefreshOnLoad: true, + sourceListScan: "auto", + sourceReintrospectInterval: 1432, + sourceReintrospectOnLoad: true, + timeout: 581, + title: "foobar-title", + url: "http://url-bazz", + version: "new-version" + }); + + expect(cluster).to.be.deep.equal({ + guardDataCubes: true, + healthCheckTimeout: 42, + introspectionStrategy: "introspection-introspection", + name: "cluster-name", + sourceListRefreshInterval: 1123, + sourceListRefreshOnLoad: true, + sourceListScan: "auto", + sourceReintrospectInterval: 1432, + sourceReintrospectOnLoad: true, + timeout: 581, + title: "foobar-title", + url: "http://url-bazz", + version: "new-version", + type: "druid", + requestDecorator: null, + retry: new RetryOptions() + }); + }); + }); +}) +; diff --git a/src/common/models/cluster/cluster.ts b/src/common/models/cluster/cluster.ts index 67c8a2752..7ead24be6 100644 --- a/src/common/models/cluster/cluster.ts +++ b/src/common/models/cluster/cluster.ts @@ -15,7 +15,6 @@ * limitations under the License. */ -import { Record } from "immutable"; import { BaseImmutable } from "immutable-class"; import { External } from "plywood"; import { URL } from "url"; @@ -25,9 +24,12 @@ import { isNil, isTruthy, optionalEnsureOneOf, verifyUrlSafeName } from "../../u export type SourceListScan = "disable" | "auto"; -export interface ClusterValue { +export type ClusterType = "druid"; + +export interface Cluster { + type: ClusterType; name: string; - url?: string; + url: string; title?: string; version?: string; timeout?: number; @@ -46,7 +48,7 @@ export interface ClusterValue { export interface ClusterJS { name: string; title?: string; - url?: string; + url: string; version?: string; timeout?: number; healthCheckTimeout?: number; @@ -61,6 +63,18 @@ export interface ClusterJS { retry?: RetryOptionsJS; } +export interface SerializedCluster { + type: ClusterType; + name: string; + timeout: number; +} + +export interface ClientCluster { + type: ClusterType; + name: string; + timeout: number; +} + function ensureNotNative(name: string): void { if (name === "native") { throw new Error("Cluster name can not be 'native'"); @@ -85,9 +99,17 @@ function validateUrl(url: string): void { const HTTP_PROTOCOL_TEST = /^http(s?):/; function readUrl(cluster: any): string { - if (isTruthy(cluster.url)) return cluster.url; + if (isTruthy(cluster.url)) { + validateUrl(cluster.url); + return cluster.url; + } const oldHost = cluster.host || cluster.druidHost || cluster.brokerHost; - return HTTP_PROTOCOL_TEST.test(oldHost) ? oldHost : `http://${oldHost}`; + if (isTruthy(oldHost)) { + const url = HTTP_PROTOCOL_TEST.test(oldHost) ? oldHost : `http://${oldHost}`; + validateUrl(url); + return url; + } + throw new Error("Cluster: missing url field"); } function readRequestDecorator(cluster: any): RequestDecorator | null { @@ -109,102 +131,79 @@ export const DEFAULT_SOURCE_REINTROSPECT_ON_LOAD = false; export const DEFAULT_INTROSPECTION_STRATEGY = "segment-metadata-fallback"; const DEFAULT_GUARD_DATA_CUBES = false; -const defaultCluster: ClusterValue = { - guardDataCubes: DEFAULT_GUARD_DATA_CUBES, - healthCheckTimeout: DEFAULT_HEALTH_CHECK_TIMEOUT, - introspectionStrategy: DEFAULT_INTROSPECTION_STRATEGY, - name: "", - requestDecorator: undefined, - sourceListRefreshInterval: DEFAULT_SOURCE_LIST_REFRESH_INTERVAL, - sourceListRefreshOnLoad: DEFAULT_SOURCE_LIST_REFRESH_ON_LOAD, - sourceListScan: DEFAULT_SOURCE_LIST_SCAN, - sourceReintrospectInterval: DEFAULT_SOURCE_REINTROSPECT_INTERVAL, - sourceReintrospectOnLoad: DEFAULT_SOURCE_REINTROSPECT_ON_LOAD, - timeout: undefined, - title: "", - url: "", - version: null -}; - -export class Cluster extends Record(defaultCluster) { - - static fromJS(params: ClusterJS): Cluster { - const { - name, - sourceListScan, - sourceListRefreshOnLoad, - sourceReintrospectOnLoad, - version, - title, - guardDataCubes, - introspectionStrategy, - healthCheckTimeout - } = params; - - verifyUrlSafeName(name); - ensureNotNative(name); - - optionalEnsureOneOf(sourceListScan, SOURCE_LIST_SCAN_VALUES, "Cluster: sourceListScan"); - - const sourceReintrospectInterval = typeof params.sourceReintrospectInterval === "string" ? parseInt(params.sourceReintrospectInterval, 10) : params.sourceListRefreshInterval; - if (isTruthy(sourceReintrospectInterval)) { - BaseImmutable.ensure.number(sourceReintrospectInterval); - ensureNotTiny(sourceReintrospectInterval); - } - - const sourceListRefreshInterval = typeof params.sourceListRefreshInterval === "string" ? parseInt(params.sourceListRefreshInterval, 10) : params.sourceListRefreshInterval; - if (isTruthy(sourceListRefreshInterval)) { - BaseImmutable.ensure.number(sourceListRefreshInterval); - ensureNotTiny(sourceListRefreshInterval); - } - - const retry = RetryOptions.fromJS(params.retry); - const requestDecorator = readRequestDecorator(params); - - const url = readUrl(params); - validateUrl(url); - - return new Cluster({ - timeout: typeof params.timeout === "string" ? parseInt(params.timeout, 10) : params.timeout, - name, - url, - retry, - requestDecorator, - sourceListScan, - sourceListRefreshInterval, - sourceListRefreshOnLoad, - sourceReintrospectInterval, - sourceReintrospectOnLoad, - version, - title, - guardDataCubes, - introspectionStrategy, - healthCheckTimeout - }); - } +function readInterval(value: number | string, defaultValue: number): number { + if (!isTruthy(value)) return defaultValue; + const numberValue = typeof value === "string" ? parseInt(value, 10) : value; + BaseImmutable.ensure.number(numberValue); + ensureNotTiny(numberValue); + return numberValue; +} - public type = "druid"; +export function fromConfig(params: ClusterJS): Cluster { + const { + name, + sourceListScan = DEFAULT_SOURCE_LIST_SCAN, + sourceListRefreshOnLoad = DEFAULT_SOURCE_LIST_REFRESH_ON_LOAD, + sourceReintrospectOnLoad = DEFAULT_SOURCE_REINTROSPECT_ON_LOAD, + version = null, + title = "", + guardDataCubes = DEFAULT_GUARD_DATA_CUBES, + introspectionStrategy = DEFAULT_INTROSPECTION_STRATEGY, + healthCheckTimeout = DEFAULT_HEALTH_CHECK_TIMEOUT + } = params; + + verifyUrlSafeName(name); + ensureNotNative(name); + + optionalEnsureOneOf(sourceListScan, SOURCE_LIST_SCAN_VALUES, "Cluster: sourceListScan"); + + const sourceReintrospectInterval = readInterval(params.sourceReintrospectInterval, DEFAULT_SOURCE_REINTROSPECT_INTERVAL); + const sourceListRefreshInterval = readInterval(params.sourceListRefreshInterval, DEFAULT_SOURCE_LIST_REFRESH_INTERVAL); + const retry = RetryOptions.fromJS(params.retry); + const requestDecorator = readRequestDecorator(params); + + const url = readUrl(params); + + return { + type: "druid", + timeout: typeof params.timeout === "string" ? parseInt(params.timeout, 10) : params.timeout, + name, + url, + retry, + requestDecorator, + sourceListScan, + sourceListRefreshInterval, + sourceListRefreshOnLoad, + sourceReintrospectInterval, + sourceReintrospectOnLoad, + version, + title, + guardDataCubes, + introspectionStrategy, + healthCheckTimeout + }; +} - public toClientCluster(): Cluster { - return new Cluster({ - name: this.name, - timeout: this.timeout - }); - } +export function serialize(cluster: Cluster): SerializedCluster { + return { + type: "druid", + name: cluster.name, + timeout: cluster.timeout + }; +} - public makeExternalFromSourceName(source: string, version?: string): External { - return External.fromValue({ - engine: "druid", - source, - version, - suppress: true, +export function makeExternalFromSourceName(source: string, version?: string): External { + return External.fromValue({ + engine: "druid", + source, + version, + suppress: true, - allowSelectQueries: true, - allowEternity: false - }); - } + allowSelectQueries: true, + allowEternity: false + }); +} - public shouldScanSources(): boolean { - return this.sourceListScan === "auto"; - } +export function shouldScanSources(cluster: Cluster): boolean { + return cluster.sourceListScan === "auto"; } diff --git a/src/common/models/cluster/find-cluster.mocha.ts b/src/common/models/cluster/find-cluster.mocha.ts index c593726f2..2d4336e62 100644 --- a/src/common/models/cluster/find-cluster.mocha.ts +++ b/src/common/models/cluster/find-cluster.mocha.ts @@ -15,12 +15,12 @@ */ import { expect } from "chai"; -import { Cluster } from "./cluster"; +import { fromConfig } from "./cluster"; import { ClusterFixtures } from "./cluster.fixtures"; import { findCluster } from "./find-cluster"; -const wikiCluster = Cluster.fromJS(ClusterFixtures.druidWikiClusterJS()); -const twitterCluster = Cluster.fromJS(ClusterFixtures.druidTwitterClusterJS()); +const wikiCluster = fromConfig(ClusterFixtures.druidWikiClusterJS()); +const twitterCluster = fromConfig(ClusterFixtures.druidTwitterClusterJS()); const clusters = [ wikiCluster, diff --git a/src/common/models/customization/customization.ts b/src/common/models/customization/customization.ts index 659fb43c0..dd668c73d 100644 --- a/src/common/models/customization/customization.ts +++ b/src/common/models/customization/customization.ts @@ -20,7 +20,7 @@ import { LOGGER } from "../../logger/logger"; import { assoc } from "../../utils/functional/functional"; import { isTruthy } from "../../utils/general/general"; import { ExternalView, ExternalViewValue } from "../external-view/external-view"; -import { fromConfig as localeFromConfig, Locale, LocaleJS, serialize as localeSerialize } from "../locale/locale"; +import { fromConfig as localeFromConfig, Locale, LocaleJS, serialize as serializeLocale } from "../locale/locale"; import { fromConfig as urlShortenerFromConfig, UrlShortener, UrlShortenerDef } from "../url-shortener/url-shortener"; export const DEFAULT_TITLE = "Turnilo (%v)"; @@ -184,7 +184,7 @@ export function fromConfig(config: CustomizationJS = {}): Customization { ? configExternalViews.map(ExternalView.fromJS) : []; - const customization = { + return { title, headerBackground, customLogoSvg, @@ -195,8 +195,6 @@ export function fromConfig(config: CustomizationJS = {}): Customization { locale: localeFromConfig(locale), externalViews }; - - return customization; } export function serialize(customization: Customization): SerializedCustomization { @@ -207,7 +205,7 @@ export function serialize(customization: Customization): SerializedCustomization hasUrlShortener: isTruthy(urlShortener), headerBackground, sentryDSN, - locale: localeSerialize(locale), + locale: serializeLocale(locale), timezones: timezones.map(t => t.toJS()) }; } diff --git a/src/common/models/data-cube/data-cube.mocha.ts b/src/common/models/data-cube/data-cube.mocha.ts index 84c76598f..1227fd112 100644 --- a/src/common/models/data-cube/data-cube.mocha.ts +++ b/src/common/models/data-cube/data-cube.mocha.ts @@ -20,18 +20,18 @@ import { $, AttributeInfo } from "plywood"; import { SinonSpy, spy } from "sinon"; import equivalent from "../../../client/utils/test-utils/equivalent"; import { deduceAttributes } from "../../utils/external/datacube-to-external"; -import { Cluster } from "../cluster/cluster"; +import { fromConfig as clusterFromConfig } from "../cluster/cluster"; import { createDimension, DimensionJS, timeDimension } from "../dimension/dimension"; -import { fromConfig as dimensionsFromConfig } from "../dimension/dimensions"; -import { allDimensions, findDimensionByExpression } from "../dimension/dimensions"; +import { allDimensions, fromConfig as dimensionsFromConfig } from "../dimension/dimensions"; import { DataCube, DataCubeJS, fromConfig } from "./data-cube"; import { addAttributes } from "./queryable-data-cube"; use(equivalent); describe("DataCube", () => { - const druidCluster = Cluster.fromJS({ - name: "druid" + const druidCluster = clusterFromConfig({ + name: "druid", + url: "http://driud" }); describe("validates", () => { diff --git a/src/common/models/data-cube/data-cube.ts b/src/common/models/data-cube/data-cube.ts index 5d8074604..a6ab40f77 100644 --- a/src/common/models/data-cube/data-cube.ts +++ b/src/common/models/data-cube/data-cube.ts @@ -42,7 +42,7 @@ import { findDimensionByName, fromConfig as dimensionsFromConfig, prepend, - serialize as dimensionsSerialize, + serialize as serializeDimensions, SerializedDimensions } from "../dimension/dimensions"; import { RelativeTimeFilterClause, TimeFilterPeriod } from "../filter-clause/filter-clause"; @@ -54,7 +54,8 @@ import { hasMeasureWithName, MeasureOrGroupJS, Measures, - serialize as measuresSerialize, SerializedMeasures + serialize as serializeMeasures, + SerializedMeasures } from "../measure/measures"; import { QueryDecoratorDefinition, QueryDecoratorDefinitionJS } from "../query-decorator/query-decorator"; import { RefreshRule, RefreshRuleJS } from "../refresh-rule/refresh-rule"; @@ -433,11 +434,11 @@ export function serialize(dataCube: DataCube): SerializedDataCube { defaultSplitDimensions, defaultTimezone: defaultTimezone.toJS(), description, - dimensions: dimensionsSerialize(dimensions), + dimensions: serializeDimensions(dimensions), extendedDescription, group, maxSplits, - measures: measuresSerialize(measures), + measures: serializeMeasures(measures), name, options, refreshRule: refreshRule.toJS(), diff --git a/src/common/models/dimension/dimensions.ts b/src/common/models/dimension/dimensions.ts index a77487cc7..0a0cfb133 100644 --- a/src/common/models/dimension/dimensions.ts +++ b/src/common/models/dimension/dimensions.ts @@ -22,7 +22,7 @@ import { Dimension, DimensionJS, fromConfig as dimensionFromConfig, - serialize as dimensionSerialize, + serialize as serializeDimension, SerializedDimension } from "./dimension"; @@ -105,7 +105,7 @@ export interface SerializedDimensions { export function serialize({ tree, byName }: Dimensions): SerializedDimensions { return { tree, - byName: mapValues(byName, dimensionSerialize) + byName: mapValues(byName, serializeDimension) }; } diff --git a/src/common/models/measure/measures.ts b/src/common/models/measure/measures.ts index 76855527a..f6eed2885 100644 --- a/src/common/models/measure/measures.ts +++ b/src/common/models/measure/measures.ts @@ -24,7 +24,7 @@ import { fromConfig as measureFromConfig, Measure, MeasureJS, - serialize as measureSerialize, + serialize as serializeMeasure, SerializedMeasure } from "./measure"; @@ -107,7 +107,7 @@ export type ClientMeasures = Measures; export function serialize({ tree, byName }: Measures): SerializedMeasures { return { tree, - byName: mapValues(byName, measureSerialize) + byName: mapValues(byName, serializeMeasure) }; } diff --git a/src/common/models/sources/sources.ts b/src/common/models/sources/sources.ts index acfca22d0..9d09ac608 100644 --- a/src/common/models/sources/sources.ts +++ b/src/common/models/sources/sources.ts @@ -16,14 +16,21 @@ import { NamedArray } from "immutable-class"; import { isTruthy } from "../../utils/general/general"; -import { Cluster, ClusterJS } from "../cluster/cluster"; +import { + ClientCluster, + Cluster, + ClusterJS, + fromConfig as clusterFromConfig, + serialize as serializeCluster, + SerializedCluster +} from "../cluster/cluster"; import { findCluster } from "../cluster/find-cluster"; import { ClientDataCube, DataCube, DataCubeJS, fromConfig as dataCubeFromConfig, - serialize as dataCubeSerialize, + serialize as serializeDataCube, SerializedDataCube } from "../data-cube/data-cube"; import { isQueryable, QueryableDataCube } from "../data-cube/queryable-data-cube"; @@ -39,12 +46,12 @@ export interface Sources { } export interface SerializedSources { - clusters: ClusterJS[]; // SerializedCluster[] + clusters: SerializedCluster[]; dataCubes: SerializedDataCube[]; } export interface ClientSources { - readonly clusters: Cluster[]; + readonly clusters: ClientCluster[]; readonly dataCubes: ClientDataCube[]; } @@ -55,9 +62,9 @@ interface ClustersConfig { } function readClusters({ clusters, druidHost, brokerHost }: ClustersConfig): Cluster[] { - if (Array.isArray(clusters)) return clusters.map(cluster => Cluster.fromJS(cluster)); + if (Array.isArray(clusters)) return clusters.map(clusterFromConfig); if (isTruthy(druidHost) || isTruthy(brokerHost)) { - return [Cluster.fromJS({ + return [clusterFromConfig({ name: "druid", url: druidHost || brokerHost })]; @@ -89,14 +96,14 @@ export function fromConfig(config: SourcesJS): Sources { } export function serialize({ - clusters: serverClusters, - dataCubes: serverDataCubes - }: Sources): SerializedSources { - const clusters = serverClusters.map(c => c.toClientCluster().toJS()); + clusters: serverClusters, + dataCubes: serverDataCubes + }: Sources): SerializedSources { + const clusters = serverClusters.map(serializeCluster); const dataCubes = serverDataCubes .filter(dc => isQueryable(dc)) - .map(dataCubeSerialize); + .map(serializeDataCube); return { clusters, diff --git a/src/server/config.ts b/src/server/config.ts index f1c827e8b..87e1b480d 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -24,11 +24,9 @@ import { fromConfig as appSettingsFromConfig } from "../common/models/app-settings/app-settings"; import { - Cluster, - DEFAULT_SOURCE_LIST_REFRESH_INTERVAL, DEFAULT_SOURCE_LIST_REFRESH_ON_LOAD, - DEFAULT_SOURCE_REINTROSPECT_INTERVAL, DEFAULT_SOURCE_REINTROSPECT_ON_LOAD + fromConfig as clusterFromConfig } from "../common/models/cluster/cluster"; -import { fromConfig } from "../common/models/data-cube/data-cube"; +import { fromConfig as dataCubeFromConfig } from "../common/models/data-cube/data-cube"; import { fromConfig as sourcesFromConfig, SourcesJS } from "../common/models/sources/sources"; import { arraySum, isTruthy } from "../common/utils/general/general"; import { appSettingsToYaml, printExtra, sourcesToYaml } from "../common/utils/yaml-helper/yaml-helper"; @@ -224,24 +222,19 @@ if (START_SERVER) { } function readConfig(config: AppSettingsJS & SourcesJS) { - return { - appSettings: appSettingsFromConfig(config), - sources: sourcesFromConfig(config) - }; + return { + appSettings: appSettingsFromConfig(config), + sources: sourcesFromConfig(config) + }; } function readArgs(file: string | undefined, url: string | undefined) { const sources = { - clusters: !isTruthy(url) ? [] : [new Cluster({ + clusters: !isTruthy(url) ? [] : [clusterFromConfig({ name: "druid", - url, - sourceListScan: "auto", - sourceListRefreshInterval: DEFAULT_SOURCE_LIST_REFRESH_INTERVAL, - sourceListRefreshOnLoad: DEFAULT_SOURCE_LIST_REFRESH_ON_LOAD, - sourceReintrospectInterval: DEFAULT_SOURCE_REINTROSPECT_INTERVAL, - sourceReintrospectOnLoad: DEFAULT_SOURCE_REINTROSPECT_ON_LOAD + url })], - dataCubes: !isTruthy(file) ? [] : [fromConfig({ + dataCubes: !isTruthy(file) ? [] : [dataCubeFromConfig({ name: path.basename(file, path.extname(file)), clusterName: "native", source: file diff --git a/src/server/utils/cluster-manager/cluster-manager.ts b/src/server/utils/cluster-manager/cluster-manager.ts index 77caa98ba..ff9bd7bf1 100644 --- a/src/server/utils/cluster-manager/cluster-manager.ts +++ b/src/server/utils/cluster-manager/cluster-manager.ts @@ -19,7 +19,7 @@ import { External } from "plywood"; import { PlywoodRequester } from "plywood-base-api"; import { DruidRequestDecorator } from "plywood-druid-requester"; import { Logger } from "../../../common/logger/logger"; -import { Cluster } from "../../../common/models/cluster/cluster"; +import { Cluster, makeExternalFromSourceName, shouldScanSources } from "../../../common/models/cluster/cluster"; import { noop } from "../../../common/utils/functional/functional"; import { loadModule } from "../module-loader/module-loader"; import { DruidRequestDecoratorModule } from "../request-decorator/request-decorator"; @@ -194,7 +194,7 @@ export class ClusterManager { this.sourceListRefreshTimer = null; } - if (this.sourceListRefreshInterval && cluster.shouldScanSources()) { + if (this.sourceListRefreshInterval && shouldScanSources(cluster)) { logger.log(`Setting up sourceListRefresh timer in cluster '${cluster.name}' (every ${this.sourceListRefreshInterval}ms)`); this.sourceListRefreshTimer = setInterval( () => { @@ -316,7 +316,7 @@ export class ClusterManager { // See if any new sources were added to the cluster public scanSourceList = (): Promise => { const { logger, cluster, verbose } = this; - if (!cluster.shouldScanSources()) return Promise.resolve(null); + if (!shouldScanSources(cluster)) return Promise.resolve(null); logger.log(`Scanning cluster '${cluster.name}' for new sources`); return (External.getConstructorFor(cluster.type) as any).getSourceList(this.requester) @@ -348,7 +348,7 @@ export class ClusterManager { } else { logger.log(`Cluster '${cluster.name}' making external for '${source}'`); - const external = cluster.makeExternalFromSourceName(source, this.version).attachRequester(this.requester); + const external = makeExternalFromSourceName(source, this.version).attachRequester(this.requester); const newManagedExternal: ManagedExternal = { name: this.generateExternalName(external), external,