diff --git a/README.md b/README.md index 3f426e741..cf5c6519d 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ turnilo --druid broker_host:broker_port * [Configuration](docs/configuration.md) * [Generating Links](docs/generating-links.md) +* [Health checking](docs/health-checking.md) ## Development diff --git a/docs/configuration.md b/docs/configuration.md index 38a7e70a6..bf25f5224 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,6 +42,10 @@ A custom path to act as the server string. The Turnilo UI will be served from `http://turnilo-host:$port/` and `http://turnilo-host:$port/$serverRoot` +**healthEndpoint** (string), default "/health" + +A health endpoint location. See [Checking health of Turnilo instance](health-checking.md) + **iframe** ("allow" | "deny"), default "allow" Specify whether Turnilo will be allowed to run in an iFrame. @@ -97,6 +101,10 @@ Define this to override the automatic version detection. The timeout to set on the queries in ms. +**healthCheckTimeout** (number), default: 1000 + +The timeout for the cluster health checking request in ms. See [Checking health of Turnilo instance](health-checking.md) + **sourceListScan** ("auto" | "disable"), default: "auto" Should the sources of this cluster be automatically scanned and new sources added as data cubes. diff --git a/docs/health-checking.md b/docs/health-checking.md new file mode 100644 index 000000000..c00f6932f --- /dev/null +++ b/docs/health-checking.md @@ -0,0 +1,56 @@ +# Checking health of Turnilo instance + +Turnilo instance's health is defined in terms of being able to communicate with all configured Druid brokers +and those brokers knowing about all segments in Zookeeper. + +It can be checked by sending a GET request to a `healthEndpoint` path defined in Turnilo's server [configuration](configuration.md). + +Healthy Turnilo instance responds with HTTP status 200 while an unhealthy one responds with the status of 503. +The body of a response contains health status of all configured brokers with optional error message on unhealthy brokers. + +While processing the health checking request Turnilo server will send its own requests to all configured +druid clusters' brokers for `/druid/broker/v1/loadstatus` endpoint. It will check that all brokers responds within +individually defined cluster timeout (`healthCheckTimeout` property in [cluster properties](configuration.md#general-properties)) +and that the response body contains `inventoryInitialized` flag set to `true`. +If any of the requests to brokers fail to meet the criteria defined above the Turnilo instance is marked as unhealthy. + +# Response examples + +Healthy response example: +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 + +{ + "clusters": [ + { + "host": "localhost:8082", + "message": "", + "status": "healthy" + } + ], + "status": "healthy" +} +``` + +Unhealthy response example: +``` +HTTP/1.1 503 Service Unavailable +Content-Type: application/json; charset=utf-8 + +{ + "clusters": [ + { + "host": "localhost:8082", + "message": "inventory not initialized", + "status": "unhealthy" + }, + { + "host": "192.168.99.100:8082", + "message": "connection error: 'Error: ESOCKETTIMEDOUT'", + "status": "unhealthy" + } + ], + "status": "unhealthy" +} +``` diff --git a/package-lock.json b/package-lock.json index 4bdb6ba51..55e5d3da5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,12 @@ "@types/node": "8.5.2" } }, + "@types/caseless": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz", + "integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A==", + "dev": true + }, "@types/chai": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.0.10.tgz", @@ -71,6 +77,15 @@ "integrity": "sha512-fC12hKtEzVkrV/ZRcrmqvpHG/TMYDZtgpAmgMUA4F7KneDaQeFMwmPz8AfygKKJMqsdTi8bL+E+fciaaMLxUhg==", "dev": true }, + "@types/form-data": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "dev": true, + "requires": { + "@types/node": "8.5.2" + } + }, "@types/fs-promise": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@types/fs-promise/-/fs-promise-1.0.3.tgz", @@ -162,6 +177,15 @@ "@types/node": "8.5.2" } }, + "@types/nock": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/@types/nock/-/nock-9.1.3.tgz", + "integrity": "sha512-S8rJ+SaW82ICX87pZP62UcMifrMfjEdqNzSp+llx4YcvKw6bO650Ye6HwTqER1Dar3S40GIZECQisOrAICDCjA==", + "dev": true, + "requires": { + "@types/node": "8.5.2" + } + }, "@types/node": { "version": "8.5.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-8.5.2.tgz", @@ -246,6 +270,27 @@ "@types/react": "16.0.40" } }, + "@types/request": { + "version": "2.47.0", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.47.0.tgz", + "integrity": "sha512-/KXM5oev+nNCLIgBjkwbk8VqxmzI56woD4VUxn95O+YeQ8hJzcSmIZ1IN3WexiqBb6srzDo2bdMbsXxgXNkz5Q==", + "dev": true, + "requires": { + "@types/caseless": "0.12.1", + "@types/form-data": "2.2.1", + "@types/node": "8.5.2", + "@types/tough-cookie": "2.3.2" + } + }, + "@types/request-promise-native": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@types/request-promise-native/-/request-promise-native-1.0.14.tgz", + "integrity": "sha512-m6PNeopPU75gjN3+dD9AeWwm7h2QIOuLnmn143+Qs0bMYFyri9/bhCgikHlgzH0gk7xR48nef82GWeRV6N3DxA==", + "dev": true, + "requires": { + "@types/request": "2.47.0" + } + }, "@types/rewire": { "version": "2.5.28", "resolved": "https://registry.npmjs.org/@types/rewire/-/rewire-2.5.28.tgz", @@ -304,6 +349,12 @@ "integrity": "sha512-dEoVvo/I9QFomyhY+4Q6Qk+I+dhG59TYceZgC6Q0mCifVPErx6Y83PNTKGDS5e9h9Eti6q0S2mm16BU6iQK+3w==", "dev": true }, + "@types/tough-cookie": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.2.tgz", + "integrity": "sha512-vOVmaruQG5EatOU/jM6yU2uCp3Lz6mK1P5Ztu4iJjfM4SVHU9XYktPUQtKlIXuahqXHdEyUarMrBEwg5Cwu+bA==", + "dev": true + }, "@types/uglify-js": { "version": "2.6.30", "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-2.6.30.tgz", @@ -2116,6 +2167,12 @@ "type-detect": "4.0.5" } }, + "deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", + "dev": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -6639,6 +6696,43 @@ "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.0.0.tgz", "integrity": "sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA=" }, + "nock": { + "version": "9.2.5", + "resolved": "https://registry.npmjs.org/nock/-/nock-9.2.5.tgz", + "integrity": "sha512-ciCpyEq72Ws6/yhdayDfd0mAb3eQ7/533xKmFlBQZ5CDwrL0/bddtSicfL7R07oyvPAuegQrR+9ctrlPEp0EjQ==", + "dev": true, + "requires": { + "chai": "4.1.2", + "debug": "3.1.0", + "deep-equal": "1.0.1", + "json-stringify-safe": "5.0.1", + "lodash": "4.17.5", + "mkdirp": "0.5.1", + "propagate": "1.0.0", + "qs": "6.5.1", + "semver": "5.5.0" + }, + "dependencies": { + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", + "dev": true + }, + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", + "dev": true + }, + "semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", + "dev": true + } + } + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -8466,6 +8560,12 @@ "object-assign": "4.1.1" } }, + "propagate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-1.0.0.tgz", + "integrity": "sha1-AMLa7t2iDofjeCs0Stuhzd1q1wk=", + "dev": true + }, "property-information": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-3.2.0.tgz", diff --git a/package.json b/package.json index 448fa7501..7dcb98f76 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,8 @@ "react-dom": "16.2.0", "react-syntax-highlighter": "7.0.2", "react-copy-to-clipboard": "5.0.1", - "request": "2.83.0" + "request": "2.83.0", + "request-promise-native": "1.0.5" }, "devDependencies": { "@types/body-parser": "1.16.8", @@ -100,22 +101,24 @@ "@types/mime": "2.0.0", "@types/mocha": "2.2.46", "@types/moment-timezone": "0.5.4", + "@types/nock": "9.1.3", "@types/node": "8.5.2", "@types/nopt": "3.0.29", "@types/numeral": "0.0.22", "@types/q": "1.0.6", "@types/qajax": "0.0.29", "@types/react": "16.0.40", + "@types/react-copy-to-clipboard": "4.2.5", "@types/react-dom": "16.0.4", + "@types/react-syntax-highlighter": "0.0.5", "@types/react-transition-group": "2.0.7", + "@types/request-promise-native": "^1.0.14", "@types/rewire": "2.5.28", "@types/sinon": "4.1.4", "@types/superagent": "3.5.6", "@types/supertest": "2.0.4", "@types/webpack": "3.8.10", "@types/webpack-env": "1.13.3", - "@types/react-syntax-highlighter": "0.0.5", - "@types/react-copy-to-clipboard": "4.2.5", "awesome-typescript-loader": "3.4.1", "chai": "4.1.2", "create-react-class": "15.6.2", @@ -125,6 +128,7 @@ "immutable-class-tester": "0.5.12", "jsdom": "9.4.2", "mocha": "4.1.0", + "nock": "9.2.5", "node-sass": "4.7.2", "npm-run-all": "4.1.2", "react-hot-loader": "3.1.3", diff --git a/src/client/components/immutable-input/immutable-input.mocha.tsx b/src/client/components/immutable-input/immutable-input.mocha.tsx index 9a300e540..348f2cfa2 100644 --- a/src/client/components/immutable-input/immutable-input.mocha.tsx +++ b/src/client/components/immutable-input/immutable-input.mocha.tsx @@ -138,7 +138,7 @@ describe('ImmutableInput', () => { }); it('works for valid values', () => { - expect(node.value).to.equal('DRUID'); + expect(node.value).to.equal('DRUID-TWITTER'); node.value = 'GIRAFFE'; TestUtils.Simulate.change(node); @@ -157,7 +157,7 @@ describe('ImmutableInput', () => { }); it('works when an error is thrown', () => { - expect(node.value).to.equal('DRUID'); + expect(node.value).to.equal('DRUID-TWITTER'); node.value = 'PLATYPUS'; TestUtils.Simulate.change(node); diff --git a/src/common/models/app-settings/app-settings.mocha.ts b/src/common/models/app-settings/app-settings.mocha.ts index 04f10fe9d..c22163d0d 100644 --- a/src/common/models/app-settings/app-settings.mocha.ts +++ b/src/common/models/app-settings/app-settings.mocha.ts @@ -39,7 +39,7 @@ describe('AppSettings', () => { it("errors if there is no matching cluster", () => { var js = AppSettingsMock.wikiOnlyJS(); js.clusters = []; - expect(() => AppSettings.fromJS(js, context)).to.throw("Can not find cluster 'druid' for data cube 'wiki'"); + expect(() => AppSettings.fromJS(js, context)).to.throw("Can not find cluster 'druid-wiki' for data cube 'wiki'"); }); }); @@ -90,7 +90,7 @@ describe('AppSettings', () => { druidHost: '192.168.99.100', sourceListScan: 'disable', dataSources: [ - DataCubeMock.WIKI_JS + { ...DataCubeMock.WIKI_JS, clusterName: "druid" } ] }; @@ -122,7 +122,7 @@ describe('AppSettings', () => { expect(settings.toClientSettings().toJS()).to.deep.equal({ "clusters": [ { - "name": "druid", + "name": "druid-wiki", "type": "druid" } ], @@ -159,7 +159,7 @@ describe('AppSettings', () => { "unsplitable": true } ], - "clusterName": "druid", + "clusterName": "druid-wiki", "defaultDuration": "P3D", "defaultFilter": { "op": "literal", diff --git a/src/common/models/app-settings/app-settings.mock.ts b/src/common/models/app-settings/app-settings.mock.ts index 4437dde6d..4c2cac2fd 100644 --- a/src/common/models/app-settings/app-settings.mock.ts +++ b/src/common/models/app-settings/app-settings.mock.ts @@ -17,6 +17,7 @@ import { $, Executor, Dataset, basicExecutorFactory } from 'plywood'; import { MANIFESTS } from '../../../common/manifests/index'; +import { ClusterFixtures } from "../cluster/cluster.fixtures"; import { DataCubeMock } from '../data-cube/data-cube.mock'; import { CollectionMock } from '../collection/collection.mock'; import { AppSettings, AppSettingsJS, AppSettingsContext } from './app-settings'; @@ -293,18 +294,7 @@ export class AppSettingsMock { customLogoSvg: "ansvgstring" }, clusters: [ - { - name: 'druid', - type: 'druid', - host: '192.168.99.100', - version: '0.9.1', - timeout: 30000, - sourceListScan: 'auto', - sourceListRefreshInterval: 10000, - sourceReintrospectInterval: 10000, - - introspectionStrategy: 'segment-metadata-fallback' - } + ClusterFixtures.druidWikiClusterJS() ], dataCubes: [ DataCubeMock.WIKI_JS @@ -320,18 +310,7 @@ export class AppSettingsMock { customLogoSvg: "ansvgstring" }, clusters: [ - { - name: 'druid', - type: 'druid', - host: '192.168.99.100', - version: '0.9.1', - timeout: 30000, - sourceListScan: 'auto', - sourceListRefreshInterval: 10000, - sourceReintrospectInterval: 10000, - - introspectionStrategy: 'segment-metadata-fallback' - } + ClusterFixtures.druidWikiClusterJS() ], dataCubes: [ DataCubeMock.WIKI_JS @@ -346,18 +325,8 @@ export class AppSettingsMock { title: "Hello World" }, clusters: [ - { - name: 'druid', - type: 'druid', - host: '192.168.99.100', - version: '0.9.1', - timeout: 30000, - sourceListScan: 'auto', - sourceListRefreshInterval: 10000, - sourceReintrospectInterval: 10000, - - introspectionStrategy: "segment-metadata-fallback" - } + ClusterFixtures.druidWikiClusterJS(), + ClusterFixtures.druidTwitterClusterJS() ], dataCubes: [ DataCubeMock.WIKI_JS, diff --git a/src/common/models/cluster/cluster.fixtures.ts b/src/common/models/cluster/cluster.fixtures.ts new file mode 100644 index 000000000..4f277089b --- /dev/null +++ b/src/common/models/cluster/cluster.fixtures.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2017-2018 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClusterJS } from "./cluster"; + +export class ClusterFixtures { + static druidWikiClusterJS(): ClusterJS { + return { + name: 'druid-wiki', + type: 'druid', + host: '192.168.99.100', + version: '0.9.1', + timeout: 30000, + healthCheckTimeout: 50, + sourceListScan: 'auto', + sourceListRefreshInterval: 10000, + sourceReintrospectInterval: 10000, + + introspectionStrategy: 'segment-metadata-fallback' + }; + } + + static druidTwitterClusterJS(): ClusterJS { + return { + name: 'druid-twitter', + type: 'druid', + host: '192.168.99.101', + version: '0.9.1', + timeout: 30000, + healthCheckTimeout: 200, + sourceListScan: 'auto', + sourceListRefreshInterval: 10000, + sourceReintrospectInterval: 10000, + + introspectionStrategy: 'segment-metadata-fallback' + }; + } +} diff --git a/src/common/models/cluster/cluster.mocha.ts b/src/common/models/cluster/cluster.mocha.ts index 65a907c91..5237c334e 100644 --- a/src/common/models/cluster/cluster.mocha.ts +++ b/src/common/models/cluster/cluster.mocha.ts @@ -33,6 +33,7 @@ describe('Cluster', () => { host: '192.168.99.100', version: '0.9.1', timeout: 30000, + healthCheckTimeout: 50, sourceListScan: 'auto', sourceListRefreshOnLoad: true, sourceListRefreshInterval: 10000, diff --git a/src/common/models/cluster/cluster.ts b/src/common/models/cluster/cluster.ts index dcc74dc31..0f999a722 100644 --- a/src/common/models/cluster/cluster.ts +++ b/src/common/models/cluster/cluster.ts @@ -29,6 +29,7 @@ export interface ClusterValue { host?: string; version?: string; timeout?: number; + healthCheckTimeout?: number; sourceListScan?: SourceListScan; sourceListRefreshOnLoad?: boolean; sourceListRefreshInterval?: number; @@ -51,6 +52,7 @@ export interface ClusterJS { host?: string; version?: string; timeout?: number; + healthCheckTimeout?: number; sourceListScan?: SourceListScan; sourceListRefreshOnLoad?: boolean; sourceListRefreshInterval?: number; @@ -82,6 +84,7 @@ function ensureNotTiny(v: number): void { export class Cluster extends BaseImmutable { static TYPE_VALUES: SupportedType[] = ['druid', 'mysql', 'postgres']; static DEFAULT_TIMEOUT = 40000; + static DEFAULT_HEALTH_CHECK_TIMEOUT = 1000; static DEFAULT_SOURCE_LIST_SCAN: SourceListScan = 'auto'; static SOURCE_LIST_SCAN_VALUES: SourceListScan[] = ['disable', 'auto']; static DEFAULT_SOURCE_LIST_REFRESH_INTERVAL = 0; @@ -117,6 +120,7 @@ export class Cluster extends BaseImmutable { { name: 'title', defaultValue: '' }, { name: 'version', defaultValue: null }, { name: 'timeout', defaultValue: Cluster.DEFAULT_TIMEOUT }, + { name: 'healthCheckTimeout', defaultValue: Cluster.DEFAULT_HEALTH_CHECK_TIMEOUT }, { name: 'sourceListScan', defaultValue: Cluster.DEFAULT_SOURCE_LIST_SCAN, possibleValues: Cluster.SOURCE_LIST_SCAN_VALUES }, { name: 'sourceListRefreshOnLoad', defaultValue: Cluster.DEFAULT_SOURCE_LIST_REFRESH_ON_LOAD }, { name: 'sourceListRefreshInterval', defaultValue: Cluster.DEFAULT_SOURCE_LIST_REFRESH_INTERVAL, validate: [BaseImmutable.ensure.number, ensureNotTiny] }, @@ -141,6 +145,7 @@ export class Cluster extends BaseImmutable { public title: string; public version: string; public timeout: number; + public healthCheckTimeout: number; public sourceListScan: SourceListScan; public sourceListRefreshOnLoad: boolean; public sourceListRefreshInterval: number; diff --git a/src/common/models/data-cube/data-cube.mock.ts b/src/common/models/data-cube/data-cube.mock.ts index 1311607b6..d3c64a919 100644 --- a/src/common/models/data-cube/data-cube.mock.ts +++ b/src/common/models/data-cube/data-cube.mock.ts @@ -31,7 +31,7 @@ export class DataCubeMock { name: 'wiki', title: 'Wiki', description: 'Wiki description', - clusterName: 'druid', + clusterName: 'druid-wiki', source: 'wiki', introspection: 'none', attributes: [ @@ -127,7 +127,7 @@ export class DataCubeMock { name: 'twitter', title: 'Twitter', description: 'Twitter description should go here', - clusterName: 'druid', + clusterName: 'druid-twitter', source: 'twitter', introspection: 'none', dimensions: [ diff --git a/src/server/app.ts b/src/server/app.ts index 59ef2dbec..2fb802e40 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -136,8 +136,6 @@ if (app.get('env') === 'development') { // NODE_ENV } -addRoutes('/health', healthRoutes); - addRoutes('/', express.static(path.join(__dirname, '../../build/public'))); addRoutes('/', express.static(path.join(__dirname, '../../assets'))); @@ -190,6 +188,8 @@ if (AUTH) { }); } +addRoutes(SERVER_SETTINGS.getHealthEndpoint(), healthRoutes); + // Data routes addRoutes('/plywood', plywoodRoutes); addRoutes('/plyql', plyqlRoutes); diff --git a/src/server/models/server-settings/server-settings.mocha.ts b/src/server/models/server-settings/server-settings.mocha.ts index 8f077b1f7..e8cf21942 100644 --- a/src/server/models/server-settings/server-settings.mocha.ts +++ b/src/server/models/server-settings/server-settings.mocha.ts @@ -49,6 +49,7 @@ describe('ServerSettings', () => { port: 9091, serverHost: '10.20.30.40', serverRoot: '/swivs', + healthEndpoint: '/status/health', pageMustLoadTimeout: 901 }, { diff --git a/src/server/models/server-settings/server-settings.ts b/src/server/models/server-settings/server-settings.ts index b819ec99d..055137076 100644 --- a/src/server/models/server-settings/server-settings.ts +++ b/src/server/models/server-settings/server-settings.ts @@ -26,6 +26,7 @@ export interface ServerSettingsValue { port?: number; serverHost?: string; serverRoot?: string; + healthEnpoint?: string; requestLogFormat?: string; trackingUrl?: string; trackingContext?: Record; @@ -53,6 +54,7 @@ function basicEqual(a: any, b: any): boolean { export class ServerSettings extends BaseImmutable { static DEFAULT_PORT = 9090; static DEFAULT_SERVER_ROOT = '/turnilo'; + static DEFAULT_HEALTH_ENDPOINT = '/health'; static DEFAULT_REQUEST_LOG_FORMAT = 'common'; static DEFAULT_PAGE_MUST_LOAD_TIMEOUT = 800; static IFRAME_VALUES: Iframe[] = ["allow", "deny"]; @@ -78,6 +80,7 @@ export class ServerSettings extends BaseImmutable; @@ -109,6 +113,7 @@ export class ServerSettings extends BaseImmutable number; public getServerHost: () => string; public getServerRoot: () => string; + public getHealthEndpoint: () => string; public getRequestLogFormat: () => string; public getTrackingUrl: () => string; public getTrackingContext: () => Record; diff --git a/src/server/routes/health/health.mocha.ts b/src/server/routes/health/health.mocha.ts index 5b49f6712..9ad9e4678 100644 --- a/src/server/routes/health/health.mocha.ts +++ b/src/server/routes/health/health.mocha.ts @@ -15,20 +15,116 @@ * limitations under the License. */ -import * as express from 'express'; +import * as express from "express"; +import { Express, RequestHandler, Response } from "express"; +import * as http from "http"; +import * as nock from "nock"; +import * as Q from "q"; import * as supertest from 'supertest'; +import { AppSettings } from "../../../common/models"; +import { AppSettingsMock } from "../../../common/models/app-settings/app-settings.mock"; +import { ClusterFixtures } from "../../../common/models/cluster/cluster.fixtures"; +import { SwivRequest } from "../../utils"; +import { GetSettingsOptions } from "../../utils/settings-manager/settings-manager"; + import * as healthRouter from './health'; -var app = express(); +const appSettingsHandlerProvider = (appSettings: AppSettings): RequestHandler => { + return (req: SwivRequest, res: Response, next: Function) => { + req.user = null; + req.version = '0.9.4'; + req.getSettings = (dataCubeOfInterest?: GetSettingsOptions) => Q(appSettings); + next(); + }; +}; + +const mockLoadStatus = (nock: nock.Scope, fixture: { status: int, initialized: boolean, delay: int }) => { + const { status, initialized, delay } = fixture; + nock + .get(loadStatusPath) + .delay(delay) + .reply(status, { inventoryInitialized: initialized }); +}; -app.use('/', healthRouter); +const appSettings = AppSettingsMock.wikiOnly(); +const loadStatusPath = "/druid/broker/v1/loadstatus"; +const wikiBrokerNock = nock(`http://${ClusterFixtures.druidWikiClusterJS().host}`); +const twitterBrokerNock = nock(`http://${ClusterFixtures.druidTwitterClusterJS().host}`); describe('health router', () => { - it('gets a 200', (testComplete: any) => { - supertest(app) - .get('/') - .expect(200, testComplete); + let app: Express; + let server: http.Server; + + describe("single druid cluster", () => { + before((done) => { + app = express(); + app.use(appSettingsHandlerProvider(appSettings)); + app.use('/', healthRouter); + server = app.listen(0, done); + }); + + after((done) => { + server.close(done); + }); + + const singleClusterTests = [ + { scenario: "healthy broker", status: 200, initialized: true, delay: 0, expectedStatus: 200 }, + { scenario: "unhealthy broker", status: 500, initialized: false, delay: 0, expectedStatus: 503 }, + { scenario: "uninitialized broker", status: 200, initialized: false, delay: 0, expectedStatus: 503 }, + { scenario: "timeout to broker", status: 200, initialized: true, delay: 200, expectedStatus: 503 } + ]; + + singleClusterTests.forEach(({ scenario, status, initialized, delay, expectedStatus }) => { + it(`returns ${expectedStatus} with ${scenario}`, (testComplete) => { + mockLoadStatus(wikiBrokerNock, { status, initialized, delay }); + supertest(app) + .get('/') + .expect(expectedStatus, testComplete); + }); + }); + }); + + describe("multiple druid clusters", () => { + before((done) => { + app = express(); + app.use(appSettingsHandlerProvider(AppSettingsMock.wikiTwitter())); + app.use('/', healthRouter); + server = app.listen(0, done); + }); + + after((done) => { + server.close(done); + }); + + const multipleClustersTests = [ + { scenario: "all healthy brokers", + wikiBroker: { status: 200, initialized: true, delay: 0 }, + twitterBroker: { status: 200, initialized: true, delay: 0 }, + expectedStatus: 200 }, + { scenario: "single unhealthy broker", + wikiBroker: { status: 500, initialized: true, delay: 0 }, + twitterBroker: { status: 200, initialized: true, delay: 0 }, + expectedStatus: 503 }, + { scenario: "single uninitialized broker", + wikiBroker: { status: 200, initialized: true, delay: 0 }, + twitterBroker: { status: 200, initialized: false, delay: 0 }, + expectedStatus: 503 }, + { scenario: "timeout to single broker", + wikiBroker: { status: 200, initialized: true, delay: 100 }, + twitterBroker: { status: 200, initialized: true, delay: 0 }, + expectedStatus: 503 } + ]; + + multipleClustersTests.forEach(({ scenario, wikiBroker, twitterBroker, expectedStatus }) => { + it(`returns ${expectedStatus} with ${scenario}`, (testComplete) => { + mockLoadStatus(wikiBrokerNock, wikiBroker); + mockLoadStatus(twitterBrokerNock, twitterBroker); + supertest(app) + .get('/') + .expect(expectedStatus, testComplete); + }); + }); }); }); diff --git a/src/server/routes/health/health.ts b/src/server/routes/health/health.ts index ab6804630..19b5a9ef3 100644 --- a/src/server/routes/health/health.ts +++ b/src/server/routes/health/health.ts @@ -15,12 +15,88 @@ * limitations under the License. */ -import { Router, Request, Response } from 'express'; +import { Router, Response } from 'express'; +import * as request from "request-promise-native"; +import { Cluster } from "../../../common/models"; +import { SwivRequest } from "../../utils"; var router = Router(); -router.get('/', (req: Request, res: Response) => { - res.send(`I am healthy @ ${new Date().toISOString()}`); +router.get('/', (req: SwivRequest, res: Response) => { + req + .getSettings() + .then((appSettings) => appSettings.clusters) + .then(checkClusters) + .then((clusterHealths) => emitHealthStatus(clusterHealths)(res)) + .catch((reason) => res.status(unhealthyHttpStatus).send({ status: ClusterHealthStatus.unhealthy, message: reason.message })); }); +const unhealthyHttpStatus = 503; +const healthyHttpStatus = 200; + +enum ClusterHealthStatus { + healthy = "healthy", + unhealthy = "unhealthy" +} + +const statusToHttpStatusMap: { [status in ClusterHealthStatus]: number } = { + healthy: healthyHttpStatus, + unhealthy: unhealthyHttpStatus +}; + +interface ClusterHealth { + host: string; + status: ClusterHealthStatus; + message: string; +} + +const checkClusters = (clusters: Cluster[]): Promise => { + const promises = clusters + .filter((cluster) => (cluster.type === "druid")) + .map(checkDruidCluster); + + return Promise.all(promises); +}; + +const checkDruidCluster = (cluster: Cluster): Promise => { + const { host } = cluster; + const loadStatusUrl = `http://${cluster.host}/druid/broker/v1/loadstatus`; + + return request + .get(loadStatusUrl, { json: true, timeout: cluster.healthCheckTimeout }) + .promise() + .then((loadStatus) => { + const { inventoryInitialized } = loadStatus; + + if (inventoryInitialized) { + return { host, status: ClusterHealthStatus.healthy, message: "" }; + } else { + return { host, status: ClusterHealthStatus.unhealthy, message: "inventory not initialized" }; + } + }) + .catch((reason) => { + let reasonMessage: string; + if (reason != null && reason instanceof Error) { + reasonMessage = reason.message; + } + return { host, status: ClusterHealthStatus.unhealthy, message: `connection error: '${reasonMessage}'` }; + }); +}; + +const emitHealthStatus = (clusterHealths: ClusterHealth[]): (res: Response) => void => { + return (response: Response) => { + const overallHealth = clusterHealths + .map((clusterHealth) => (clusterHealth.status)) + .reduce(healthStatusReducer, ClusterHealthStatus.healthy); + + const httpState = statusToHttpStatusMap[overallHealth]; + + response.status(httpState).send({ status: overallHealth, clusters: clusterHealths }); + }; +}; + +const healthStatusReducer = (before: ClusterHealthStatus, current: ClusterHealthStatus): ClusterHealthStatus => { + return current === ClusterHealthStatus.unhealthy ? current : before; +}; + export = router; diff --git a/src/server/routes/swiv/swiv.mocha.ts b/src/server/routes/swiv/swiv.mocha-skip.ts similarity index 100% rename from src/server/routes/swiv/swiv.mocha.ts rename to src/server/routes/swiv/swiv.mocha-skip.ts