From 0732a9defbc7e4966bae045fcc0abdd4d72b145d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Tue, 18 Feb 2020 18:05:00 +0000 Subject: [PATCH 01/11] [Telemetry] Report the Application Usage (time of usage + number of clicks) --- .../core_plugins/application_usage/index.ts | 31 ++ .../application_usage/mappings.ts | 28 ++ .../application_usage/package.json | 4 + .../ui/public/chrome/api/sub_url_hooks.js | 9 + .../ui/public/new_platform/new_platform.ts | 2 + src/plugins/application_usage/kibana.json | 9 + .../application_usage/public/constants.ts | 34 ++ src/plugins/application_usage/public/index.ts | 25 ++ .../application_usage/public/plugin.ts | 178 +++++++++ .../application_usage/server/index.test.ts | 27 ++ src/plugins/application_usage/server/index.ts | 25 ++ .../application_usage/server/plugin.ts | 350 ++++++++++++++++++ 12 files changed, 722 insertions(+) create mode 100644 src/legacy/core_plugins/application_usage/index.ts create mode 100644 src/legacy/core_plugins/application_usage/mappings.ts create mode 100644 src/legacy/core_plugins/application_usage/package.json create mode 100644 src/plugins/application_usage/kibana.json create mode 100644 src/plugins/application_usage/public/constants.ts create mode 100644 src/plugins/application_usage/public/index.ts create mode 100644 src/plugins/application_usage/public/plugin.ts create mode 100644 src/plugins/application_usage/server/index.test.ts create mode 100644 src/plugins/application_usage/server/index.ts create mode 100644 src/plugins/application_usage/server/plugin.ts diff --git a/src/legacy/core_plugins/application_usage/index.ts b/src/legacy/core_plugins/application_usage/index.ts new file mode 100644 index 0000000000000..752d6eaa19bb0 --- /dev/null +++ b/src/legacy/core_plugins/application_usage/index.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Legacy } from '../../../../kibana'; +import { mappings } from './mappings'; + +// eslint-disable-next-line import/no-default-export +export default function ApplicationUsagePlugin(kibana: any) { + const config: Legacy.PluginSpecOptions = { + id: 'application_usage', + uiExports: { mappings }, // Needed to define the mappings for the SavedObjects + }; + + return new kibana.Plugin(config); +} diff --git a/src/legacy/core_plugins/application_usage/mappings.ts b/src/legacy/core_plugins/application_usage/mappings.ts new file mode 100644 index 0000000000000..3ae5d0e5b892e --- /dev/null +++ b/src/legacy/core_plugins/application_usage/mappings.ts @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const mappings = { + application_usage: { + properties: { + appId: { type: 'keyword' }, + numberOfClicks: { type: 'long' }, + minutesOnScreen: { type: 'float' }, + }, + }, +}; diff --git a/src/legacy/core_plugins/application_usage/package.json b/src/legacy/core_plugins/application_usage/package.json new file mode 100644 index 0000000000000..5ab10a2f8d237 --- /dev/null +++ b/src/legacy/core_plugins/application_usage/package.json @@ -0,0 +1,4 @@ +{ + "name": "application_usage", + "version": "kibana" +} \ No newline at end of file diff --git a/src/legacy/ui/public/chrome/api/sub_url_hooks.js b/src/legacy/ui/public/chrome/api/sub_url_hooks.js index 3ff262f546e3c..4ce83fbb962fb 100644 --- a/src/legacy/ui/public/chrome/api/sub_url_hooks.js +++ b/src/legacy/ui/public/chrome/api/sub_url_hooks.js @@ -21,6 +21,7 @@ import url from 'url'; import { unhashUrl } from '../../../../../plugins/kibana_utils/public'; import { toastNotifications } from '../../notify/toasts'; +import { npStart } from '../../new_platform'; export function registerSubUrlHooks(angularModule, internals) { angularModule.run(($rootScope, Private, $location) => { @@ -40,6 +41,7 @@ export function registerSubUrlHooks(angularModule, internals) { function onRouteChange($event) { if (subUrlRouteFilter($event)) { + updateUsage($event); updateSubUrls(); } } @@ -67,6 +69,13 @@ export function registerSubUrlHooks(angularModule, internals) { }); } +function updateUsage($event) { + const scope = $event.targetScope; + const app = scope.chrome.getApp(); + const appId = app.id === 'kibana' ? scope.getFirstPathSegment() : app.id; + npStart.plugins.applicationUsage?.__LEGACY.appChanged(appId); +} + /** * Creates a function that will be called on each route change * to determine if the event should be used to update the last diff --git a/src/legacy/ui/public/new_platform/new_platform.ts b/src/legacy/ui/public/new_platform/new_platform.ts index b7994c7f68afb..d653856a906ff 100644 --- a/src/legacy/ui/public/new_platform/new_platform.ts +++ b/src/legacy/ui/public/new_platform/new_platform.ts @@ -20,6 +20,7 @@ import { IScope } from 'angular'; import { UiActionsStart, UiActionsSetup } from 'src/plugins/ui_actions/public'; import { IEmbeddableStart, IEmbeddableSetup } from 'src/plugins/embeddable/public'; +import { ApplicationUsagePluginStart } from 'src/plugins/application_usage/public'; import { LegacyCoreSetup, LegacyCoreStart, App, AppMountDeprecated } from '../../../../core/public'; import { Plugin as DataPlugin } from '../../../../plugins/data/public'; import { Plugin as ExpressionsPlugin } from '../../../../plugins/expressions/public'; @@ -82,6 +83,7 @@ export interface PluginsStart { management: ManagementStart; advancedSettings: AdvancedSettingsStart; telemetry?: TelemetryPluginStart; + applicationUsage?: ApplicationUsagePluginStart; } export const npSetup = { diff --git a/src/plugins/application_usage/kibana.json b/src/plugins/application_usage/kibana.json new file mode 100644 index 0000000000000..5f01d676d52e3 --- /dev/null +++ b/src/plugins/application_usage/kibana.json @@ -0,0 +1,9 @@ +{ + "id": "applicationUsage", + "version": "kibana", + "server": true, + "ui": true, + "optionalPlugins": [ + "usageCollection" + ] +} diff --git a/src/plugins/application_usage/public/constants.ts b/src/plugins/application_usage/public/constants.ts new file mode 100644 index 0000000000000..5a97b828dc445 --- /dev/null +++ b/src/plugins/application_usage/public/constants.ts @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Key for the localStorage service + */ +export const LOCALSTORAGE_KEY = 'application_usage.aggregated'; +export const LOCALSTORAGE_KEY_LAST_REPORTED = 'application_usage.lastReported'; + +/** + * List of appIds not to report usage from (due to legacy hacks) + */ +export const DO_NOT_REPORT = ['kibana']; + +/** + * Report to server every 10 minutes + */ +export const REPORT_INTERVAL = 10 * 60 * 1000; diff --git a/src/plugins/application_usage/public/index.ts b/src/plugins/application_usage/public/index.ts new file mode 100644 index 0000000000000..e7b883dafecdb --- /dev/null +++ b/src/plugins/application_usage/public/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ApplicationUsagePlugin } from './plugin'; +export { ApplicationUsagePluginStart, ApplicationUsagePluginSetup } from './plugin'; + +export function plugin() { + return new ApplicationUsagePlugin(); +} diff --git a/src/plugins/application_usage/public/plugin.ts b/src/plugins/application_usage/public/plugin.ts new file mode 100644 index 0000000000000..2c76cd96135ed --- /dev/null +++ b/src/plugins/application_usage/public/plugin.ts @@ -0,0 +1,178 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Plugin, CoreStart, CoreSetup } from 'kibana/public'; +import moment, { Moment } from 'moment'; +import { filter, distinctUntilChanged, map } from 'rxjs/operators'; +import { Subject, merge } from 'rxjs'; +import { Storage } from '../../kibana_utils/public'; +import { + LOCALSTORAGE_KEY, + LOCALSTORAGE_KEY_LAST_REPORTED, + DO_NOT_REPORT, + REPORT_INTERVAL, +} from './constants'; + +export type ApplicationUsagePluginSetup = void; +export interface ApplicationUsagePluginStart { + __LEGACY: { + /** + * Legacy handler so we can report the actual app being used inside "kibana#/{appId}". + * To be removed when we get rid of the legacy world + * + * @deprecated + */ + appChanged: (appId: string) => void; + }; +} + +interface CurrentUsage { + appId: string; + startTime: Moment; + numberOfClicks: number; +} + +interface AggregatedUsage { + [appId: string]: { + minutesOnScreen: number; + numberOfClicks: number; + }; +} + +export class ApplicationUsagePlugin + implements Plugin { + private readonly legacyAppId$ = new Subject(); + private readonly localStorage = new Storage(window.localStorage); + private currentUsage?: CurrentUsage; + private lastAppId?: string; + private readonly infraSubApps = new Set(); + + public setup({}: CoreSetup): ApplicationUsagePluginSetup {} + + public start({ http, application }: CoreStart): ApplicationUsagePluginStart { + merge(application.currentAppId$, this.legacyAppId$) + .pipe( + filter(appId => typeof appId === 'string' && !DO_NOT_REPORT.includes(appId)), + map(appId => { + if (appId === 'infra') { + // Hack for infra because of legacy multiple ways of registering apps + const [, hash] = (window.location.hash || '').match(/#\/(\w+)\//) || []; + this.infraSubApps.add(hash); + return hash; + } + return appId; + }), + distinctUntilChanged() + ) + .subscribe(appId => appId && this.appChanged(appId)); + + // Before leaving the page, make sure we store the current usage + window.addEventListener('beforeunload', () => this.onUnload()); + + // Hack for legacy apps that only change the hash ('infra' explicitly) + window.addEventListener('hashchange', () => { + if (this.infraSubApps.has(this.lastAppId || '')) { + this.legacyAppId$.next('infra'); + } + }); + + // Monitoring dashboards might be open in background and we are fine with that + // but we don't want to report hours if the user goes to another tab and Kibana is not shown + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible' && this.lastAppId) { + this.appChanged(this.lastAppId); + } else if (document.visibilityState === 'hidden') { + this.onUnload(); + this.reportUsage(http); + } + }); + + // Count any clicks and assign it to the current app + window.addEventListener('click', () => this.currentUsage && this.currentUsage.numberOfClicks++); + + // Send the data to the server every REPORT_INTERVAL + setInterval(() => this.reportUsage(http), REPORT_INTERVAL); + // If we haven't reported to the server in a long time, trigger it now + const { lastReported } = this.localStorage.get(LOCALSTORAGE_KEY_LAST_REPORTED) || { + lastReported: 0, + }; + if (moment().diff(lastReported, 'milliseconds') > REPORT_INTERVAL) { + this.reportUsage(http); + } + + return { + __LEGACY: { + appChanged: appId => this.legacyAppId$.next(appId), + }, + }; + } + + private async reportUsage(http: CoreStart['http']) { + // Ensure we store, at least, the last usage of the current app. + if (this.currentUsage) this.appChanged(this.currentUsage.appId); + + const existingData = this.localStorage.get(LOCALSTORAGE_KEY) as AggregatedUsage | null; + if (!existingData) { + return; + } + this.localStorage.remove(LOCALSTORAGE_KEY); + const usage = Object.entries(existingData).map(([appId, value]) => ({ appId, ...value })); + try { + await http.post('/api/application-usage', { body: JSON.stringify({ usage }) }); + this.localStorage.set(LOCALSTORAGE_KEY_LAST_REPORTED, { lastReported: moment() }); + } catch (err) { + // We failed to post the data, let's save it back to the local storage and we'll try later + usage.forEach(({ appId, numberOfClicks, minutesOnScreen }) => + this.mergeAggregatedUsage(appId, numberOfClicks, minutesOnScreen) + ); + } + } + + private onUnload() { + if (this.currentUsage) this.aggregateUsage(this.currentUsage); + this.currentUsage = void 0; + } + + private appChanged(appId: string) { + if (this.currentUsage) this.aggregateUsage(this.currentUsage); + + this.lastAppId = appId; + this.currentUsage = { appId, startTime: moment(), numberOfClicks: 0 }; + } + + private aggregateUsage({ appId, startTime, numberOfClicks }: CurrentUsage) { + this.mergeAggregatedUsage(appId, numberOfClicks, moment().diff(startTime, 'minutes', true)); + } + + private mergeAggregatedUsage(appId: string, numberOfClicks: number, minutesOnScreen: number) { + const existingData = (this.localStorage.get(LOCALSTORAGE_KEY) || {}) as AggregatedUsage; + + const appExistingData = existingData[appId] || { + minutesOnScreen: 0, + numberOfClicks: 0, + }; + + this.localStorage.set(LOCALSTORAGE_KEY, { + ...existingData, + [appId]: { + minutesOnScreen: appExistingData.minutesOnScreen + minutesOnScreen, + numberOfClicks: appExistingData.numberOfClicks + numberOfClicks, + }, + }); + } +} diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts new file mode 100644 index 0000000000000..71933976399b4 --- /dev/null +++ b/src/plugins/application_usage/server/index.test.ts @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { coreMock } from '../../../core/server/mocks'; +import { plugin } from './'; + +describe('ApplicationUsagePlugin/server', () => { + const applicationUsagePlugin = plugin(coreMock); + + // TODO: Add tests +}); diff --git a/src/plugins/application_usage/server/index.ts b/src/plugins/application_usage/server/index.ts new file mode 100644 index 0000000000000..382e701d80140 --- /dev/null +++ b/src/plugins/application_usage/server/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PluginInitializerContext } from 'kibana/server'; +import { ApplicationUsagePlugin } from './plugin'; + +export function plugin(initializerContext: PluginInitializerContext) { + return new ApplicationUsagePlugin(initializerContext); +} diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts new file mode 100644 index 0000000000000..1b096a47e5d7f --- /dev/null +++ b/src/plugins/application_usage/server/plugin.ts @@ -0,0 +1,350 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + Plugin, + CoreSetup, + CoreStart, + IRouter, + ICustomClusterClient, + ISavedObjectsRepository, + PluginInitializerContext, + Logger, + SavedObjectAttributes, +} from 'kibana/server'; +import { schema } from '@kbn/config-schema'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { SearchResponse } from 'elasticsearch'; + +/** + * Roll indices every 24h + */ +const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000; + +/** + * Set index to store the time-based records + */ +const INDEX_APP_USAGE = '.kibana-application-usage'; + +/** + * Plugin type: used for saved objects and telemetry + */ +const PLUGIN_TYPE = 'application_usage'; + +export interface ApplicationUsageTelemetryReport { + [appId: string]: { + clicks_total: number; + clicks_30_days: number; + clicks_90_days: number; + minutes_on_screen_total: number; + minutes_on_screen_30_days: number; + minutes_on_screen_90_days: number; + }; +} + +export interface ApplicationUsagePluginDepsSetup { + usageCollection?: UsageCollectionSetup; +} + +interface ApplicationUsageSavedObject extends SavedObjectAttributes { + appId: string; + minutesOnScreen: number; + numberOfClicks: number; +} + +interface SearchAggregationBucket { + key: string; + doc_count: number; + perDay: { + buckets: { + [key in 'last30Days' | 'last90Days' | 'total']: { + doc_count: number; + minutesOnScreen: { value: number }; + numberOfClicks: { value: number }; + }; + }; + }; +} + +interface SearchAggregationResult extends SearchResponse { + aggregations?: { + appId: { + buckets: SearchAggregationBucket[]; + }; + }; +} + +export class ApplicationUsagePlugin implements Plugin { + private readonly log: Logger; + private intervalId?: NodeJS.Timer; + private esClient?: ICustomClusterClient; + private savedObjectsClient?: ISavedObjectsRepository; + + constructor({ logger }: PluginInitializerContext) { + this.log = logger.get(); + } + + public async setup( + { http, elasticsearch }: CoreSetup, + { usageCollection }: ApplicationUsagePluginDepsSetup + ) { + const router = http.createRouter(); + this.registerIndexRoute(router); + + this.esClient = elasticsearch.createClient('application-usage'); + await this.ensureIndex(this.esClient); + + if (usageCollection) { + const usageCollector = usageCollection.makeUsageCollector({ + isReady: () => true, + type: PLUGIN_TYPE, + fetch: callCluster => this.fetchUsage(callCluster), + }); + usageCollection.registerCollector(usageCollector); + } + } + + public async start({ savedObjects }: CoreStart) { + const savedObjectsClient = (this.savedObjectsClient = savedObjects.createInternalRepository()); + this.intervalId = setInterval( + () => + this.esClient && + this.rollTotals(this.esClient, savedObjectsClient).catch(err => { + this.log.warn(`Failed to roll totals`, err); + }), + ROLL_INDICES_INTERVAL + ); + await this.rollTotals(this.esClient!, savedObjectsClient); + } + + public stop() { + if (this.intervalId) clearInterval(this.intervalId); + } + + private registerIndexRoute(router: IRouter) { + router.post( + { + path: '/api/application-usage', + validate: { + body: schema.object({ + usage: schema.arrayOf( + schema.object({ + appId: schema.string(), + numberOfClicks: schema.number(), + minutesOnScreen: schema.number(), + }) + ), + }), + }, + }, + async (context, req, res) => { + const { usage } = req.body; + const now = new Date().toISOString(); + const _index = INDEX_APP_USAGE; + await context.core.elasticsearch.dataClient.callAsInternalUser('bulk', { + body: usage.reduce( + (acc, { appId, numberOfClicks, minutesOnScreen }) => [ + ...acc, + { index: { _index } }, + { timestamp: now, appId, numberOfClicks, minutesOnScreen }, + ], + [] as object[] + ), + }); + return res.ok(); + } + ); + } + + private async ensureIndex(elasticsearch: ICustomClusterClient) { + await elasticsearch.callAsInternalUser('indices.putTemplate', { + name: INDEX_APP_USAGE, + body: { + index_patterns: `${INDEX_APP_USAGE}`, + settings: { + number_of_shards: 1, + }, + mappings: { + properties: { + timestamp: { type: 'date' }, + appId: { type: 'keyword' }, + numberOfClicks: { type: 'long' }, + minutesOnScreen: { type: 'float' }, + }, + }, + }, + }); + } + + private async getSavedObjectTotals() { + if (!this.savedObjectsClient) { + return {}; + } + try { + const { saved_objects } = await this.savedObjectsClient.find({ + type: PLUGIN_TYPE, + perPage: 100, + }); + return saved_objects.reduce( + (acc, { attributes: { appId, minutesOnScreen, numberOfClicks } }) => ({ + ...acc, + [appId]: { + minutesOnScreen: minutesOnScreen + (acc[appId]?.minutesOnScreen || 0), + numberOfClicks: numberOfClicks + (acc[appId]?.numberOfClicks || 0), + }, + }), + {} as { [appId: string]: { minutesOnScreen: number; numberOfClicks: number } } + ); + } catch (err) { + if (err.output?.statusCode === 404) { + return {}; + } else { + throw err; + } + } + } + + private async rollTotals( + elasticsearch: ICustomClusterClient, + savedObjectsClient: ISavedObjectsRepository + ) { + // Query for everything older than 90d + const query = { bool: { filter: { range: { timestamp: { lte: 'now-90d' } } } } }; + const { aggregations } = await this.fetchAggregation(elasticsearch.callAsInternalUser, query); + + const attributes = await this.getSavedObjectTotals(); + const newTotals = (aggregations?.appId.buckets || []).map(({ key, perDay }) => { + const { numberOfClicks, minutesOnScreen } = attributes[key] || { + numberOfClicks: 0, + minutesOnScreen: 0, + }; + return { + appId: key, + numberOfClicks: numberOfClicks + perDay.buckets.total.numberOfClicks.value, + minutesOnScreen: minutesOnScreen + perDay.buckets.total.minutesOnScreen.value, + }; + }); + if (newTotals.length === 0) { + return; + } + await savedObjectsClient.bulkCreate( + newTotals.map(entry => ({ + type: PLUGIN_TYPE, + id: entry.appId, + attributes: entry, + })), + { overwrite: true } + ); + await elasticsearch.callAsInternalUser('deleteByQuery', { + index: INDEX_APP_USAGE, + ignore_unavailable: true, + allow_no_indices: true, + body: { query }, + }); + } + + private fetchAggregation( + callCluster: ICustomClusterClient['callAsInternalUser'], + query: object = { match_all: {} } + ): Promise { + const lastNDays = (numberOfDays: number) => ({ + [`last${numberOfDays}Days`]: { + bool: { + filter: { + range: { + timestamp: { + gte: `now-${numberOfDays}d`, + lte: 'now', + }, + }, + }, + }, + }, + }); + + return callCluster('search', { + index: INDEX_APP_USAGE, + ignore_unavailable: true, + allow_no_indices: true, + body: { + size: 0, + query, + aggs: { + appId: { + terms: { + field: 'appId', + }, + aggs: { + perDay: { + filters: { + filters: { + ...lastNDays(30), + ...lastNDays(90), + total: { match_all: {} }, + }, + }, + aggs: { + numberOfClicks: { sum: { field: 'numberOfClicks' } }, + minutesOnScreen: { sum: { field: 'minutesOnScreen' } }, + }, + }, + }, + }, + }, + }, + }); + } + + private async fetchUsage(callCluster: ICustomClusterClient['callAsInternalUser']) { + const response = await this.fetchAggregation(callCluster); + const totals = await this.getSavedObjectTotals(); + + const usage = (response.aggregations?.appId.buckets || []).reduce( + (acc, { key, perDay }) => ({ + ...acc, + [key]: { + clicks_total: + perDay.buckets.total.numberOfClicks.value + (totals[key]?.numberOfClicks || 0), + clicks_30_days: perDay.buckets.last30Days.numberOfClicks.value, + clicks_90_days: perDay.buckets.last90Days.numberOfClicks.value, + minutes_on_screen_total: + perDay.buckets.total.minutesOnScreen.value + (totals[key]?.minutesOnScreen || 0), + minutes_on_screen_30_days: perDay.buckets.last30Days.minutesOnScreen.value, + minutes_on_screen_90_days: perDay.buckets.last90Days.minutesOnScreen.value, + }, + }), + {} as ApplicationUsageTelemetryReport + ); + + return Object.entries(totals).reduce( + (acc, [key, { numberOfClicks, minutesOnScreen }]) => ({ + ...acc, + [key]: acc[key] || { + clicks_total: numberOfClicks, + clicks_30_days: 0, + clicks_90_days: 0, + minutes_on_screen_total: minutesOnScreen, + minutes_on_screen_30_days: 0, + minutes_on_screen_90_days: 0, + }, + }), + usage + ); + } +} From 0eaf76372f8405641e8f54ed141121e5ff4e9d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Wed, 19 Feb 2020 14:39:53 +0000 Subject: [PATCH 02/11] Add Unit tests to the server side --- .../application_usage/server/index.test.ts | 251 +++++++++++++++++- .../application_usage/server/plugin.ts | 5 +- 2 files changed, 253 insertions(+), 3 deletions(-) diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts index 71933976399b4..c04222b7d4210 100644 --- a/src/plugins/application_usage/server/index.test.ts +++ b/src/plugins/application_usage/server/index.test.ts @@ -17,11 +17,260 @@ * under the License. */ +import { RequestHandler } from 'kibana/server'; import { coreMock } from '../../../core/server/mocks'; import { plugin } from './'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { mockRouter } from '../../../core/server/http/router/router.mock'; + +class UsageCollection { + public usageCollector: any; + public makeUsageCollector = (opts: any) => opts; + public registerCollector = (opts: any) => (this.usageCollector = opts); +} describe('ApplicationUsagePlugin/server', () => { - const applicationUsagePlugin = plugin(coreMock); + const applicationUsagePlugin = plugin(coreMock.createPluginInitializerContext()); + + test('it registers the routes, ensures the index template but does not register the usage collector because it is not initialised', async () => { + await applicationUsagePlugin.setup(coreMock.createSetup(), {}); + }); + + test('it registers the routes, ensures the index template and registers the usage collector', async () => { + const usageCollection = new UsageCollection() as any; + await applicationUsagePlugin.setup(coreMock.createSetup(), { usageCollection }); + expect(usageCollection.usageCollector).not.toBeUndefined(); + }); + + describe('fetchUsage', () => { + const usageCollection = new UsageCollection() as any; + + beforeAll(async () => { + await applicationUsagePlugin.setup(coreMock.createSetup(), { usageCollection }); + }); + + test('it should not fail if no aggregations are returned', async () => { + const esClient = jest.fn((method, options) => { + expect(method).toBe('search'); + return {}; + }); + + expect(usageCollection.usageCollector.isReady()).toBe(true); + const usage = await usageCollection.usageCollector.fetch(esClient); + expect(usage).toStrictEqual({}); + }); + + test('it should match an aggregation of application statistics', async () => { + const esClient = jest.fn((method, options) => { + expect(method).toBe('search'); + return { + aggregations: { + appId: { + buckets: [ + { + key: 'test-app', + perDay: { + buckets: { + total: { numberOfClicks: { value: 10 }, minutesOnScreen: { value: 20 } }, + last30Days: { numberOfClicks: { value: 1 }, minutesOnScreen: { value: 2 } }, + last90Days: { numberOfClicks: { value: 5 }, minutesOnScreen: { value: 10 } }, + }, + }, + }, + { + key: 'test-plugin', + perDay: { + buckets: { + total: { numberOfClicks: { value: 20 }, minutesOnScreen: { value: 40 } }, + last30Days: { numberOfClicks: { value: 2 }, minutesOnScreen: { value: 4 } }, + last90Days: { numberOfClicks: { value: 15 }, minutesOnScreen: { value: 20 } }, + }, + }, + }, + ], + }, + }, + }; + }); + + const usage = await usageCollection.usageCollector.fetch(esClient); + expect(usage).toStrictEqual({ + 'test-app': { + clicks_total: 10, + clicks_30_days: 1, + clicks_90_days: 5, + minutes_on_screen_total: 20, + minutes_on_screen_30_days: 2, + minutes_on_screen_90_days: 10, + }, + 'test-plugin': { + clicks_total: 20, + clicks_30_days: 2, + clicks_90_days: 15, + minutes_on_screen_total: 40, + minutes_on_screen_30_days: 4, + minutes_on_screen_90_days: 20, + }, + }); + }); + + test('Fetch the usage while adding the totals from the saved objects', async () => { + jest.useFakeTimers(); + + const localUsageCollection = new UsageCollection() as any; + + const esClient = jest.fn((method, options) => { + if (method !== 'search') { + return {}; + } + return { + aggregations: { + appId: { + buckets: [ + { + key: 'test-app', + perDay: { + buckets: { + total: { numberOfClicks: { value: 10 }, minutesOnScreen: { value: 20 } }, + last30Days: { numberOfClicks: { value: 1 }, minutesOnScreen: { value: 2 } }, + last90Days: { numberOfClicks: { value: 5 }, minutesOnScreen: { value: 10 } }, + }, + }, + }, + { + key: 'test-plugin', + perDay: { + buckets: { + total: { numberOfClicks: { value: 20 }, minutesOnScreen: { value: 40 } }, + last30Days: { numberOfClicks: { value: 2 }, minutesOnScreen: { value: 4 } }, + last90Days: { numberOfClicks: { value: 15 }, minutesOnScreen: { value: 20 } }, + }, + }, + }, + ], + }, + }, + }; + }); + + const appPlugin = plugin(coreMock.createPluginInitializerContext()); + + const coreSetup = coreMock.createSetup(); + coreSetup.elasticsearch.createClient.mockImplementation( + () => + ({ + callAsInternalUser: esClient, + } as any) + ); + + await appPlugin.setup(coreSetup, { usageCollection: localUsageCollection }); + + const coreStart = coreMock.createStart(); + coreStart.savedObjects.createInternalRepository.mockImplementation( + () => + ({ + find: (opts: any) => ({ + saved_objects: [ + { + attributes: { appId: 'total-test-app', minutesOnScreen: 90, numberOfClicks: 100 }, + }, + { + attributes: { appId: 'test-app', minutesOnScreen: 95, numberOfClicks: 105 }, + }, + ], + }), + bulkCreate: (opts: any) => { + expect(opts).toStrictEqual([ + { + type: 'application_usage', + id: 'test-app', + attributes: { appId: 'test-app', minutesOnScreen: 115, numberOfClicks: 115 }, + }, + { + type: 'application_usage', + id: 'test-plugin', + attributes: { appId: 'test-plugin', minutesOnScreen: 40, numberOfClicks: 20 }, + }, + ]); + }, + } as any) + ); + + await appPlugin.start(coreStart); + + const usage = await localUsageCollection.usageCollector.fetch(esClient); + expect(usage).toStrictEqual({ + 'total-test-app': { + clicks_total: 100, + clicks_30_days: 0, + clicks_90_days: 0, + minutes_on_screen_total: 90, + minutes_on_screen_30_days: 0, + minutes_on_screen_90_days: 0, + }, + 'test-app': { + clicks_total: 115, + clicks_30_days: 1, + clicks_90_days: 5, + minutes_on_screen_total: 115, + minutes_on_screen_30_days: 2, + minutes_on_screen_90_days: 10, + }, + 'test-plugin': { + clicks_total: 20, + clicks_30_days: 2, + clicks_90_days: 15, + minutes_on_screen_total: 40, + minutes_on_screen_30_days: 4, + minutes_on_screen_90_days: 20, + }, + }); + + jest.runTimersToTime(24 * 60 * 60 * 1000); + appPlugin.stop(); + }); + }); + + test('POST /api/application-usage handler', async () => { + let routeHandler: RequestHandler; + const appPlugin = plugin(coreMock.createPluginInitializerContext()); + const coreSetup = coreMock.createSetup(); + coreSetup.http.createRouter.mockImplementation(() => { + const router = mockRouter.create({}); + router.post.mockImplementation((options, handler) => (routeHandler = handler)); + return router; + }); + await appPlugin.setup(coreSetup, {}); + + expect(routeHandler!).not.toBeUndefined(); + const context = coreMock.createRequestHandlerContext(); + context.elasticsearch.dataClient.callAsInternalUser.mockImplementation( + async (method, options) => { + expect(method).toBe('bulk'); + expect(options).toHaveProperty('body'); + expect(options!.body).toBeInstanceOf(Array); + const body = options!.body.map(({ timestamp, ...rest }: any) => rest); + expect(body).toStrictEqual([ + { index: { _index: '.kibana-application-usage' } }, + { appId: 'test-app', numberOfClicks: 10, minutesOnScreen: 9 }, + { index: { _index: '.kibana-application-usage' } }, + { appId: 'test-plugin', numberOfClicks: 11, minutesOnScreen: 8 }, + ]); + } + ); + await routeHandler!( + { core: context }, + { + body: { + usage: [ + { appId: 'test-app', numberOfClicks: 10, minutesOnScreen: 9 }, + { appId: 'test-plugin', numberOfClicks: 11, minutesOnScreen: 8 }, + ], + }, + } as any, + { ok: jest.fn() } as any + ); + }); // TODO: Add tests }); diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts index 1b096a47e5d7f..a31803af09b04 100644 --- a/src/plugins/application_usage/server/plugin.ts +++ b/src/plugins/application_usage/server/plugin.ts @@ -259,7 +259,7 @@ export class ApplicationUsagePlugin implements Plugin { }); } - private fetchAggregation( + private async fetchAggregation( callCluster: ICustomClusterClient['callAsInternalUser'], query: object = { match_all: {} } ): Promise { @@ -278,7 +278,7 @@ export class ApplicationUsagePlugin implements Plugin { }, }); - return callCluster('search', { + const response = await callCluster('search', { index: INDEX_APP_USAGE, ignore_unavailable: true, allow_no_indices: true, @@ -309,6 +309,7 @@ export class ApplicationUsagePlugin implements Plugin { }, }, }); + return response || {}; } private async fetchUsage(callCluster: ICustomClusterClient['callAsInternalUser']) { From 8c2cc289a6a269cb7d4bbfc457aae443a46d9feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Thu, 20 Feb 2020 11:11:18 +0000 Subject: [PATCH 03/11] Do not use optional chaining in JS --- src/legacy/ui/public/chrome/api/sub_url_hooks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/legacy/ui/public/chrome/api/sub_url_hooks.js b/src/legacy/ui/public/chrome/api/sub_url_hooks.js index 4ce83fbb962fb..8a1c048af08a2 100644 --- a/src/legacy/ui/public/chrome/api/sub_url_hooks.js +++ b/src/legacy/ui/public/chrome/api/sub_url_hooks.js @@ -73,7 +73,7 @@ function updateUsage($event) { const scope = $event.targetScope; const app = scope.chrome.getApp(); const appId = app.id === 'kibana' ? scope.getFirstPathSegment() : app.id; - npStart.plugins.applicationUsage?.__LEGACY.appChanged(appId); + if (npStart.plugins.applicationUsage) npStart.plugins.applicationUsage.__LEGACY.appChanged(appId); } /** From 799540f75141e4e515bf47a32ffe658dc7bbf58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Thu, 20 Feb 2020 14:19:05 +0000 Subject: [PATCH 04/11] Add tests on the public end --- .../public/__snapshots__/index.test.ts.snap | 57 +++++ .../application_usage/public/index.test.ts | 206 ++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 src/plugins/application_usage/public/__snapshots__/index.test.ts.snap create mode 100644 src/plugins/application_usage/public/index.test.ts diff --git a/src/plugins/application_usage/public/__snapshots__/index.test.ts.snap b/src/plugins/application_usage/public/__snapshots__/index.test.ts.snap new file mode 100644 index 0000000000000..7f8c492ca7283 --- /dev/null +++ b/src/plugins/application_usage/public/__snapshots__/index.test.ts.snap @@ -0,0 +1,57 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ApplicationUsagePlugin/public click handler 1`] = ` +Object { + "appId": "test", + "numberOfClicks": 0, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public click handler 2`] = ` +Object { + "appId": "test", + "numberOfClicks": 1, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public hashchange handler => when there is an "infra" app (and hash) 1`] = ` +Object { + "appId": "test", + "numberOfClicks": 0, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public hashchange handler => when there is an "infra" app (and new hash) 1`] = ` +Object { + "appId": "testInfraPlugin", + "numberOfClicks": 0, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public hashchange handler => when there is an "infra" app (and no hash change) 1`] = ` +Object { + "appId": "test", + "numberOfClicks": 0, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public the currentUsage should be initially undefined until an app is changed 1`] = ` +Object { + "appId": "test-app", + "numberOfClicks": 0, + "startTime": Any, +} +`; + +exports[`ApplicationUsagePlugin/public visibilitychange handler => visible 1`] = ` +Object { + "appId": "testInfraPlugin", + "numberOfClicks": 0, + "startTime": Any, +} +`; diff --git a/src/plugins/application_usage/public/index.test.ts b/src/plugins/application_usage/public/index.test.ts new file mode 100644 index 0000000000000..d535551f03873 --- /dev/null +++ b/src/plugins/application_usage/public/index.test.ts @@ -0,0 +1,206 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment'; +import { coreMock } from '../../../core/public/mocks'; +import { plugin } from './'; +import { LOCALSTORAGE_KEY, REPORT_INTERVAL } from './constants'; + +function getHandler(spy: jest.SpyInstance, event: string) { + const [, handlerCb] = spy.mock.calls.find(([method, fn]) => method === event) || []; + return handlerCb; +} + +describe('ApplicationUsagePlugin/public', () => { + jest.useFakeTimers(); + + const locationSpy = jest.spyOn(window, 'location', 'get'); + const windowAddListenerSpy = jest.spyOn(window, 'addEventListener'); + const documentAddListenerSpy = jest.spyOn(document, 'addEventListener'); + const visibilityStateSpy = jest.spyOn(document, 'visibilityState', 'get'); + + const appUsagePlugin = plugin(); + + const localStorageSpy = { + get: jest.spyOn((appUsagePlugin as any).localStorage, 'get'), + set: jest.spyOn((appUsagePlugin as any).localStorage, 'set'), + remove: jest.spyOn((appUsagePlugin as any).localStorage, 'remove'), + }; + + appUsagePlugin.setup(coreMock.createSetup()); + const coreStart = coreMock.createStart(); + const { + __LEGACY: { appChanged }, + } = appUsagePlugin.start(coreStart); + + const beforeunloadCb = getHandler(windowAddListenerSpy, 'beforeunload'); + const hashchangeCb = getHandler(windowAddListenerSpy, 'hashchange'); + const clickCb = getHandler(windowAddListenerSpy, 'click'); + const visibilitychangeCb = getHandler(documentAddListenerSpy, 'visibilitychange'); + + beforeEach(() => { + localStorageSpy.get.mockReset(); + localStorageSpy.set.mockReset(); + localStorageSpy.remove.mockReset(); + visibilityStateSpy.mockReset(); + }); + + test('all listeners are initialised', () => { + expect(beforeunloadCb).toBeInstanceOf(Function); + expect(hashchangeCb).toBeInstanceOf(Function); + expect(clickCb).toBeInstanceOf(Function); + expect(visibilitychangeCb).toBeInstanceOf(Function); + }); + + test('visibilitychange handler => back to visible', () => { + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + expect((appUsagePlugin as any).lastAppId).toBeUndefined(); + visibilityStateSpy.mockReturnValue('visible'); + visibilitychangeCb(); + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + expect(localStorageSpy.set).not.toHaveBeenCalled(); + }); + + test('the currentUsage should be initially undefined until an app is changed', () => { + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + appChanged('test-app'); + expect((appUsagePlugin as any).lastAppId).toBe('test-app'); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'test-app', + startTime: expect.any(moment), + numberOfClicks: 0, + }); + }); + + test('beforeunload handler => it should clean the currentUsage and update the localStorage', () => { + expect((appUsagePlugin as any).currentUsage).not.toBeUndefined(); + beforeunloadCb(); + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + expect(localStorageSpy.set).toHaveBeenLastCalledWith(LOCALSTORAGE_KEY, { + 'test-app': { + numberOfClicks: 0, + minutesOnScreen: expect.any(Number), + }, + }); + }); + + test('hashchange handler => when no app is around', () => { + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + hashchangeCb(); + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + expect(localStorageSpy.set).not.toHaveBeenCalled(); + }); + + test('hashchange handler => when there is an "infra" app', () => { + // Infra app but no hash => nothing to call + appChanged('infra'); + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + expect(localStorageSpy.set).not.toHaveBeenCalled(); + }); + + test('hashchange handler => when there is an "infra" app (and hash)', () => { + locationSpy.mockReturnValue({ hash: '#/test/' } as any); + + // Infra app with hash => set the currentUsage + appChanged('infra'); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'test', + numberOfClicks: 0, + startTime: expect.any(moment), + }); + expect(localStorageSpy.set).not.toHaveBeenCalled(); + }); + + test('hashchange handler => when there is an "infra" app (and no hash change)', () => { + locationSpy.mockReturnValue({ hash: '#/test/' } as any); + + // No change in the hash, so nothing else should be done + hashchangeCb(); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'test', + numberOfClicks: 0, + startTime: expect.any(moment), + }); + expect(localStorageSpy.set).not.toHaveBeenCalled(); + }); + + test('hashchange handler => when there is an "infra" app (and new hash)', () => { + locationSpy.mockReturnValue({ hash: '#/testInfraPlugin/' } as any); + + // Infra app with a new hash => set the currentUsage + hashchangeCb(); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'testInfraPlugin', + numberOfClicks: 0, + startTime: expect.any(moment), + }); + expect(localStorageSpy.set).toHaveBeenLastCalledWith(LOCALSTORAGE_KEY, { + test: { + numberOfClicks: 0, + minutesOnScreen: expect.any(Number), + }, + }); + }); + + test('visibilitychange handler => visible', () => { + visibilityStateSpy.mockReturnValue('visible'); + visibilitychangeCb(); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'testInfraPlugin', + numberOfClicks: 0, + startTime: expect.any(moment), + }); + expect(localStorageSpy.set).toHaveBeenLastCalledWith(LOCALSTORAGE_KEY, { + testInfraPlugin: { + numberOfClicks: 0, + minutesOnScreen: expect.any(Number), + }, + }); + }); + + test('visibilitychange handler => hidden', () => { + localStorageSpy.get.mockReturnValue({ appId: { numberOfClicks: 0, minutesOnScreen: 3 } }); + expect((appUsagePlugin as any).currentUsage).not.toBeUndefined(); + visibilityStateSpy.mockReturnValue('hidden'); + visibilitychangeCb(); + expect(localStorageSpy.set).toHaveBeenLastCalledWith(LOCALSTORAGE_KEY, { + appId: { numberOfClicks: 0, minutesOnScreen: 3 }, + testInfraPlugin: { + numberOfClicks: 0, + minutesOnScreen: expect.any(Number), + }, + }); + expect((appUsagePlugin as any).currentUsage).toBeUndefined(); + }); + + test('click handler', () => { + appChanged('test'); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'test', + numberOfClicks: 0, + startTime: expect.any(moment), + }); + clickCb(); + expect((appUsagePlugin as any).currentUsage).toMatchSnapshot({ + appId: 'test', + numberOfClicks: 1, + startTime: expect.any(moment), + }); + }); +}); From 665dd4e3cef237fc5fad0ef21365ab4b0f5a6820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Thu, 20 Feb 2020 14:23:33 +0000 Subject: [PATCH 05/11] Fix jslint errors --- src/plugins/application_usage/public/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/application_usage/public/index.test.ts b/src/plugins/application_usage/public/index.test.ts index d535551f03873..f552d17c9daab 100644 --- a/src/plugins/application_usage/public/index.test.ts +++ b/src/plugins/application_usage/public/index.test.ts @@ -20,7 +20,7 @@ import moment from 'moment'; import { coreMock } from '../../../core/public/mocks'; import { plugin } from './'; -import { LOCALSTORAGE_KEY, REPORT_INTERVAL } from './constants'; +import { LOCALSTORAGE_KEY } from './constants'; function getHandler(spy: jest.SpyInstance, event: string) { const [, handlerCb] = spy.mock.calls.find(([method, fn]) => method === event) || []; From 91fca2e666adb8bd1c219702d91904771b131f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Thu, 20 Feb 2020 20:35:37 +0000 Subject: [PATCH 06/11] jest.useFakeTimers() + jest.clearAllTimers() --- src/plugins/application_usage/public/index.test.ts | 2 ++ src/plugins/application_usage/server/index.test.ts | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/plugins/application_usage/public/index.test.ts b/src/plugins/application_usage/public/index.test.ts index f552d17c9daab..454574e6395e5 100644 --- a/src/plugins/application_usage/public/index.test.ts +++ b/src/plugins/application_usage/public/index.test.ts @@ -30,6 +30,8 @@ function getHandler(spy: jest.SpyInstance, event: string) { describe('ApplicationUsagePlugin/public', () => { jest.useFakeTimers(); + afterAll(() => jest.clearAllTimers()); + const locationSpy = jest.spyOn(window, 'location', 'get'); const windowAddListenerSpy = jest.spyOn(window, 'addEventListener'); const documentAddListenerSpy = jest.spyOn(document, 'addEventListener'); diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts index c04222b7d4210..1cb51b6f66e63 100644 --- a/src/plugins/application_usage/server/index.test.ts +++ b/src/plugins/application_usage/server/index.test.ts @@ -30,8 +30,11 @@ class UsageCollection { } describe('ApplicationUsagePlugin/server', () => { + jest.useFakeTimers(); const applicationUsagePlugin = plugin(coreMock.createPluginInitializerContext()); + afterAll(() => jest.clearAllTimers()); + test('it registers the routes, ensures the index template but does not register the usage collector because it is not initialised', async () => { await applicationUsagePlugin.setup(coreMock.createSetup(), {}); }); @@ -115,8 +118,6 @@ describe('ApplicationUsagePlugin/server', () => { }); test('Fetch the usage while adding the totals from the saved objects', async () => { - jest.useFakeTimers(); - const localUsageCollection = new UsageCollection() as any; const esClient = jest.fn((method, options) => { From 2e35ef5f9170b810b6ca4ba7d51cb4e69c460ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Fri, 21 Feb 2020 07:56:12 +0000 Subject: [PATCH 07/11] Remove Jest timer handlers from my tests (only affecting to a minimum coverage bit) --- src/plugins/application_usage/public/index.test.ts | 4 ---- src/plugins/application_usage/server/index.test.ts | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/plugins/application_usage/public/index.test.ts b/src/plugins/application_usage/public/index.test.ts index 454574e6395e5..6616cf7923288 100644 --- a/src/plugins/application_usage/public/index.test.ts +++ b/src/plugins/application_usage/public/index.test.ts @@ -28,10 +28,6 @@ function getHandler(spy: jest.SpyInstance, event: string) { } describe('ApplicationUsagePlugin/public', () => { - jest.useFakeTimers(); - - afterAll(() => jest.clearAllTimers()); - const locationSpy = jest.spyOn(window, 'location', 'get'); const windowAddListenerSpy = jest.spyOn(window, 'addEventListener'); const documentAddListenerSpy = jest.spyOn(document, 'addEventListener'); diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts index 1cb51b6f66e63..4bf717f26aec5 100644 --- a/src/plugins/application_usage/server/index.test.ts +++ b/src/plugins/application_usage/server/index.test.ts @@ -30,11 +30,8 @@ class UsageCollection { } describe('ApplicationUsagePlugin/server', () => { - jest.useFakeTimers(); const applicationUsagePlugin = plugin(coreMock.createPluginInitializerContext()); - afterAll(() => jest.clearAllTimers()); - test('it registers the routes, ensures the index template but does not register the usage collector because it is not initialised', async () => { await applicationUsagePlugin.setup(coreMock.createSetup(), {}); }); @@ -227,7 +224,6 @@ describe('ApplicationUsagePlugin/server', () => { }, }); - jest.runTimersToTime(24 * 60 * 60 * 1000); appPlugin.stop(); }); }); From 9226712785a416befd3fac31a39e6a1e1301ee17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Fri, 21 Feb 2020 09:37:48 +0000 Subject: [PATCH 08/11] Catch ES actions in the setup/start steps because it broke core_services tests --- .../application_usage/public/index.test.ts | 7 + .../application_usage/server/index.test.ts | 13 +- .../application_usage/server/plugin.ts | 127 ++++++++++-------- 3 files changed, 93 insertions(+), 54 deletions(-) diff --git a/src/plugins/application_usage/public/index.test.ts b/src/plugins/application_usage/public/index.test.ts index 6616cf7923288..cdfe0625c5577 100644 --- a/src/plugins/application_usage/public/index.test.ts +++ b/src/plugins/application_usage/public/index.test.ts @@ -28,6 +28,13 @@ function getHandler(spy: jest.SpyInstance, event: string) { } describe('ApplicationUsagePlugin/public', () => { + beforeAll(() => jest.useFakeTimers()); + + afterAll(() => { + jest.clearAllTimers(); + jest.resetAllMocks(); + }); + const locationSpy = jest.spyOn(window, 'location', 'get'); const windowAddListenerSpy = jest.spyOn(window, 'addEventListener'); const documentAddListenerSpy = jest.spyOn(document, 'addEventListener'); diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts index 4bf717f26aec5..510cee38adc18 100644 --- a/src/plugins/application_usage/server/index.test.ts +++ b/src/plugins/application_usage/server/index.test.ts @@ -22,6 +22,7 @@ import { coreMock } from '../../../core/server/mocks'; import { plugin } from './'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { mockRouter } from '../../../core/server/http/router/router.mock'; +import { ApplicationUsagePlugin } from './plugin'; class UsageCollection { public usageCollector: any; @@ -30,7 +31,16 @@ class UsageCollection { } describe('ApplicationUsagePlugin/server', () => { - const applicationUsagePlugin = plugin(coreMock.createPluginInitializerContext()); + let applicationUsagePlugin: ApplicationUsagePlugin; + beforeEach(() => { + jest.useFakeTimers(); + applicationUsagePlugin = plugin(coreMock.createPluginInitializerContext()); + }); + + afterEach(() => { + jest.clearAllTimers(); + jest.resetAllMocks(); + }); test('it registers the routes, ensures the index template but does not register the usage collector because it is not initialised', async () => { await applicationUsagePlugin.setup(coreMock.createSetup(), {}); @@ -224,6 +234,7 @@ describe('ApplicationUsagePlugin/server', () => { }, }); + jest.runTimersToTime(24 * 60 * 60 * 1000); appPlugin.stop(); }); }); diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts index a31803af09b04..264e54c636ba6 100644 --- a/src/plugins/application_usage/server/plugin.ts +++ b/src/plugins/application_usage/server/plugin.ts @@ -95,6 +95,7 @@ export class ApplicationUsagePlugin implements Plugin { private intervalId?: NodeJS.Timer; private esClient?: ICustomClusterClient; private savedObjectsClient?: ISavedObjectsRepository; + private indexTemplateInitialised = false; constructor({ logger }: PluginInitializerContext) { this.log = logger.get(); @@ -108,7 +109,14 @@ export class ApplicationUsagePlugin implements Plugin { this.registerIndexRoute(router); this.esClient = elasticsearch.createClient('application-usage'); - await this.ensureIndex(this.esClient); + try { + await this.ensureIndex(this.esClient); + } catch (err) { + // If I don't catch this, I'll get errors in core_services.test.ts tests :o + this.log.warn( + `Failed to ensure the index. I'll try again later when I need to store any info.` + ); + } if (usageCollection) { const usageCollector = usageCollection.makeUsageCollector({ @@ -123,14 +131,10 @@ export class ApplicationUsagePlugin implements Plugin { public async start({ savedObjects }: CoreStart) { const savedObjectsClient = (this.savedObjectsClient = savedObjects.createInternalRepository()); this.intervalId = setInterval( - () => - this.esClient && - this.rollTotals(this.esClient, savedObjectsClient).catch(err => { - this.log.warn(`Failed to roll totals`, err); - }), + () => this.esClient && this.rollTotals(this.esClient, savedObjectsClient), ROLL_INDICES_INTERVAL ); - await this.rollTotals(this.esClient!, savedObjectsClient); + this.rollTotals(this.esClient!, savedObjectsClient); } public stop() { @@ -157,6 +161,7 @@ export class ApplicationUsagePlugin implements Plugin { const { usage } = req.body; const now = new Date().toISOString(); const _index = INDEX_APP_USAGE; + if (this.indexTemplateInitialised === false) await this.ensureIndex(this.esClient!); await context.core.elasticsearch.dataClient.callAsInternalUser('bulk', { body: usage.reduce( (acc, { appId, numberOfClicks, minutesOnScreen }) => [ @@ -173,23 +178,35 @@ export class ApplicationUsagePlugin implements Plugin { } private async ensureIndex(elasticsearch: ICustomClusterClient) { - await elasticsearch.callAsInternalUser('indices.putTemplate', { - name: INDEX_APP_USAGE, - body: { - index_patterns: `${INDEX_APP_USAGE}`, - settings: { - number_of_shards: 1, - }, - mappings: { - properties: { - timestamp: { type: 'date' }, - appId: { type: 'keyword' }, - numberOfClicks: { type: 'long' }, - minutesOnScreen: { type: 'float' }, + // Skip if already done + if (this.indexTemplateInitialised === false) return; + + const mappings = { + properties: { + timestamp: { type: 'date' }, + appId: { type: 'keyword' }, + numberOfClicks: { type: 'long' }, + minutesOnScreen: { type: 'float' }, + }, + }; + + if (await elasticsearch.callAsInternalUser('indices.exists', { index: INDEX_APP_USAGE })) { + await elasticsearch.callAsInternalUser('indices.putMapping', { + index: INDEX_APP_USAGE, + body: mappings, + }); + } else { + await elasticsearch.callAsInternalUser('indices.create', { + index: INDEX_APP_USAGE, + body: { + settings: { + number_of_shards: 1, }, + mappings, }, - }, - }); + }); + } + this.indexTemplateInitialised = true; } private async getSavedObjectTotals() { @@ -224,39 +241,43 @@ export class ApplicationUsagePlugin implements Plugin { elasticsearch: ICustomClusterClient, savedObjectsClient: ISavedObjectsRepository ) { - // Query for everything older than 90d - const query = { bool: { filter: { range: { timestamp: { lte: 'now-90d' } } } } }; - const { aggregations } = await this.fetchAggregation(elasticsearch.callAsInternalUser, query); + try { + // Query for everything older than 90d + const query = { bool: { filter: { range: { timestamp: { lte: 'now-90d' } } } } }; + const { aggregations } = await this.fetchAggregation(elasticsearch.callAsInternalUser, query); - const attributes = await this.getSavedObjectTotals(); - const newTotals = (aggregations?.appId.buckets || []).map(({ key, perDay }) => { - const { numberOfClicks, minutesOnScreen } = attributes[key] || { - numberOfClicks: 0, - minutesOnScreen: 0, - }; - return { - appId: key, - numberOfClicks: numberOfClicks + perDay.buckets.total.numberOfClicks.value, - minutesOnScreen: minutesOnScreen + perDay.buckets.total.minutesOnScreen.value, - }; - }); - if (newTotals.length === 0) { - return; + const attributes = await this.getSavedObjectTotals(); + const newTotals = (aggregations?.appId.buckets || []).map(({ key, perDay }) => { + const { numberOfClicks, minutesOnScreen } = attributes[key] || { + numberOfClicks: 0, + minutesOnScreen: 0, + }; + return { + appId: key, + numberOfClicks: numberOfClicks + perDay.buckets.total.numberOfClicks.value, + minutesOnScreen: minutesOnScreen + perDay.buckets.total.minutesOnScreen.value, + }; + }); + if (newTotals.length === 0) { + return; + } + await savedObjectsClient.bulkCreate( + newTotals.map(entry => ({ + type: PLUGIN_TYPE, + id: entry.appId, + attributes: entry, + })), + { overwrite: true } + ); + await elasticsearch.callAsInternalUser('deleteByQuery', { + index: INDEX_APP_USAGE, + ignore_unavailable: true, + allow_no_indices: true, + body: { query }, + }); + } catch (err) { + this.log.warn(`Failed to roll totals`, err); } - await savedObjectsClient.bulkCreate( - newTotals.map(entry => ({ - type: PLUGIN_TYPE, - id: entry.appId, - attributes: entry, - })), - { overwrite: true } - ); - await elasticsearch.callAsInternalUser('deleteByQuery', { - index: INDEX_APP_USAGE, - ignore_unavailable: true, - allow_no_indices: true, - body: { query }, - }); } private async fetchAggregation( From ec22dad4ce0971ff67cb1ef26b5d09217336d1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Fri, 21 Feb 2020 10:54:23 +0000 Subject: [PATCH 09/11] Fix boolean check --- src/plugins/application_usage/server/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts index 264e54c636ba6..4f0d71ddce529 100644 --- a/src/plugins/application_usage/server/plugin.ts +++ b/src/plugins/application_usage/server/plugin.ts @@ -179,7 +179,7 @@ export class ApplicationUsagePlugin implements Plugin { private async ensureIndex(elasticsearch: ICustomClusterClient) { // Skip if already done - if (this.indexTemplateInitialised === false) return; + if (this.indexTemplateInitialised === true) return; const mappings = { properties: { From 531267bcb439f4c5403ee9aaf43b17985abf79bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Fri, 21 Feb 2020 12:46:52 +0000 Subject: [PATCH 10/11] Use core's ES.adminCLient over .createClient --- src/plugins/application_usage/server/plugin.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts index 4f0d71ddce529..f781e5eebe180 100644 --- a/src/plugins/application_usage/server/plugin.ts +++ b/src/plugins/application_usage/server/plugin.ts @@ -22,7 +22,7 @@ import { CoreSetup, CoreStart, IRouter, - ICustomClusterClient, + IClusterClient, ISavedObjectsRepository, PluginInitializerContext, Logger, @@ -93,7 +93,7 @@ interface SearchAggregationResult extends SearchResponse { export class ApplicationUsagePlugin implements Plugin { private readonly log: Logger; private intervalId?: NodeJS.Timer; - private esClient?: ICustomClusterClient; + private esClient?: IClusterClient; private savedObjectsClient?: ISavedObjectsRepository; private indexTemplateInitialised = false; @@ -108,7 +108,7 @@ export class ApplicationUsagePlugin implements Plugin { const router = http.createRouter(); this.registerIndexRoute(router); - this.esClient = elasticsearch.createClient('application-usage'); + this.esClient = elasticsearch.adminClient; try { await this.ensureIndex(this.esClient); } catch (err) { @@ -177,7 +177,7 @@ export class ApplicationUsagePlugin implements Plugin { ); } - private async ensureIndex(elasticsearch: ICustomClusterClient) { + private async ensureIndex(elasticsearch: IClusterClient) { // Skip if already done if (this.indexTemplateInitialised === true) return; @@ -238,7 +238,7 @@ export class ApplicationUsagePlugin implements Plugin { } private async rollTotals( - elasticsearch: ICustomClusterClient, + elasticsearch: IClusterClient, savedObjectsClient: ISavedObjectsRepository ) { try { @@ -281,7 +281,7 @@ export class ApplicationUsagePlugin implements Plugin { } private async fetchAggregation( - callCluster: ICustomClusterClient['callAsInternalUser'], + callCluster: IClusterClient['callAsInternalUser'], query: object = { match_all: {} } ): Promise { const lastNDays = (numberOfDays: number) => ({ @@ -333,7 +333,7 @@ export class ApplicationUsagePlugin implements Plugin { return response || {}; } - private async fetchUsage(callCluster: ICustomClusterClient['callAsInternalUser']) { + private async fetchUsage(callCluster: IClusterClient['callAsInternalUser']) { const response = await this.fetchAggregation(callCluster); const totals = await this.getSavedObjectTotals(); From 9fd132456f4a1dcbe9b470bec6a49ded17f3a9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Fri, 21 Feb 2020 12:57:55 +0000 Subject: [PATCH 11/11] Fix tests after ES.adminClient --- src/plugins/application_usage/server/index.test.ts | 7 +------ src/plugins/application_usage/server/plugin.ts | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/plugins/application_usage/server/index.test.ts b/src/plugins/application_usage/server/index.test.ts index 510cee38adc18..529f718747d24 100644 --- a/src/plugins/application_usage/server/index.test.ts +++ b/src/plugins/application_usage/server/index.test.ts @@ -164,12 +164,7 @@ describe('ApplicationUsagePlugin/server', () => { const appPlugin = plugin(coreMock.createPluginInitializerContext()); const coreSetup = coreMock.createSetup(); - coreSetup.elasticsearch.createClient.mockImplementation( - () => - ({ - callAsInternalUser: esClient, - } as any) - ); + (coreSetup.elasticsearch.adminClient.callAsInternalUser as any).mockImplementation(esClient); await appPlugin.setup(coreSetup, { usageCollection: localUsageCollection }); diff --git a/src/plugins/application_usage/server/plugin.ts b/src/plugins/application_usage/server/plugin.ts index f781e5eebe180..f65d5d6ae5127 100644 --- a/src/plugins/application_usage/server/plugin.ts +++ b/src/plugins/application_usage/server/plugin.ts @@ -134,7 +134,7 @@ export class ApplicationUsagePlugin implements Plugin { () => this.esClient && this.rollTotals(this.esClient, savedObjectsClient), ROLL_INDICES_INTERVAL ); - this.rollTotals(this.esClient!, savedObjectsClient); + await this.rollTotals(this.esClient!, savedObjectsClient); } public stop() {