Merge branch 'master' into alerting/view-in-app

* master:
  Endpoint: Change the input type for @kbn/config-schema to work with more schemas (elastic#60007)
  Using re2 for Timelion regular expressions (elastic#55208)
  [Monitoring] Re-enable logstash tests (elastic#59815)
  fix karma debug typo (elastic#60029)
  Adds telemetry support to alerting and actions plugins (elastic#58081)

.gitignore

@@ -4,6 +4,7 @@
 /.es
 .DS_Store
 .node_binaries
+.native_modules
 node_modules
 !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules
 !/src/dev/notice/__fixtures__/node_modules

package.json

@@ -227,6 +227,7 @@
     "pug": "^2.0.4",
     "query-string": "6.10.1",
     "raw-loader": "3.1.0",
+    "re2": "1.10.5",
     "react": "^16.12.0",
     "react-color": "^2.13.8",
     "react-dom": "^16.12.0",

src/dev/build/build_distributables.js

@@ -45,6 +45,7 @@ import {
   InstallDependenciesTask,
   BuildKibanaPlatformPluginsTask,
   OptimizeBuildTask,
+  PatchNativeModulesTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
@@ -132,6 +133,7 @@ export async function buildDistributables(options) {
    * directories and perform platform-specific steps
    */
   await run(CreateArchivesSourcesTask);
+  await run(PatchNativeModulesTask);
   await run(CleanExtraBinScriptsTask);
   await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);

src/dev/build/tasks/nodejs/download.js → src/dev/build/lib/download.js

@@ -24,7 +24,7 @@ import chalk from 'chalk';
 import { createHash } from 'crypto';
 import Axios from 'axios';
 
-import { mkdirp } from '../../lib';
+import { mkdirp } from './fs';
 
 function tryUnlink(path) {
   try {

src/dev/build/lib/index.js

@@ -33,5 +33,6 @@ export {
   compress,
   isFileAccessible,
 } from './fs';
+export { download } from './download';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/index.js

@@ -32,6 +32,7 @@ export * from './nodejs_modules';
 export * from './notice_file_task';
 export * from './optimize_task';
 export * from './os_packages';
+export * from './patch_native_modules_task';
 export * from './transpile_babel_task';
 export * from './transpile_scss_task';
 export * from './verify_env_task';

src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js

@@ -22,7 +22,7 @@ import expect from '@kbn/expect';
 
 import * as NodeShasumsNS from '../node_shasums';
 import * as NodeDownloadInfoNS from '../node_download_info';
-import * as DownloadNS from '../download';
+import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
 import { DownloadNodeBuildsTask } from '../download_node_builds_task';
 
 describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

src/dev/build/tasks/nodejs/download_node_builds_task.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { download } from './download';
+import { download } from '../../lib';
 import { getNodeShasums } from './node_shasums';
 import { getNodeDownloadInfo } from './node_download_info';
 

src/dev/build/tasks/patch_native_modules_task.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import path from 'path';
+import util from 'util';
+import { deleteAll, download, untar } from '../lib';
+
+const BASE_URL = `https://storage.googleapis.com/native-modules`;
+const DOWNLOAD_DIRECTORY = '.native_modules';
+
+const packages = [
+  {
+    name: 're2',
+    version: '1.10.5',
+    destinationPath: 'node_modules/re2/build/Release/',
+    shas: {
+      darwin: '066533b592094f91e00412499e44c338ce2466d63c9eaf0dc32be8214bde2099',
+      linux: '0322cac3c2e106129b650a8eac509f598ed283791d6116984fec4c151b24e574',
+      windows: '65b5bef7de2352f4787224c2c76a619b6683a868c8d4d71e0fdd500786fc422b',
+    },
+  },
+];
+
+async function getInstalledVersion(config, packageName) {
+  const packageJSONPath = config.resolveFromRepo(
+    path.join('node_modules', packageName, 'package.json')
+  );
+  const buffer = await util.promisify(fs.readFile)(packageJSONPath);
+  const packageJSON = JSON.parse(buffer);
+  return packageJSON.version;
+}
+
+async function patchModule(config, log, build, platform, pkg) {
+  const installedVersion = await getInstalledVersion(config, pkg.name);
+  if (installedVersion !== pkg.version) {
+    throw new Error(
+      `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
+    );
+  }
+  const platformName = platform.getName();
+  const archiveName = `${pkg.version}-${platformName}.tar.gz`;
+  const downloadUrl = `${BASE_URL}/${pkg.name}/${archiveName}`;
+  const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, archiveName);
+  const extractedPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
+  log.debug(`Patching ${pkg.name} binaries from ${downloadUrl} to ${extractedPath}`);
+
+  await deleteAll([extractedPath], log);
+  await download({
+    log,
+    url: downloadUrl,
+    destination: downloadPath,
+    sha256: pkg.shas[platformName],
+    retries: 3,
+  });
+  await untar(downloadPath, extractedPath);
+}
+
+export const PatchNativeModulesTask = {
+  description: 'Patching platform-specific native modules',
+  async run(config, log, build) {
+    for (const pkg of packages) {
+      await Promise.all(
+        config.getTargetPlatforms().map(async platform => {
+          await patchModule(config, log, build, platform, pkg);
+        })
+      );
+    }
+  },
+};

src/plugins/timelion/server/series_functions/label.js

@@ -51,7 +51,10 @@ export default new Chainable('label', {
     const config = args.byName;
     return alter(args, function(eachSeries) {
       if (config.regex) {
-        eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
+        // not using a standard `import` so that if there's an issue with the re2 native module
+        // that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
+        const RE2 = require('re2');
+        eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
       } else {
         eachSeries.label = config.label;
       }

tasks/test.js

@@ -61,7 +61,7 @@ module.exports = function(grunt) {
     'run:apiIntegrationTests',
   ]);
 
-  grunt.registerTask('test:karmaDebug', ['checkPlugins', 'run:karmaDebugServer', 'karma:dev']);
+  grunt.registerTask('test:karmaDebug', ['checkPlugins', 'run:karmaTestDebugServer', 'karma:dev']);
   grunt.registerTask('test:mochaCoverage', ['run:mochaCoverage']);
 
   grunt.registerTask('test', subTask => {

x-pack/legacy/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.js

@@ -33,7 +33,7 @@ export async function getPaginatedPipelines(
   { clusterUuid, logstashUuid },
   { throughputMetric, nodesCountMetric },
   pagination,
-  sort,
+  sort = { field: null },
   queryText
 ) {
   const sortField = sort.field;

x-pack/plugins/actions/kibana.json

@@ -5,6 +5,6 @@
   "kibanaVersion": "kibana",
   "configPath": ["xpack", "actions"],
   "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "eventLog"],
-  "optionalPlugins": ["spaces"],
+  "optionalPlugins": ["usageCollection", "spaces"],
   "ui": false
 }

x-pack/plugins/actions/server/plugin.test.ts

@@ -11,8 +11,13 @@ import { licensingMock } from '../../licensing/server/mocks';
 import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks';
 import { taskManagerMock } from '../../task_manager/server/mocks';
 import { eventLogMock } from '../../event_log/server/mocks';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 
 describe('Actions Plugin', () => {
+  const usageCollectionMock: jest.Mocked<UsageCollectionSetup> = ({
+    makeUsageCollector: jest.fn(),
+    registerCollector: jest.fn(),
+  } as unknown) as jest.Mocked<UsageCollectionSetup>;
   describe('setup()', () => {
     let context: PluginInitializerContext;
     let plugin: ActionsPlugin;
@@ -23,11 +28,13 @@ describe('Actions Plugin', () => {
       context = coreMock.createPluginInitializerContext();
       plugin = new ActionsPlugin(context);
       coreSetup = coreMock.createSetup();
+
       pluginsSetup = {
         taskManager: taskManagerMock.createSetup(),
         encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(),
         licensing: licensingMock.createSetup(),
         eventLog: eventLogMock.createSetup(),
+        usageCollection: usageCollectionMock,
       };
     });
 
@@ -108,6 +115,7 @@ describe('Actions Plugin', () => {
         encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(),
         licensing: licensingMock.createSetup(),
         eventLog: eventLogMock.createSetup(),
+        usageCollection: usageCollectionMock,
       };
       pluginsStart = {
         taskManager: taskManagerMock.createStart(),

x-pack/plugins/actions/server/plugin.ts

@@ -5,6 +5,7 @@
  */
 
 import { first, map } from 'rxjs/operators';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import {
   PluginInitializerContext,
   Plugin,
@@ -35,6 +36,7 @@ import { ActionTypeRegistry } from './action_type_registry';
 import { ExecuteOptions } from './create_execute_function';
 import { createExecuteFunction } from './create_execute_function';
 import { registerBuiltInActionTypes } from './builtin_action_types';
+import { registerActionsUsageCollector } from './usage';
 
 import { getActionsConfigurationUtilities } from './actions_config';
 
@@ -49,6 +51,7 @@ import {
 } from './routes';
 import { LicenseState } from './lib/license_state';
 import { IEventLogger, IEventLogService } from '../../event_log/server';
+import { initializeActionsTelemetry, scheduleActionsTelemetry } from './usage/task';
 
 const EVENT_LOG_PROVIDER = 'actions';
 export const EVENT_LOG_ACTIONS = {
@@ -71,6 +74,7 @@ export interface ActionsPluginsSetup {
   licensing: LicensingPluginSetup;
   spaces?: SpacesPluginSetup;
   eventLog: IEventLogService;
+  usageCollection?: UsageCollectionSetup;
 }
 export interface ActionsPluginsStart {
   encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
@@ -91,6 +95,7 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
   private spaces?: SpacesServiceSetup;
   private eventLogger?: IEventLogger;
   private isESOUsingEphemeralEncryptionKey?: boolean;
+  private readonly telemetryLogger: Logger;
 
   constructor(initContext: PluginInitializerContext) {
     this.config = initContext.config
@@ -106,6 +111,7 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
       .toPromise();
 
     this.logger = initContext.logger.get('actions');
+    this.telemetryLogger = initContext.logger.get('telemetry');
   }
 
   public async setup(core: CoreSetup, plugins: ActionsPluginsSetup): Promise<PluginSetupContract> {
@@ -140,6 +146,8 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
     const actionExecutor = new ActionExecutor({
       isESOUsingEphemeralEncryptionKey: this.isESOUsingEphemeralEncryptionKey,
     });
+
+    // get executions count
     const taskRunnerFactory = new TaskRunnerFactory(actionExecutor);
     const actionsConfigUtils = getActionsConfigurationUtilities(
       (await this.config) as ActionsConfig
@@ -162,6 +170,20 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
       actionsConfigUtils,
     });
 
+    const usageCollection = plugins.usageCollection;
+    if (usageCollection) {
+      core.getStartServices().then(async ([coreStart, startPlugins]: [CoreStart, any]) => {
+        registerActionsUsageCollector(usageCollection, startPlugins.taskManager);
+
+        initializeActionsTelemetry(
+          this.telemetryLogger,
+          plugins.taskManager,
+          core,
+          await this.kibanaIndex
+        );
+      });
+    }
+
     core.http.registerRouteHandlerContext(
       'actions',
       this.createRouteHandlerContext(await this.kibanaIndex)
@@ -211,6 +233,8 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
       getScopedSavedObjectsClient: core.savedObjects.getScopedClient,
     });
 
+    scheduleActionsTelemetry(this.telemetryLogger, plugins.taskManager);
+
     return {
       execute: createExecuteFunction({
         taskManager: plugins.taskManager,