[ml] migrate file_data_visualizer/import route to file_upload plugin (#…

…89640)

* migrate file_upload plugin to maps_file_upload

* update plugins list

* migrate ml import endpoint

* migrate ml telemetry to file_upload plugin

* add fileUpload plugin to ml

* add TS project

* update ML to use file_upload endpoint

* move types to file_upload plugin

* ignore error

* clean up

* i18n clean-up

* remove schemas from ml

* remove usageCollection from ml

* node scripts/build_plugin_list_docs

* update telemety collector

* revert changes to ingestPipeline schema

* change name of TELEMETRY_DOC_ID to unique value

* remove ImportFile from ml/server/routes/apidoc.json

* fix typo in x=pack/tsconfig.json

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/developer/plugin-list.asciidoc

@@ -370,6 +370,10 @@ and actions.
 |The features plugin enhance Kibana with a per-feature privilege system.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload]
+|WARNING: Missing README.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/fleet/README.md[fleet]
 |Fleet needs to have Elasticsearch API keys enabled, and also to have TLS enabled on kibana, (if you want to run Kibana without TLS you can provide the following config flag --xpack.fleet.agents.tlsCheckDisabled=false)
 

src/plugins/telemetry/schema/oss_plugins.json

@@ -4097,6 +4097,9 @@
         "xpackDashboardMode:roles": {
           "type": "keyword"
         },
+        "securitySolution:ipReputationLinks": {
+          "type": "keyword"
+        },
         "visualize:enableLabs": {
           "type": "boolean"
         },
@@ -4115,9 +4118,6 @@
         "visualization:tileMap:maxPrecision": {
           "type": "long"
         },
-        "securitySolution:ipReputationLinks": {
-          "type": "keyword"
-        },
         "csv:separator": {
           "type": "keyword"
         },

x-pack/plugins/file_upload/common/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './constants';
+export * from './types';

x-pack/plugins/file_upload/common/types.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface ImportResponse {
+  success: boolean;
+  id: string;
+  index?: string;
+  pipelineId?: string;
+  docCount: number;
+  failures: ImportFailure[];
+  error?: any;
+  ingestError?: boolean;
+}
+
+export interface ImportFailure {
+  item: number;
+  reason: string;
+  doc: ImportDoc;
+}
+
+export interface Doc {
+  message: string;
+}
+
+export type ImportDoc = Doc | string;
+
+export interface Settings {
+  pipeline?: string;
+  index: string;
+  body: any[];
+  [key: string]: any;
+}
+
+export interface Mappings {
+  _meta?: {
+    created_by: string;
+  };
+  properties: {
+    [key: string]: any;
+  };
+}
+
+export interface IngestPipelineWrapper {
+  id: string;
+  pipeline: IngestPipeline;
+}
+
+export interface IngestPipeline {
+  description: string;
+  processors: any[];
+}

x-pack/plugins/file_upload/jest.config.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/file_upload'],
+};

x-pack/plugins/file_upload/kibana.json

@@ -0,0 +1,8 @@
+{
+  "id": "fileUpload",
+  "version": "8.0.0",
+  "kibanaVersion": "kibana",
+  "server": true,
+  "ui": false,
+  "requiredPlugins": ["usageCollection"]
+}

x-pack/plugins/file_upload/server/error_wrapper.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { boomify, isBoom } from '@hapi/boom';
+import { ResponseError, CustomHttpResponseOptions } from 'kibana/server';
+
+export function wrapError(error: any): CustomHttpResponseOptions<ResponseError> {
+  const boom = isBoom(error)
+    ? error
+    : boomify(error, { statusCode: error.status ?? error.statusCode });
+  const statusCode = boom.output.statusCode;
+  return {
+    body: {
+      message: boom,
+      ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}),
+    },
+    headers: boom.output.headers as { [key: string]: string },
+    statusCode,
+  };
+}

x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts → x-pack/plugins/file_upload/server/import_data.ts

@@ -5,19 +5,20 @@
  */
 
 import { IScopedClusterClient } from 'kibana/server';
-import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer';
+import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
 import {
   ImportResponse,
   ImportFailure,
   Settings,
   Mappings,
   IngestPipelineWrapper,
-} from '../../../common/types/file_datavisualizer';
-import { InputData } from './file_data_visualizer';
+} from '../common';
+
+export type InputData = any[];
 
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
-    id: string,
+    id: string | undefined,
     index: string,
     settings: Settings,
     mappings: Mappings,
@@ -77,7 +78,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
     } catch (error) {
       return {
         success: false,
-        id,
+        id: id!,
         index: createdIndex,
         pipelineId: createdPipelineId,
         error: error.body !== undefined ? error.body : error,

x-pack/plugins/file_upload/server/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FileUploadPlugin } from './plugin';
+
+export const plugin = () => new FileUploadPlugin();

x-pack/plugins/file_upload/server/plugin.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CoreSetup, CoreStart, Plugin } from 'src/core/server';
+import { fileUploadRoutes } from './routes';
+import { initFileUploadTelemetry } from './telemetry';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
+
+interface SetupDeps {
+  usageCollection: UsageCollectionSetup;
+}
+
+export class FileUploadPlugin implements Plugin {
+  async setup(coreSetup: CoreSetup, plugins: SetupDeps) {
+    fileUploadRoutes(coreSetup.http.createRouter());
+
+    initFileUploadTelemetry(coreSetup, plugins.usageCollection);
+  }
+
+  start(core: CoreStart) {}
+}

x-pack/plugins/file_upload/server/routes.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter, IScopedClusterClient } from 'kibana/server';
+import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common';
+import { wrapError } from './error_wrapper';
+import { InputData, importDataProvider } from './import_data';
+
+import { updateTelemetry } from './telemetry';
+import { importFileBodySchema, importFileQuerySchema } from './schemas';
+
+function importData(
+  client: IScopedClusterClient,
+  id: string | undefined,
+  index: string,
+  settings: Settings,
+  mappings: Mappings,
+  ingestPipeline: IngestPipelineWrapper,
+  data: InputData
+) {
+  const { importData: importDataFunc } = importDataProvider(client);
+  return importDataFunc(id, index, settings, mappings, ingestPipeline, data);
+}
+
+/**
+ * Routes for the file upload.
+ */
+export function fileUploadRoutes(router: IRouter) {
+  /**
+   * @apiGroup FileDataVisualizer
+   *
+   * @api {post} /api/file_upload/import Import file data
+   * @apiName ImportFile
+   * @apiDescription Imports file data into elasticsearch index.
+   *
+   * @apiSchema (query) importFileQuerySchema
+   * @apiSchema (body) importFileBodySchema
+   */
+  router.post(
+    {
+      path: '/api/file_upload/import',
+      validate: {
+        query: importFileQuerySchema,
+        body: importFileBodySchema,
+      },
+      options: {
+        body: {
+          accepts: ['application/json'],
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+        tags: ['access:ml:canFindFileStructure'],
+      },
+    },
+    async (context, request, response) => {
+      try {
+        const { id } = request.query;
+        const { index, data, settings, mappings, ingestPipeline } = request.body;
+
+        // `id` being `undefined` tells us that this is a new import due to create a new index.
+        // follow-up import calls to just add additional data will include the `id` of the created
+        // index, we'll ignore those and don't increment the counter.
+        if (id === undefined) {
+          await updateTelemetry();
+        }
+
+        const result = await importData(
+          context.core.elasticsearch.client,
+          id,
+          index,
+          settings,
+          mappings,
+          // @ts-expect-error
+          ingestPipeline,
+          data
+        );
+        return response.ok({ body: result });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    }
+  );
+}

x-pack/plugins/file_upload/server/schemas.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const importFileQuerySchema = schema.object({
+  id: schema.maybe(schema.string()),
+});
+
+export const importFileBodySchema = schema.object({
+  index: schema.string(),
+  data: schema.arrayOf(schema.any()),
+  settings: schema.maybe(schema.any()),
+  /** Mappings */
+  mappings: schema.any(),
+  /** Ingest pipeline definition */
+  ingestPipeline: schema.object({
+    id: schema.maybe(schema.string()),
+    pipeline: schema.maybe(schema.any()),
+  }),
+});

x-pack/plugins/ml/server/lib/telemetry/index.ts → x-pack/plugins/file_upload/server/telemetry/index.ts

@@ -4,5 +4,5 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { initMlTelemetry } from './ml_usage_collector';
+export { initFileUploadTelemetry } from './usage_collector';
 export { updateTelemetry } from './telemetry';

x-pack/plugins/ml/server/lib/telemetry/mappings.ts → x-pack/plugins/file_upload/server/telemetry/mappings.ts

@@ -7,13 +7,13 @@
 import { SavedObjectsType } from 'src/core/server';
 import { TELEMETRY_DOC_ID } from './telemetry';
 
-export const mlTelemetryMappingsType: SavedObjectsType = {
+export const telemetryMappingsType: SavedObjectsType = {
   name: TELEMETRY_DOC_ID,
   hidden: false,
   namespaceType: 'agnostic',
   mappings: {
     properties: {
-      file_data_visualizer: {
+      file_upload: {
         properties: {
           index_creation_count: {
             type: 'long',

x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts → x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts

@@ -34,7 +34,7 @@ describe('ml plugin telemetry', () => {
     it('should update existing telemetry', async () => {
       const internalRepo = mockInit({
         attributes: {
-          file_data_visualizer: {
+          file_upload: {
             index_creation_count: 2,
           },
         },

x-pack/plugins/ml/server/lib/telemetry/telemetry.ts → x-pack/plugins/file_upload/server/telemetry/telemetry.ts

@@ -9,10 +9,10 @@ import { ISavedObjectsRepository } from 'kibana/server';
 
 import { getInternalRepository } from './internal_repository';
 
-export const TELEMETRY_DOC_ID = 'ml-telemetry';
+export const TELEMETRY_DOC_ID = 'file-upload-usage-collection-telemetry';
 
 export interface Telemetry {
-  file_data_visualizer: {
+  file_upload: {
     index_creation_count: number;
   };
 }
@@ -23,7 +23,7 @@ export interface TelemetrySavedObject {
 
 export function initTelemetry(): Telemetry {
   return {
-    file_data_visualizer: {
+    file_upload: {
       index_creation_count: 0,
     },
   };
@@ -74,8 +74,8 @@ export async function updateTelemetry(internalRepo?: ISavedObjectsRepository) {
 
 function incrementCounts(telemetry: Telemetry) {
   return {
-    file_data_visualizer: {
-      index_creation_count: telemetry.file_data_visualizer.index_creation_count + 1,
+    file_upload: {
+      index_creation_count: telemetry.file_upload.index_creation_count + 1,
     },
   };
 }