Merge dynamic known fields, added new cron task

init.js

@@ -18,10 +18,13 @@ import { Monitoring } from './server/monitoring';
 import { WazuhApiRoutes } from './server/routes/wazuh-api';
 import { WazuhReportingRoutes } from './server/routes/wazuh-reporting';
 import { WazuhUtilsRoutes } from './server/routes/wazuh-utils';
+import { IndexPatternCronJob } from './server/index-pattern-cron-job';
 import { log } from './server/logger';
 
 export function initApp(server) {
   const monitoringInstance = new Monitoring(server);
+  const indexPatternCronJobInstance = new IndexPatternCronJob(server);
+
   log('[initApp]', `Waiting for awaitMigration()`, 'info');
   server.kibanaMigrator
     .awaitMigration()
@@ -35,6 +38,7 @@ export function initApp(server) {
       WazuhElasticRouter(server);
       WazuhApiElasticRoutes(server);
       monitoringInstance.run();
+      indexPatternCronJobInstance.run();
       WazuhApiRoutes(server);
       WazuhReportingRoutes(server);
       WazuhUtilsRoutes(server);

public/services/routes.js

@@ -81,26 +81,20 @@ function nestedResolve(
 ) {
   assignPreviousLocation($rootScope, $location);
   const location = $location.path();
-  return getWzConfig($q, genericReq, errorHandler, wazuhConfig)
-    .then(() =>
-      settingsWizard(
-        $location,
-        $q,
-        $window,
-        testAPI,
-        appState,
-        genericReq,
-        errorHandler,
-        wzMisc,
-        wazuhConfig,
-        location && location.includes('/health-check')
-      )
+  return getWzConfig($q, genericReq, errorHandler, wazuhConfig).then(() =>
+    settingsWizard(
+      $location,
+      $q,
+      $window,
+      testAPI,
+      appState,
+      genericReq,
+      errorHandler,
+      wzMisc,
+      wazuhConfig,
+      location && location.includes('/health-check')
     )
-    .then(async () => {
-      try {
-        await this.genericReq.request('GET', '/elastic/known-fields/all', {});
-      } catch (error) {} //eslint-disable-line
-    });
+  );
 }
 
 function savedSearch(

server/controllers/wazuh-elastic.js

@@ -27,6 +27,7 @@ export class WazuhElasticCtrl {
    * @param {*} server
    */
   constructor(server) {
+    this._server = server;
     this.wzWrapper = new ElasticWrapper(server);
   }
 
@@ -588,7 +589,6 @@ export class WazuhElasticCtrl {
   async refreshIndex(req, reply) {
     try {
       if (!req.params.pattern) throw new Error('Missing parameters');
-
       const output =
         ((req || {}).params || {}).pattern === 'all'
           ? await checkKnownFields(this.wzWrapper, false, false, false, true)

server/index-pattern-cron-job.js

@@ -0,0 +1,67 @@
+/*
+ * Wazuh app - Module for refreshing all known fields every 1 minute
+ * Copyright (C) 2015-2019 Wazuh, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Find more information about this on the LICENSE file.
+ */
+import { checkKnownFields } from './lib/refresh-known-fields';
+import cron from 'node-cron';
+import { ElasticWrapper } from './lib/elastic-wrapper';
+import { log } from './logger';
+
+export class IndexPatternCronJob {
+  /**
+   * @param {Object} server Hapi.js server object provided by Kibana
+   */
+  constructor(server) {
+    this.server = server;
+    this.wzWrapper = new ElasticWrapper(server);
+    this.CRON_FREQ = '0 */2 * * * *'; // Every minute
+  }
+
+  /**
+   * Check all known fields for all Wazuh valid index patterns, every "this.CRON_FREQ".
+   * This function is wrapped into a double try/catch block because we
+   * don't want to kill the Node.js process or to stop the app execution.
+   * This is not a reason to stop, a log is just enough to advice the user.
+   */
+  async run() {
+    try {
+      // Launch the Cron job
+      cron.schedule(
+        this.CRON_FREQ,
+        async () => {
+          try {
+            // Call the proper method to refresh the known fields
+            await checkKnownFields(
+              this.wzWrapper,
+              false,
+              this.server,
+              false,
+              true
+            );
+          } catch (error) {
+            // Await execution failed
+            log(
+              '[IndexPatternCronJob][checkKnownFields]',
+              error.message || error
+            );
+          }
+        },
+        true
+      );
+      log(
+        '[IndexPatternCronJob][create-job]',
+        'Index pattern cron job started'
+      );
+    } catch (error) {
+      // Cron job creation failed
+      log('[IndexPatternCronJob][create-job]', error.message || error);
+    }
+  }
+}

server/lib/elastic-wrapper.js

@@ -11,9 +11,10 @@
  */
 import { knownFields } from '../integration-files/known-fields';
 import { monitoringKnownFields } from '../integration-files/monitoring-known-fields';
-
+import querystring from 'querystring';
 export class ElasticWrapper {
   constructor(server) {
+    this._server = server;
     this.usingSearchGuard = ((server || {}).plugins || {}).searchguard || false;
     this.elasticRequest = server.plugins.elasticsearch.getCluster('data');
     this.WZ_KIBANA_INDEX =
@@ -234,7 +235,7 @@ export class ElasticWrapper {
 
   /**
    * Updates index-pattern known fields
-   * @param {*} patternId 'index-pattern:' + id
+   * @param {*} id 'index-pattern:' + id
    */
   async updateIndexPatternKnownFields(id) {
     try {
@@ -243,6 +244,23 @@ export class ElasticWrapper {
           new Error('No valid index pattern id for update index pattern')
         );
 
+      let detectedFields = [];
+      try {
+        // Merge fields logic
+        const patternId = id.includes('index-pattern')
+          ? id.split('index-pattern:')[1]
+          : id;
+        const meta_fields = ['_source', '_id', '_type', '_index', '_score'];
+        const standardRequest = {
+          url: `/api/index_patterns/_fields_for_wildcard?${querystring.stringify(
+            { pattern: patternId, meta_fields }
+          )}`,
+          method: 'GET'
+        };
+        const standardResponse = await this._server.inject(standardRequest);
+        detectedFields = ((standardResponse || {}).result || {}).fields || [];
+      } catch (error) {} //eslint-disable-line
+
       const pattern = await this.getIndexPatternUsingGet(id);
 
       let currentFields = [];
@@ -281,6 +299,20 @@ export class ElasticWrapper {
         currentFields = knownFields;
       }
 
+      // Iterate over dynamic fields
+      for (const field of detectedFields) {
+        // It has this field?
+        const index = currentFields.map(item => item.name).indexOf(field.name);
+
+        if (index >= 0 && currentFields[index]) {
+          // If field already exists, update its type
+          currentFields[index].type = field.type;
+        } else {
+          // If field doesn't exist, add it
+          currentFields.push(field);
+        }
+      }
+
       // This array always must has items
       if (!currentFields || !currentFields.length) {
         return Promise.reject(