Rollup index patterns with live data (#21187)

* Use new rollup capabilities keyed by rollup index name ES api, remove wildcard restriction * Adjust plugins to work with new typeMeta information on rollup index patterns * Disable rollup index patterns from: - Timelion `index` suggestions - ML - Graph * Set Discover to use default search strategy
elastic · Jul 27, 2018 · fdc5de7 · fdc5de7
1 parent 189d35c
commit fdc5de7
Show file tree

Hide file tree

Showing 35 changed files with 719 additions and 303 deletions.
diff --git a/src/core_plugins/kibana/public/discover/controllers/discover.js b/src/core_plugins/kibana/public/discover/controllers/discover.js
@@ -201,6 +201,10 @@ function discoverController(
     .setField('highlightAll', true)
     .setField('version', true);
 
+  // Even when searching rollups, we want to use the default strategy so that we get back a
+  // document-like response.
+  $scope.searchSource.setPreferredSearchStrategyId('default');
+
   // searchSource which applies time range
   const timeRangeSearchSource = savedSearch.searchSource.create();
   timeRangeSearchSource.setField('filter', () => {

diff --git a/...ces/create_index_pattern_wizard/components/step_index_pattern/components/header/header.js b/...ces/create_index_pattern_wizard/components/step_index_pattern/components/header/header.js
@@ -38,7 +38,6 @@ export const Header = ({
   onQueryChanged,
   goToNextStep,
   isNextStepDisabled,
-  allowWildcards,
   ...rest
 }) => (
   <div {...rest}>
@@ -59,18 +58,14 @@ export const Header = ({
             error={errors}
             helpText={
               <div>
-                {
-                  allowWildcards ? (
-                    <p>You can use a <strong>*</strong> as a wildcard in your index pattern.</p>
-                  ) : ''
-                }
+                <p>You can use a <strong>*</strong> as a wildcard in your index pattern.</p>
                 <p>You can&apos;t use spaces or the characters <strong>{characterList}</strong>.</p>
               </div>
             }
           >
             <EuiFieldText
               name="indexPattern"
-              placeholder={allowWildcards ? 'index-name-*' : 'index-name'}
+              placeholder="index-name-*"
               value={query}
               isInvalid={isInputInvalid}
               onChange={onQueryChanged}

diff --git a/...s/indices/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.js b/...s/indices/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.js
@@ -69,11 +69,10 @@ export class StepIndexPattern extends Component {
       query: props.initialQuery,
       appendedWildcard: false,
       showingIndexPatternQueryErrors: false,
-      allowWildcards: indexPatternCreationType.getAllowWildcards(),
       indexPatternName: indexPatternCreationType.getIndexPatternName(),
     };
 
-    this.ILLEGAL_CHARACTERS = indexPatternCreationType.illegalCharacters(ILLEGAL_CHARACTERS);
+    this.ILLEGAL_CHARACTERS = [...ILLEGAL_CHARACTERS];
     this.lastQuery = null;
   }
 
@@ -137,11 +136,11 @@ export class StepIndexPattern extends Component {
   }
 
   onQueryChanged = e => {
-    const { appendedWildcard, allowWildcards } = this.state;
+    const { appendedWildcard } = this.state;
     const { target } = e;
 
     let query = target.value;
-    if (query.length === 1 && allowWildcards && canAppendWildcard(query)) {
+    if (query.length === 1 && canAppendWildcard(query)) {
       query += '*';
       this.setState({ appendedWildcard: true });
       setTimeout(() => target.setSelectionRange(1, 1));
@@ -227,7 +226,7 @@ export class StepIndexPattern extends Component {
 
   renderHeader({ exactMatchedIndices: indices }) {
     const { goToNextStep, indexPatternCreationType } = this.props;
-    const { query, showingIndexPatternQueryErrors, indexPatternExists, indexPatternName, allowWildcards } = this.state;
+    const { query, showingIndexPatternQueryErrors, indexPatternExists, indexPatternName } = this.state;
 
     let containsErrors = false;
     const errors = [];
@@ -259,7 +258,6 @@ export class StepIndexPattern extends Component {
         onQueryChanged={this.onQueryChanged}
         goToNextStep={goToNextStep}
         isNextStepDisabled={isNextStepDisabled}
-        allowWildcards={allowWildcards}
       />
     );
   }

diff --git a/...ections/indices/create_index_pattern_wizard/components/step_time_field/step_time_field.js b/...ections/indices/create_index_pattern_wizard/components/step_time_field/step_time_field.js
@@ -47,6 +47,11 @@ export class StepTimeField extends Component {
   constructor(props) {
     super(props);
 
+    const {
+      getIndexPatternType,
+      getIndexPatternName,
+    } = props.indexPatternCreationType;
+
     this.state = {
       timeFields: [],
       selectedTimeField: undefined,
@@ -55,8 +60,8 @@ export class StepTimeField extends Component {
       isFetchingTimeFields: false,
       isCreating: false,
       indexPatternId: '',
-      indexPatternType: props.indexPatternCreationType.getIndexPatternType(),
-      indexPatternName: props.indexPatternCreationType.getIndexPatternName(),
+      indexPatternType: getIndexPatternType(),
+      indexPatternName: getIndexPatternName(),
     };
   }
 
@@ -66,10 +71,12 @@ export class StepTimeField extends Component {
 
   fetchTimeFields = async () => {
     const { indexPatternsService, indexPattern } = this.props;
-    const { indexPatternType } = this.state;
+    const { getFetchForWildcardOptions } = this.props.indexPatternCreationType;
 
     this.setState({ isFetchingTimeFields: true });
-    const fields = await ensureMinimumTime(indexPatternsService.fieldsFetcher.fetchForWildcard(indexPattern, indexPatternType));
+    const fields = await ensureMinimumTime(
+      indexPatternsService.fieldsFetcher.fetchForWildcard(indexPattern, getFetchForWildcardOptions())
+    );
     const timeFields = extractTimeFields(fields);
 
     this.setState({ timeFields, isFetchingTimeFields: false });

diff --git a/.../kibana/public/management/sections/indices/create_index_pattern_wizard/lib/get_indices.js b/.../kibana/public/management/sections/indices/create_index_pattern_wizard/lib/get_indices.js
@@ -58,7 +58,6 @@ export async function getIndices(es, indexPatternCreationType, rawPattern, limit
           }
         }
       },
-      ...indexPatternCreationType.getIndexPatternCreationQuery(),
     }
   };
 
@@ -72,10 +71,10 @@ export async function getIndices(es, indexPatternCreationType, rawPattern, limit
       response.aggregations.indices.buckets.map(bucket => {
         return bucket.key;
       })
-        .map((index) => {
+        .map((indexName) => {
           return {
-            name: index,
-            tags: indexPatternCreationType.getIndexTags(index)
+            name: indexName,
+            tags: indexPatternCreationType.getIndexTags(indexName)
           };
         })
       , 'name'

diff --git a/...gement/sections/indices/edit_index_pattern/indexed_fields_table/components/table/table.js b/...gement/sections/indices/edit_index_pattern/indexed_fields_table/components/table/table.js
@@ -50,7 +50,7 @@ export class Table extends PureComponent {
               type="questionInCircle"
               color="primary"
               aria-label="Additional field information"
-              content={field.info.map(info => <div>{info}</div>)}
+              content={field.info.map((info, i) => <div key={i}>{info}</div>)}
             />
           </span>
         ) : null}

diff --git a/src/core_plugins/kibana/public/management/sections/indices/index.js b/src/core_plugins/kibana/public/management/sections/indices/index.js
@@ -98,9 +98,6 @@ uiModules.get('apps/management')
           $scope.$evalAsync(() => kbnUrl.change(url));
         });
 
-        $scope.editingId = $route.current.params.indexPatternId;
-        config.bindToScope($scope, 'defaultIndex');
-
         const renderList = () => {
           $scope.indexPatternList = $route.current.locals.indexPatterns.map(pattern => {
             const id = pattern.id;
@@ -133,13 +130,11 @@ uiModules.get('apps/management')
           updateIndexPatternList($scope, indexPatternCreationOptions, $scope.defaultIndex, $scope.indexPatternList);
         };
 
-        $scope.$watch('defaultIndex', renderList);
-
-        renderList();
-
-        $scope.$on('$destroy', () => {
-          destroyIndexPatternList();
-        });
+        $scope.$on('$destroy', () => destroyIndexPatternList());
+        $scope.editingId = $route.current.params.indexPatternId;
+        $scope.$watch('defaultIndex', () => renderList());
+        config.bindToScope($scope, 'defaultIndex');
+        $scope.$apply();
       }
     };
   });

diff --git a/...ugins/timelion/public/directives/timelion_expression_suggestions/arg_value_suggestions.js b/...ugins/timelion/public/directives/timelion_expression_suggestions/arg_value_suggestions.js
@@ -67,14 +67,16 @@ export function ArgValueSuggestionsProvider(Private, indexPatterns) {
         const search = partial ? `${partial}*` : '*';
         const resp = await savedObjectsClient.find({
           type: 'index-pattern',
-          fields: ['title'],
+          fields: ['title', 'type'],
           search: `${search}`,
           search_fields: ['title'],
           perPage: 25
         });
-        return resp.savedObjects.map(savedObject => {
-          return { name: savedObject.attributes.title };
-        });
+        return resp.savedObjects
+          .filter(savedObject => !savedObject.get('type'))
+          .map(savedObject => {
+            return { name: savedObject.attributes.title };
+          });
       },
       metric: async function (partial, functionArgs) {
         if (!partial || !partial.includes(':')) {

diff --git a/src/server/index_patterns/routes/fields_for_wildcard_route.js b/src/server/index_patterns/routes/fields_for_wildcard_route.js
@@ -27,7 +27,8 @@ export const createFieldsForWildcardRoute = pre => ({
     validate: {
       query: Joi.object().keys({
         pattern: Joi.string().required(),
-        meta_fields: Joi.array().items(Joi.string()).default([])
+        meta_fields: Joi.array().items(Joi.string()).default([]),
+        params: Joi.object(),
       }).default()
     },
     handler(req, reply) {

diff --git a/src/ui/public/courier/search_source/search_source.js b/src/ui/public/courier/search_source/search_source.js
@@ -130,6 +130,7 @@ export function SearchSourceProvider(Promise, Private, config) {
     constructor(initialFields) {
       this._id = _.uniqueId('data_source');
 
+      this._searchStrategyId = undefined;
       this._fields = parseInitialFields(initialFields);
       this._parent = undefined;
 
@@ -164,6 +165,14 @@ export function SearchSourceProvider(Promise, Private, config) {
      * PUBLIC API
      *****/
 
+    setPreferredSearchStrategyId(searchStrategyId) {
+      this._searchStrategyId = searchStrategyId;
+    }
+
+    getPreferredSearchStrategyId() {
+      return this._searchStrategyId;
+    }
+
     setFields(newFields) {
       this._fields = newFields;
       return this;

diff --git a/src/ui/public/courier/search_strategy/is_default_type_index_pattern.js b/src/ui/public/courier/search_strategy/is_default_type_index_pattern.js
@@ -19,5 +19,5 @@
 
 export const isDefaultTypeIndexPattern = indexPattern => {
   // Default index patterns don't have `type` defined.
-  return indexPattern.type == null;
+  return !indexPattern.type;
 };
diff --git a/src/ui/public/courier/search_strategy/search_strategy_registry.js b/src/ui/public/courier/search_strategy/search_strategy_registry.js
@@ -27,12 +27,31 @@ export const addSearchStrategy = searchStrategy => {
   searchStrategies.push(searchStrategy);
 };
 
-const getSearchStrategy = indexPattern => {
+const getSearchStrategyByViability = indexPattern => {
   return searchStrategies.find(searchStrategy => {
     return searchStrategy.isViable(indexPattern);
   });
 };
 
+const getSearchStrategyById = searchStrategyId => {
+  return searchStrategies.find(searchStrategy => {
+    return searchStrategy.id === searchStrategyId;
+  });
+};
+
+const getSearchStrategyForSearchRequest = searchRequest => {
+  // Allow the searchSource to declare the correct strategy with which to execute its searches.
+  const preferredSearchStrategyId = searchRequest.source.getPreferredSearchStrategyId();
+  if (preferredSearchStrategyId != null) {
+    return getSearchStrategyById(preferredSearchStrategyId);
+  }
+
+  // Otherwise try to match it to a strategy.
+  const indexPattern = searchRequest.source.getField('index');
+  return getSearchStrategyByViability(indexPattern);
+};
+
+
 /**
  * Build a structure like this:
  *
@@ -52,9 +71,7 @@ export const assignSearchRequestsToSearchStrategies = searchRequests => {
   const searchStrategyById = {};
 
   searchRequests.forEach(searchRequest => {
-    const indexPattern = searchRequest.source.getField('index');
-    const matchingSearchStrategy = getSearchStrategy(indexPattern);
-
+    const matchingSearchStrategy = getSearchStrategyForSearchRequest(searchRequest);
     const { id } = matchingSearchStrategy;
     let searchStrategyWithRequest = searchStrategyById[id];
 
@@ -76,5 +93,5 @@ export const assignSearchRequestsToSearchStrategies = searchRequests => {
 };
 
 export const hasSearchStategyForIndexPattern = indexPattern => {
-  return Boolean(getSearchStrategy(indexPattern));
+  return Boolean(getSearchStrategyByViability(indexPattern));
 };
diff --git a/src/ui/public/index_patterns/fields_fetcher.js b/src/ui/public/index_patterns/fields_fetcher.js
@@ -25,7 +25,10 @@ export function createFieldsFetcher(apiClient, config) {
         return this.fetchForTimePattern(indexPattern.title, interval);
       }
 
-      return this.fetchForWildcard(indexPattern.title, indexPattern.type);
+      return this.fetchForWildcard(indexPattern.title, {
+        type: indexPattern.type,
+        params: indexPattern.typeMeta && indexPattern.typeMeta.params,
+      });
     }
 
     fetchForTimePattern(indexPatternId) {
@@ -36,11 +39,12 @@ export function createFieldsFetcher(apiClient, config) {
       });
     }
 
-    fetchForWildcard(indexPatternId, indexPatternType) {
+    fetchForWildcard(indexPatternId, options = {}) {
       return apiClient.getFieldsForWildcard({
         pattern: indexPatternId,
         metaFields: config.get('metaFields'),
-        type: indexPatternType,
+        type: options.type,
+        params: options.params || {},
       });
     }
   }

diff --git a/src/ui/public/index_patterns/index_patterns_api_client.js b/src/ui/public/index_patterns/index_patterns_api_client.js
@@ -86,26 +86,16 @@ export function createIndexPatternsApiClient($http, basePath) {
         pattern,
         metaFields,
         type,
+        params,
       } = options;
 
-      const url = getUrl(['_fields_for_wildcard'], {
+      const url = getUrl([type, '_fields_for_wildcard'], {
         pattern,
         meta_fields: metaFields,
+        params: JSON.stringify(params),
       });
 
-      return request('GET', url).then(resp => {
-        if(type) {
-          const typeUrl = type ? getUrl([type, '_fields_for_wildcard'], {
-            pattern,
-            fields: resp.fields,
-            meta_fields: metaFields,
-          }) : null;
-
-          return request('GET', typeUrl).then(typeResp => typeResp.fields);
-        } else {
-          return resp.fields;
-        }
-      });
+      return request('GET', url).then(resp => resp.fields);
     }
   }