[Infra UI] Only display available fields in Metric Explorer

x-pack/plugins/infra/public/components/metrics_explorer/metrics.tsx

@@ -65,7 +65,9 @@ export const MetricsExplorerMetrics = injectI18n(
       [options, onChange]
     );
 
-    const comboOptions = fields.map(field => ({ label: field.name, value: field.name }));
+    const comboOptions = fields
+      .filter(field => field.aggregatable)
+      .map(field => ({ label: field.name, value: field.name }));
     const selectedOptions = options.metrics
       .filter(m => m.aggregation !== MetricsExplorerAggregation.count)
       .map(metric => ({

x-pack/plugins/infra/public/hooks/use_fields.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useState, useEffect } from 'react';
+import DateMath from '@elastic/datemath';
+import { AvailableFieldsResponse } from '../../server/routes/available_fields/types';
+import { fetch } from '../utils/fetch';
+
+export function useFields(indexPattern: string, timeField: string, from: string, to: string) {
+  const [error, setError] = useState<Error | null>(null);
+  const [loading, setLoading] = useState<boolean>(true);
+  const [data, setData] = useState<AvailableFieldsResponse | null>(null);
+
+  const fromTime = DateMath.parse(from);
+  const toTime = DateMath.parse(to, { roundUp: true });
+
+  if (!toTime || !fromTime) {
+    throw new Error('Unable to parse timerange');
+  }
+
+  useEffect(
+    () => {
+      (async () => {
+        setLoading(true);
+        try {
+          const response = await fetch.post<AvailableFieldsResponse>(
+            '../api/infra/available_fields',
+            { indexPattern, timeField, to: toTime.valueOf(), from: fromTime.valueOf() }
+          );
+          setData(response.data);
+          setError(null);
+        } catch (e) {
+          setError(e);
+          setLoading(false);
+        }
+        setLoading(false);
+      })();
+    },
+    [indexPattern, timeField, from, to]
+  );
+  return { fields: (data && data.fields) || [], loading, error };
+}

x-pack/plugins/infra/public/pages/infrastructure/metrics_explorer/index.tsx

@@ -13,6 +13,7 @@ import { MetricsExplorerToolbar } from '../../../components/metrics_explorer/too
 import { SourceQuery } from '../../../../common/graphql/types';
 import { NoData } from '../../../components/empty_states';
 import { useMetricsExplorerState } from './use_metric_explorer_state';
+import { useFields } from '../../../hooks/use_fields';
 
 interface MetricsExplorerPageProps {
   intl: InjectedIntl;
@@ -41,6 +42,17 @@ export const MetricsExplorerPage = injectI18n(
       handleLoadMore,
     } = useMetricsExplorerState(source, derivedIndexPattern);
 
+    const { fields } = useFields(
+      source.metricAlias,
+      source.fields.timestamp,
+      currentTimerange.from,
+      currentTimerange.to
+    );
+    const alteredDerivedIndexPattern = {
+      ...derivedIndexPattern,
+      fields,
+    };
+
     return (
       <div>
         <DocumentTitle
@@ -57,7 +69,7 @@ export const MetricsExplorerPage = injectI18n(
           }
         />
         <MetricsExplorerToolbar
-          derivedIndexPattern={derivedIndexPattern}
+          derivedIndexPattern={alteredDerivedIndexPattern}
           timeRange={currentTimerange}
           options={options}
           onRefresh={handleRefresh}

x-pack/plugins/infra/server/infra_server.ts

@@ -16,6 +16,7 @@ import { createSourcesResolvers } from './graphql/sources';
 import { InfraBackendLibs } from './lib/infra_types';
 import { initLegacyLoggingRoutes } from './logging_legacy';
 import { initMetricExplorerRoute } from './routes/metrics_explorer';
+import { initAvailableFieldsAPI } from './routes/available_fields';
 
 export const initInfraServer = (libs: InfraBackendLibs) => {
   const schema = makeExecutableSchema({
@@ -35,4 +36,5 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
   initLegacyLoggingRoutes(libs.framework);
   initIpToHostName(libs);
   initMetricExplorerRoute(libs);
+  initAvailableFieldsAPI(libs);
 };

x-pack/plugins/infra/server/routes/available_fields/helpers/extract_fields.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { extractFields } from './extract_fields';
+describe('extractFields()', () => {
+  it('should just work', () => {
+    const doc = {
+      level1: {
+        level2String: 'test',
+        level2Array: ['test'],
+        level2: {
+          level3: {
+            level4Null: null,
+            level4Number: 1,
+          },
+        },
+      },
+    };
+    expect(extractFields(doc, [], ['level1.level2String'])).toEqual([
+      'level1.level2String',
+      'level1.level2Array',
+      'level1.level2.level3.level4Null',
+      'level1.level2.level3.level4Number',
+    ]);
+  });
+});

x-pack/plugins/infra/server/routes/available_fields/helpers/extract_fields.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { isPlainObject } from 'lodash';
+export const extractFields = (obj: any, path: string[] = [], fields: string[] = []): string[] => {
+  if (!isPlainObject(obj)) {
+    const newPath = path.join('.');
+    if (!fields.includes(newPath)) {
+      return [...fields, newPath];
+    }
+    return fields;
+  }
+  return Object.keys(obj).reduce((acc: string[], key: string) => {
+    const value = obj[key];
+    return extractFields(value, path.concat([key]), acc);
+  }, fields);
+};

x-pack/plugins/infra/server/routes/available_fields/helpers/get_sample_field_names.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  AvailableFieldsAggregation,
+  AvailableFieldsBucket,
+  AvailableFieldsHit,
+  AvailableFieldsRequest,
+} from '../types';
+import { extractFields } from './extract_fields';
+import { InfraDatabaseSearchResponse } from '../../../lib/adapters/framework';
+
+export const getSampleFieldNames = async (
+  search: <Aggregation>(options: object) => Promise<InfraDatabaseSearchResponse<{}, Aggregation>>,
+  options: AvailableFieldsRequest
+): Promise<string[]> => {
+  const params = {
+    index: options.indexPattern,
+    body: {
+      query: {
+        range: {
+          [options.timeField]: {
+            gte: options.from,
+            lte: options.to,
+            format: 'epoch_millis',
+          },
+        },
+      },
+      size: 0,
+      aggs: {
+        events: {
+          composite: {
+            size: 100,
+            sources: [{ dataset: { terms: { field: 'event.dataset' } } }],
+          },
+          aggs: {
+            docs: {
+              top_hits: {
+                size: 1,
+                sort: [{ [options.timeField]: { order: 'desc' } }],
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+  const response = await search<AvailableFieldsAggregation>(params);
+  if (!response.aggregations) {
+    throw new Error('Oops! Request is missing aggregations');
+  }
+  return response.aggregations.events.buckets.reduce(
+    (fields: string[], bucket: AvailableFieldsBucket) => {
+      return bucket.docs.hits.hits.reduce((acc: string[], hit: AvailableFieldsHit) => {
+        return extractFields(hit._source, [], acc);
+      }, fields);
+    },
+    []
+  );
+};

x-pack/plugins/infra/server/routes/available_fields/index.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import Joi from 'joi';
+import { boomify } from 'boom';
+import { InfraBackendLibs } from '../../lib/infra_types';
+import { InfraWrappableRequest } from '../../lib/adapters/framework';
+import { AvailableFieldsRequest, AvailableFieldsResponse } from './types';
+import { getSampleFieldNames } from './helpers/get_sample_field_names';
+
+type AvailableFieldsWrappedRequest = InfraWrappableRequest<AvailableFieldsRequest>;
+
+const availableFieldsSchema = Joi.object({
+  timeField: Joi.string().required(),
+  indexPattern: Joi.string().required(),
+  to: Joi.number().required(),
+  from: Joi.number().required(),
+});
+
+export const initAvailableFieldsAPI = ({ framework }: InfraBackendLibs) => {
+  const { callWithRequest } = framework;
+  framework.registerRoute<AvailableFieldsWrappedRequest, Promise<AvailableFieldsResponse>>({
+    method: 'POST',
+    path: '/api/infra/available_fields',
+    options: {
+      validate: { payload: availableFieldsSchema },
+    },
+    handler: async req => {
+      const search = <Aggregation>(searchOptions: object) =>
+        callWithRequest<{}, Aggregation>(req, 'search', searchOptions);
+      try {
+        const indexPatternsService = framework.getIndexPatternsService(req);
+        const fields = await indexPatternsService.getFieldsForWildcard({
+          pattern: req.payload.indexPattern,
+        });
+        const sampleFieldNames = await getSampleFieldNames(search, req.payload);
+        return {
+          fields: fields
+            .filter(f => sampleFieldNames.includes(f.name))
+            .map(f => ({
+              name: f.name,
+              type: f.type,
+              aggregatable: f.aggregatable,
+              searchable: f.searchable,
+            })),
+        };
+      } catch (e) {
+        throw boomify(e);
+      }
+    },
+  });
+};

x-pack/plugins/infra/server/routes/available_fields/types.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface AvailableFieldsRequest {
+  indexPattern: string;
+  timeField: string;
+  to: number;
+  from: number;
+}
+
+export interface AvailableField {
+  name: string;
+  type: string;
+  aggregatable: boolean;
+  searchable: boolean;
+}
+
+export interface AvailableFieldsResponse {
+  fields: AvailableField[];
+}
+
+export interface AvailableFieldsHit {
+  _source: object;
+}
+
+export interface AvailableFieldsBucket {
+  key: { dataset: string };
+  doc_count: number;
+  docs: {
+    hits: {
+      total: { value: number; relation: string };
+      hits: Array<{
+        _source: object;
+      }>;
+    };
+  };
+}
+
+export interface AvailableFieldsAggregation {
+  events: {
+    after_key: {
+      events: string;
+    };
+    buckets: AvailableFieldsBucket[];
+  };
+}