[7.x] [Logs UI] HTTP API for log entries (#53798) (#54024)

* [Logs UI] HTTP API for log entries (#53798)

* Scaffold `log_entries/entries` route

* Scaffold a log entry response

* Add `after` pagination

* Add `before` pagination

* Process `query` parameter

* Use pre-existing structure for the columns

* Change type of date ranges

We will move the responsibility to parse the dates to the client. The
API will only take timestamps

* Add `center` parameter

Allows consumers of the API to get log items around a certain cursor

* Change default page size

* Test the defaults of the API

* Add optional `size` parameter

This makes easier to test the pagination. By default it returns a 200
size page.

* Test the pagination

* Test centering around a point

* Handle `0` sizes

Co-Authored-By: Zacqary Adam Xeper <Zacqary@users.noreply.github.com>

* Add highlights endpoint

* Refactor `processCursor`

* Tweak cursor handling in the routes

* Refine `LogEntry` type

* Add tests for highlights endpoint

* Tweak the types for the LogEntry

Co-authored-by: Zacqary Adam Xeper <Zacqary@users.noreply.github.com>

* Skip failing test (#54100)

ES behaves differently in master and in 7.x, causing the test to fail in
the latter.

Co-authored-by: Zacqary Adam Xeper <Zacqary@users.noreply.github.com>

x-pack/legacy/plugins/infra/common/http_api/log_entries/common.ts

@@ -10,3 +10,4 @@ export const logEntriesCursorRT = rt.type({
   time: rt.number,
   tiebreaker: rt.number,
 });
+export type LogEntriesCursor = rt.TypeOf<typeof logEntriesCursorRT>;

x-pack/legacy/plugins/infra/common/http_api/log_entries/entries.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { logEntriesCursorRT } from './common';
+
+export const LOG_ENTRIES_PATH = '/api/log_entries/entries';
+
+export const logEntriesBaseRequestRT = rt.intersection([
+  rt.type({
+    sourceId: rt.string,
+    startDate: rt.number,
+    endDate: rt.number,
+  }),
+  rt.partial({
+    query: rt.string,
+    size: rt.number,
+  }),
+]);
+
+export const logEntriesBeforeRequestRT = rt.intersection([
+  logEntriesBaseRequestRT,
+  rt.type({ before: rt.union([logEntriesCursorRT, rt.literal('last')]) }),
+]);
+
+export const logEntriesAfterRequestRT = rt.intersection([
+  logEntriesBaseRequestRT,
+  rt.type({ after: rt.union([logEntriesCursorRT, rt.literal('first')]) }),
+]);
+
+export const logEntriesCenteredRT = rt.intersection([
+  logEntriesBaseRequestRT,
+  rt.type({ center: logEntriesCursorRT }),
+]);
+
+export const logEntriesRequestRT = rt.union([
+  logEntriesBaseRequestRT,
+  logEntriesBeforeRequestRT,
+  logEntriesAfterRequestRT,
+  logEntriesCenteredRT,
+]);
+
+export type LogEntriesRequest = rt.TypeOf<typeof logEntriesRequestRT>;
+
+// JSON value
+const valueRT = rt.union([rt.string, rt.number, rt.boolean, rt.object, rt.null, rt.undefined]);
+
+export const logMessagePartRT = rt.union([
+  rt.type({
+    constant: rt.string,
+  }),
+  rt.type({
+    field: rt.string,
+    value: valueRT,
+    highlights: rt.array(rt.string),
+  }),
+]);
+
+export const logColumnRT = rt.union([
+  rt.type({ columnId: rt.string, timestamp: rt.number }),
+  rt.type({
+    columnId: rt.string,
+    field: rt.string,
+    value: rt.union([rt.string, rt.undefined]),
+    highlights: rt.array(rt.string),
+  }),
+  rt.type({
+    columnId: rt.string,
+    message: rt.array(logMessagePartRT),
+  }),
+]);
+
+export const logEntryRT = rt.type({
+  id: rt.string,
+  cursor: logEntriesCursorRT,
+  columns: rt.array(logColumnRT),
+});
+
+export type LogMessagepart = rt.TypeOf<typeof logMessagePartRT>;
+export type LogColumn = rt.TypeOf<typeof logColumnRT>;
+export type LogEntry = rt.TypeOf<typeof logEntryRT>;
+
+export const logEntriesResponseRT = rt.type({
+  data: rt.type({
+    entries: rt.array(logEntryRT),
+    topCursor: logEntriesCursorRT,
+    bottomCursor: logEntriesCursorRT,
+  }),
+});
+
+export type LogEntriesResponse = rt.TypeOf<typeof logEntriesResponseRT>;

x-pack/legacy/plugins/infra/common/http_api/log_entries/highlights.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import {
+  logEntriesBaseRequestRT,
+  logEntriesBeforeRequestRT,
+  logEntriesAfterRequestRT,
+  logEntriesCenteredRT,
+  logEntryRT,
+} from './entries';
+import { logEntriesCursorRT } from './common';
+
+export const LOG_ENTRIES_HIGHLIGHTS_PATH = '/api/log_entries/highlights';
+
+const highlightsRT = rt.type({
+  highlightTerms: rt.array(rt.string),
+});
+
+export const logEntriesHighlightsBaseRequestRT = rt.intersection([
+  logEntriesBaseRequestRT,
+  highlightsRT,
+]);
+
+export const logEntriesHighlightsBeforeRequestRT = rt.intersection([
+  logEntriesBeforeRequestRT,
+  highlightsRT,
+]);
+
+export const logEntriesHighlightsAfterRequestRT = rt.intersection([
+  logEntriesAfterRequestRT,
+  highlightsRT,
+]);
+
+export const logEntriesHighlightsCenteredRequestRT = rt.intersection([
+  logEntriesCenteredRT,
+  highlightsRT,
+]);
+
+export const logEntriesHighlightsRequestRT = rt.union([
+  logEntriesHighlightsBaseRequestRT,
+  logEntriesHighlightsBeforeRequestRT,
+  logEntriesHighlightsAfterRequestRT,
+  logEntriesHighlightsCenteredRequestRT,
+]);
+
+export type LogEntriesHighlightsRequest = rt.TypeOf<typeof logEntriesHighlightsRequestRT>;
+
+export const logEntriesHighlightsResponseRT = rt.type({
+  data: rt.array(
+    rt.type({
+      topCursor: logEntriesCursorRT,
+      bottomCursor: logEntriesCursorRT,
+      entries: rt.array(logEntryRT),
+    })
+  ),
+});
+
+export type LogEntriesHighlightsResponse = rt.TypeOf<typeof logEntriesHighlightsResponseRT>;

x-pack/legacy/plugins/infra/common/http_api/log_entries/index.ts

@@ -4,6 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export * from './common';
+export * from './entries';
+export * from './highlights';
 export * from './item';
 export * from './summary';
 export * from './summary_highlights';

x-pack/legacy/plugins/infra/server/infra_server.ts

@@ -20,6 +20,8 @@ import { initMetadataRoute } from './routes/metadata';
 import { initSnapshotRoute } from './routes/snapshot';
 import { initNodeDetailsRoute } from './routes/node_details';
 import {
+  initLogEntriesRoute,
+  initLogEntriesHighlightsRoute,
   initLogEntriesSummaryRoute,
   initLogEntriesSummaryHighlightsRoute,
   initLogEntriesItemRoute,
@@ -43,6 +45,8 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
   initSnapshotRoute(libs);
   initNodeDetailsRoute(libs);
   initValidateLogAnalysisIndicesRoute(libs);
+  initLogEntriesRoute(libs);
+  initLogEntriesHighlightsRoute(libs);
   initLogEntriesSummaryRoute(libs);
   initLogEntriesSummaryHighlightsRoute(libs);
   initLogEntriesItemRoute(libs);

x-pack/legacy/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts

@@ -8,6 +8,7 @@
 
 import { timeMilliseconds } from 'd3-time';
 import * as runtimeTypes from 'io-ts';
+import { compact } from 'lodash';
 import first from 'lodash/fp/first';
 import get from 'lodash/fp/get';
 import has from 'lodash/fp/has';
@@ -17,12 +18,14 @@ import { map, fold } from 'fp-ts/lib/Either';
 import { identity, constant } from 'fp-ts/lib/function';
 import { RequestHandlerContext } from 'src/core/server';
 import { compareTimeKeys, isTimeKey, TimeKey } from '../../../../common/time';
-import { JsonObject } from '../../../../common/typed_json';
+import { JsonObject, JsonValue } from '../../../../common/typed_json';
 import {
   LogEntriesAdapter,
+  LogEntriesParams,
   LogEntryDocument,
   LogEntryQuery,
   LogSummaryBucket,
+  LOG_ENTRIES_PAGE_SIZE,
 } from '../../domains/log_entries_domain';
 import { InfraSourceConfiguration } from '../../sources';
 import { SortedSearchHit } from '../framework';
@@ -82,6 +85,84 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter {
     return direction === 'asc' ? documents : documents.reverse();
   }
 
+  public async getLogEntries(
+    requestContext: RequestHandlerContext,
+    sourceConfiguration: InfraSourceConfiguration,
+    fields: string[],
+    params: LogEntriesParams
+  ): Promise<LogEntryDocument[]> {
+    const { startDate, endDate, query, cursor, size, highlightTerm } = params;
+
+    const { sortDirection, searchAfterClause } = processCursor(cursor);
+
+    const highlightQuery = createHighlightQuery(highlightTerm, fields);
+
+    const highlightClause = highlightQuery
+      ? {
+          highlight: {
+            boundary_scanner: 'word',
+            fields: fields.reduce(
+              (highlightFieldConfigs, fieldName) => ({
+                ...highlightFieldConfigs,
+                [fieldName]: {},
+              }),
+              {}
+            ),
+            fragment_size: 1,
+            number_of_fragments: 100,
+            post_tags: [''],
+            pre_tags: [''],
+            highlight_query: highlightQuery,
+          },
+        }
+      : {};
+
+    const sort = {
+      [sourceConfiguration.fields.timestamp]: sortDirection,
+      [sourceConfiguration.fields.tiebreaker]: sortDirection,
+    };
+
+    const esQuery = {
+      allowNoIndices: true,
+      index: sourceConfiguration.logAlias,
+      ignoreUnavailable: true,
+      body: {
+        size: typeof size !== 'undefined' ? size : LOG_ENTRIES_PAGE_SIZE,
+        track_total_hits: false,
+        _source: fields,
+        query: {
+          bool: {
+            filter: [
+              ...createFilterClauses(query, highlightQuery),
+              {
+                range: {
+                  [sourceConfiguration.fields.timestamp]: {
+                    gte: startDate,
+                    lte: endDate,
+                    format: TIMESTAMP_FORMAT,
+                  },
+                },
+              },
+            ],
+          },
+        },
+        sort,
+        ...highlightClause,
+        ...searchAfterClause,
+      },
+    };
+
+    const esResult = await this.framework.callWithRequest<SortedSearchHit>(
+      requestContext,
+      'search',
+      esQuery
+    );
+
+    const hits = sortDirection === 'asc' ? esResult.hits.hits : esResult.hits.hits.reverse();
+    return mapHitsToLogEntryDocuments(hits, sourceConfiguration.fields.timestamp, fields);
+  }
+
+  /** @deprecated */
   public async getContainedLogEntryDocuments(
     requestContext: RequestHandlerContext,
     sourceConfiguration: InfraSourceConfiguration,
@@ -319,6 +400,34 @@ function getLookupIntervals(start: number, direction: 'asc' | 'desc'): Array<[nu
   return intervals;
 }
 
+function mapHitsToLogEntryDocuments(
+  hits: SortedSearchHit[],
+  timestampField: string,
+  fields: string[]
+): LogEntryDocument[] {
+  return hits.map(hit => {
+    const logFields = fields.reduce<{ [fieldName: string]: JsonValue }>(
+      (flattenedFields, field) => {
+        if (has(field, hit._source)) {
+          flattenedFields[field] = get(field, hit._source);
+        }
+        return flattenedFields;
+      },
+      {}
+    );
+
+    return {
+      gid: hit._id,
+      // timestamp: hit._source[timestampField],
+      // FIXME s/key/cursor/g
+      key: { time: hit.sort[0], tiebreaker: hit.sort[1] },
+      fields: logFields,
+      highlights: hit.highlight || {},
+    };
+  });
+}
+
+/** @deprecated */
 const convertHitToLogEntryDocument = (fields: string[]) => (
   hit: SortedSearchHit
 ): LogEntryDocument => ({
@@ -352,9 +461,62 @@ const convertDateRangeBucketToSummaryBucket = (
   })),
 });
 
+const createHighlightQuery = (
+  highlightTerm: string | undefined,
+  fields: string[]
+): LogEntryQuery | undefined => {
+  if (highlightTerm) {
+    return {
+      multi_match: {
+        fields,
+        lenient: true,
+        query: highlightTerm,
+        type: 'phrase',
+      },
+    };
+  }
+};
+
+const createFilterClauses = (
+  filterQuery?: LogEntryQuery,
+  highlightQuery?: LogEntryQuery
+): LogEntryQuery[] => {
+  if (filterQuery && highlightQuery) {
+    return [{ bool: { filter: [filterQuery, highlightQuery] } }];
+  }
+
+  return compact([filterQuery, highlightQuery]) as LogEntryQuery[];
+};
+
 const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) =>
   filterQuery ? [filterQuery] : [];
 
+function processCursor(
+  cursor: LogEntriesParams['cursor']
+): {
+  sortDirection: 'asc' | 'desc';
+  searchAfterClause: { search_after?: readonly [number, number] };
+} {
+  if (cursor) {
+    if ('before' in cursor) {
+      return {
+        sortDirection: 'desc',
+        searchAfterClause:
+          cursor.before !== 'last'
+            ? { search_after: [cursor.before.time, cursor.before.tiebreaker] as const }
+            : {},
+      };
+    } else if (cursor.after !== 'first') {
+      return {
+        sortDirection: 'asc',
+        searchAfterClause: { search_after: [cursor.after.time, cursor.after.tiebreaker] as const },
+      };
+    }
+  }
+
+  return { sortDirection: 'asc', searchAfterClause: {} };
+}
+
 const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([
   runtimeTypes.type({
     doc_count: runtimeTypes.number,