[8.11] [ES|QL] Force ES client timeout after 2 minutes (#168929) (#16…

…9206)

# Backport

This will backport the following commits from `main` to `8.11`:
- [[ES|QL] Force ES client timeout after 2 minutes
(#168929)](#168929)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Stratoula
Kalafateli","email":"efstratia.kalafateli@elastic.co"},"sourceCommit":{"committedDate":"2023-10-18T08:32:29Z","message":"[ES|QL]
Force ES client timeout after 2 minutes (#168929)\n\n##
Summary\r\n\r\nWe found out that for ES|QL queries while the kibana
server (and\r\nbrowser) timeouts in 2 minutes, the requests are still
running in ES. So\r\ninstead of being aborted in 2 minutes, the hit a
5minute timeout from\r\nthe proxy which retries 3 times and then
aborts.\r\n\r\nThis happens ONLY when bfetch is enabled (which is the
default)\r\n\r\nAfter an investigation with Rudolf it seems that we
don't abort\r\ncorrectly in bsearch. Lukas is fixing it
here\r\nhttps://github.com//pull/169041
.","sha":"6037805fb068c8dc703999f656b110e986279614","branchLabelMapping":{"^v8.12.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:prev-minor","v8.11.0","Feature:ES|QL","v8.12.0"],"number":168929,"url":"https://github.com/elastic/kibana/pull/168929","mergeCommit":{"message":"[ES|QL]
Force ES client timeout after 2 minutes (#168929)\n\n##
Summary\r\n\r\nWe found out that for ES|QL queries while the kibana
server (and\r\nbrowser) timeouts in 2 minutes, the requests are still
running in ES. So\r\ninstead of being aborted in 2 minutes, the hit a
5minute timeout from\r\nthe proxy which retries 3 times and then
aborts.\r\n\r\nThis happens ONLY when bfetch is enabled (which is the
default)\r\n\r\nAfter an investigation with Rudolf it seems that we
don't abort\r\ncorrectly in bsearch. Lukas is fixing it
here\r\nhttps://github.com//pull/169041
.","sha":"6037805fb068c8dc703999f656b110e986279614"}},"sourceBranch":"main","suggestedTargetBranches":["8.11"],"targetPullRequestStates":[{"branch":"8.11","label":"v8.11.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.12.0","labelRegex":"^v8.12.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/168929","number":168929,"mergeCommit":{"message":"[ES|QL]
Force ES client timeout after 2 minutes (#168929)\n\n##
Summary\r\n\r\nWe found out that for ES|QL queries while the kibana
server (and\r\nbrowser) timeouts in 2 minutes, the requests are still
running in ES. So\r\ninstead of being aborted in 2 minutes, the hit a
5minute timeout from\r\nthe proxy which retries 3 times and then
aborts.\r\n\r\nThis happens ONLY when bfetch is enabled (which is the
default)\r\n\r\nAfter an investigation with Rudolf it seems that we
don't abort\r\ncorrectly in bsearch. Lukas is fixing it
here\r\nhttps://github.com//pull/169041
.","sha":"6037805fb068c8dc703999f656b110e986279614"}}]}] BACKPORT-->

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>

src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts

@@ -11,6 +11,8 @@ import type { Logger } from '@kbn/core/server';
 import { getKbnServerError, KbnServerError } from '@kbn/kibana-utils-plugin/server';
 import type { ISearchStrategy } from '../../types';
 
+const ES_TIMEOUT_IN_MS = 120000;
+
 export const esqlSearchStrategyProvider = (
   logger: Logger,
   useInternalUser: boolean = false
@@ -23,6 +25,17 @@ export const esqlSearchStrategyProvider = (
    * @returns `Observable<IEsSearchResponse<any>>`
    */
   search: (request, { abortSignal, ...options }, { esClient, uiSettingsClient }) => {
+    const abortController = new AbortController();
+    // We found out that there are cases where we are not aborting correctly
+    // For this reasons we want to manually cancel he abort signal after 2 mins
+
+    abortSignal?.addEventListener('abort', () => {
+      abortController.abort();
+    });
+
+    // Also abort after two mins
+    setTimeout(() => abortController.abort(), ES_TIMEOUT_IN_MS);
+
     // Only default index pattern type is supported here.
     // See ese for other type support.
     if (request.indexType) {
@@ -41,8 +54,10 @@ export const esqlSearchStrategyProvider = (
             },
           },
           {
-            signal: abortSignal,
+            signal: abortController.signal,
             meta: true,
+            // we don't want the ES client to retry (default value is 3)
+            maxRetries: 0,
           }
         );
         return {