Merge branch '7.x' into backport/7.x/pr-48268

Author: elasticmachine

packages/kbn-es/src/custom_snapshots.js

@@ -26,8 +26,8 @@ function isVersionFlag(a) {
 function getCustomSnapshotUrl() {
   // force use of manually created snapshots until live ones are available
   if (!process.env.KBN_ES_SNAPSHOT_URL && !process.argv.some(isVersionFlag)) {
-    // return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}';
-    return undefined;
+    return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}';
+    // return undefined;
   }
 
   if (process.env.KBN_ES_SNAPSHOT_URL && process.env.KBN_ES_SNAPSHOT_URL !== 'false') {

src/dev/ci_setup/checkout_sibling_es.sh

@@ -88,10 +88,7 @@ function checkout_sibling {
 }
 
 checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES"
-
-# export TEST_ES_FROM=${TEST_ES_FROM:-snapshot}
-# temporarily build from source until 7.6.0 snapshot is available.
-export TEST_ES_FROM=source
+export TEST_ES_FROM=${TEST_ES_FROM:-snapshot}
 
 # Set the JAVA_HOME based on the Java property file in the ES repo
 # This assumes the naming convention used on CI (ex: ~/.java/java10)

x-pack/legacy/plugins/code/common/constants.ts

@@ -6,3 +6,4 @@
 
 export const APP_TITLE = 'Code (Beta)';
 export const APP_USAGE_TYPE = 'code';
+export const SAVED_OBJ_REPO = 'code-repo';

x-pack/legacy/plugins/code/index.ts

@@ -10,9 +10,10 @@ import { Legacy } from 'kibana';
 import { resolve } from 'path';
 import { CoreSetup } from 'src/core/server';
 
-import { APP_TITLE } from './common/constants';
+import { APP_TITLE, SAVED_OBJ_REPO } from './common/constants';
 import { codePlugin } from './server';
 import { PluginSetupContract } from '../../../plugins/code/server';
+import { mappings } from './mappings';
 
 export type RequestFacade = Legacy.Request;
 export type RequestQueryFacade = RequestQuery;
@@ -43,6 +44,12 @@ export const code = (kibana: any) =>
         };
       },
       hacks: ['plugins/code/hacks/toggle_app_link_in_nav'],
+      savedObjectSchemas: {
+        [SAVED_OBJ_REPO]: {
+          isNamespaceAgnostic: false,
+        },
+      },
+      mappings,
     },
     config(Joi: typeof JoiNamespace) {
       return Joi.object({

x-pack/legacy/plugins/code/mappings.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SAVED_OBJ_REPO } from './common/constants';
+
+export const mappings = {
+  [SAVED_OBJ_REPO]: {
+    properties: {
+      uri: {
+        type: 'keyword',
+      },
+    },
+  },
+};

x-pack/legacy/plugins/code/model/search.ts

@@ -231,3 +231,64 @@ export interface SearchOptions {
   defaultRepoScopeOn: boolean;
   defaultRepoScope?: Repository;
 }
+
+export function emptySearchResult(): SearchResult {
+  return {
+    total: 0,
+    took: 0,
+  };
+}
+
+export function emptyRepositorySearchResult(): RepositorySearchResult {
+  return {
+    ...emptySearchResult(),
+    repositories: [],
+    from: 0,
+    page: 0,
+    totalPage: 0,
+  };
+}
+
+export function emptySymbolSearchResult(): SymbolSearchResult {
+  return {
+    ...emptySearchResult(),
+    symbols: [],
+  };
+}
+
+export function emptyDocumentSearchResult(query: string): DocumentSearchResult {
+  return {
+    ...emptySearchResult(),
+    query,
+    from: 0,
+    page: 0,
+    totalPage: 0,
+    stats: {
+      total: 0,
+      from: 0,
+      to: 0,
+      page: 0,
+      totalPage: 0,
+      repoStats: [],
+      languageStats: [],
+    },
+    results: [],
+    repoAggregations: [],
+    langAggregations: [],
+  };
+}
+
+export function emptyCommitSearchResult(query: string): CommitSearchResult {
+  return {
+    ...emptyDocumentSearchResult(query),
+    commits: [],
+  };
+}
+
+export function emptyIntegrationsSearchResult(): IntegrationsSearchResult {
+  return {
+    ...emptySearchResult(),
+    results: [],
+    fallback: false,
+  };
+}

x-pack/legacy/plugins/code/server/routes/file.ts

@@ -14,6 +14,7 @@ import { EsClientWithRequest } from '../utils/esclient_with_request';
 import { decodeRevisionString } from '../../common/uri_util';
 import { CodeServices } from '../distributed/code_services';
 import { GitServiceDefinition } from '../distributed/apis';
+import { getReferenceHelper } from '../utils/repository_reference_helper';
 
 export function fileRoute(router: CodeServerRouter, codeServices: CodeServices) {
   const gitService = codeServices.serviceFor(GitServiceDefinition);
@@ -26,6 +27,7 @@ export function fileRoute(router: CodeServerRouter, codeServices: CodeServices)
 
     try {
       const repo = await repoObjectClient.getRepository(repoUri);
+      await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repo.uri);
       return repo.uri;
     } catch (e) {
       return undefined;

x-pack/legacy/plugins/code/server/routes/lsp.ts

@@ -26,6 +26,8 @@ import { RequestFacade, ResponseToolkitFacade } from '../..';
 import { CodeServices } from '../distributed/code_services';
 import { GitServiceDefinition, LspServiceDefinition } from '../distributed/apis';
 import { findTitleFromHover, groupFiles } from '../utils/lsp_utils';
+import { getReferenceHelper } from '../utils/repository_reference_helper';
+import { SymbolSearchResult } from '../../model';
 
 const LANG_SERVER_ERROR = 'language server error';
 
@@ -48,6 +50,7 @@ export function lspRoute(
             const params = (req.payload as unknown) as any;
             const uri = params.textDocument.uri;
             const { repoUri } = parseLspUrl(uri)!;
+            await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
             const endpoint = await codeServices.locate(req, repoUri);
             const requestPromise = lspService.sendRequest(endpoint, {
               method: `textDocument/${method}`,
@@ -95,7 +98,9 @@ export function lspRoute(
       // @ts-ignore
       const { textDocument, position } = req.payload;
       const { uri } = textDocument;
-      const endpoint = await codeServices.locate(req, parseLspUrl(uri).repoUri);
+      const { repoUri } = parseLspUrl(uri);
+      await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
+      const endpoint = await codeServices.locate(req, repoUri);
       const response: ResponseMessage = await promiseTimeout(
         serverOptions.lsp.requestTimeoutMs,
         lspService.sendRequest(endpoint, {
@@ -115,11 +120,12 @@ export function lspRoute(
 
       const locators = response.result as SymbolLocator[];
       const locations = [];
+      const repoScope = await getReferenceHelper(req.getSavedObjectsClient()).findReferences();
       for (const locator of locators) {
         if (locator.location) {
           locations.push(locator.location);
-        } else if (locator.qname) {
-          const searchResults = await symbolSearchClient.findByQname(req.params.qname);
+        } else if (locator.qname && repoScope.length > 0) {
+          const searchResults = await symbolSearchClient.findByQname(req.params.qname, repoScope);
           for (const symbol of searchResults.symbols) {
             locations.push(symbol.symbolInformation.location);
           }
@@ -141,7 +147,9 @@ export function lspRoute(
         // @ts-ignore
         const { textDocument, position } = req.payload;
         const { uri } = textDocument;
-        const endpoint = await codeServices.locate(req, parseLspUrl(uri).repoUri);
+        const { repoUri } = parseLspUrl(uri);
+        await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
+        const endpoint = await codeServices.locate(req, repoUri);
         const response: ResponseMessage = await promiseTimeout(
           serverOptions.lsp.requestTimeoutMs,
           lspService.sendRequest(endpoint, {
@@ -189,7 +197,15 @@ export function symbolByQnameRoute(router: CodeServerRouter, log: Logger) {
     async handler(req: RequestFacade) {
       try {
         const symbolSearchClient = new SymbolSearchClient(new EsClientWithRequest(req), log);
-        return await symbolSearchClient.findByQname(req.params.qname);
+        const repoScope = await getReferenceHelper(req.getSavedObjectsClient()).findReferences();
+        if (repoScope.length === 0) {
+          return {
+            symbols: [],
+            total: 0,
+            took: 0,
+          } as SymbolSearchResult;
+        }
+        return await symbolSearchClient.findByQname(req.params.qname, repoScope);
       } catch (error) {
         return Boom.internal(`Search Exception`);
       }

x-pack/legacy/plugins/code/server/routes/repository.ts

@@ -6,6 +6,7 @@
 
 import Boom from 'boom';
 
+import { i18n } from '@kbn/i18n';
 import { RequestFacade, ResponseToolkitFacade } from '../..';
 import { validateGitUrl } from '../../common/git_url_utils';
 import { RepositoryUtils } from '../../common/repository_utils';
@@ -19,6 +20,7 @@ import { EsClientWithRequest } from '../utils/esclient_with_request';
 import { CodeServerRouter } from '../security';
 import { CodeServices } from '../distributed/code_services';
 import { RepositoryServiceDefinition } from '../distributed/apis';
+import { getReferenceHelper } from '../utils/repository_reference_helper';
 
 export function repositoryRoute(
   router: CodeServerRouter,
@@ -56,12 +58,32 @@ export function repositoryRoute(
       try {
         // Check if the repository already exists
         await repoObjectClient.getRepository(repo.uri);
+        // distinguish between that the repository exists in the current space and that the repository exists in
+        // another space, and return the default message if error happens during reference checking.
+        try {
+          const hasRef = await getReferenceHelper(req.getSavedObjectsClient()).hasReference(
+            repo.uri
+          );
+          if (!hasRef) {
+            return Boom.conflict(
+              i18n.translate('xpack.code.repositoryManagement.repoOtherSpaceImportedMessage', {
+                defaultMessage: 'The repository has already been imported in another space!',
+              })
+            );
+          }
+        } catch (e) {
+          log.error(`Failed to check reference for ${repo.uri} in current space`);
+        }
         const msg = `Repository ${repoUrl} already exists. Skip clone.`;
         log.info(msg);
         return h.response(msg).code(304); // Not Modified
       } catch (error) {
         log.info(`Repository ${repoUrl} does not exist. Go ahead with clone.`);
         try {
+          // create the reference first, and make the creation idempotent, to avoid potential dangling repositories
+          // which have no references from any space, in case the writes to ES may fail independently
+          await getReferenceHelper(req.getSavedObjectsClient()).createReference(repo.uri);
+
           // Create the index for the repository
           const initializer = (await repoIndexInitializerFactory.create(
             repo.uri,
@@ -106,6 +128,9 @@ export function repositoryRoute(
       const repoUri: string = req.params.uri as string;
       const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
       try {
+        // make sure the repo belongs to the current space
+        getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
+
         // Check if the repository already exists. If not, an error will be thrown.
         await repoObjectClient.getRepository(repoUri);
 
@@ -129,6 +154,9 @@ export function repositoryRoute(
         };
         const endpoint = await codeServices.locate(req, repoUri);
         await repositoryService.delete(endpoint, payload);
+
+        // delete the reference last to avoid dangling repositories
+        await getReferenceHelper(req.getSavedObjectsClient()).deleteReference(repoUri);
         return {};
       } catch (error) {
         const msg = `Issue repository delete request for ${repoUri} error`;
@@ -146,6 +174,7 @@ export function repositoryRoute(
     async handler(req: RequestFacade) {
       const repoUri = req.params.uri as string;
       try {
+        await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
         const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
         return await repoObjectClient.getRepository(repoUri);
       } catch (error) {
@@ -164,25 +193,30 @@ export function repositoryRoute(
       const repoUri = req.params.uri as string;
       try {
         const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
-        let gitStatus = null;
-        try {
-          gitStatus = await repoObjectClient.getRepositoryGitStatus(repoUri);
-        } catch (error) {
-          log.debug(`Get repository git status ${repoUri} error: ${error}`);
-        }
 
+        let gitStatus = null;
         let indexStatus = null;
-        try {
-          indexStatus = await repoObjectClient.getRepositoryIndexStatus(repoUri);
-        } catch (error) {
-          log.debug(`Get repository index status ${repoUri} error: ${error}`);
-        }
-
         let deleteStatus = null;
-        try {
-          deleteStatus = await repoObjectClient.getRepositoryDeleteStatus(repoUri);
-        } catch (error) {
-          log.debug(`Get repository delete status ${repoUri} error: ${error}`);
+        const hasRef = await getReferenceHelper(req.getSavedObjectsClient()).hasReference(repoUri);
+
+        if (hasRef) {
+          try {
+            gitStatus = await repoObjectClient.getRepositoryGitStatus(repoUri);
+          } catch (error) {
+            log.debug(`Get repository git status ${repoUri} error: ${error}`);
+          }
+
+          try {
+            indexStatus = await repoObjectClient.getRepositoryIndexStatus(repoUri);
+          } catch (error) {
+            log.debug(`Get repository index status ${repoUri} error: ${error}`);
+          }
+
+          try {
+            deleteStatus = await repoObjectClient.getRepositoryDeleteStatus(repoUri);
+          } catch (error) {
+            log.debug(`Get repository delete status ${repoUri} error: ${error}`);
+          }
         }
         return {
           gitStatus,
@@ -204,8 +238,9 @@ export function repositoryRoute(
     method: 'GET',
     async handler(req: RequestFacade) {
       try {
+        const uris = await getReferenceHelper(req.getSavedObjectsClient()).findReferences();
         const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
-        return await repoObjectClient.getAllRepositories();
+        return await repoObjectClient.getRepositories(uris);
       } catch (error) {
         const msg = `Get all repositories error`;
         log.error(msg);
@@ -226,6 +261,7 @@ export function repositoryRoute(
       const repoUri = req.params.uri as string;
       const reindex: boolean = (req.payload as any).reindex;
       try {
+        await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
         const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
         const cloneStatus = await repoObjectClient.getRepositoryGitStatus(repoUri);
 
@@ -258,6 +294,7 @@ export function repositoryRoute(
 
       try {
         // Check if the repository exists
+        await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
         await repoObjectClient.getRepository(repoUri);
       } catch (error) {
         return Boom.badRequest(`Repository not existed for ${repoUri}`);
@@ -284,6 +321,7 @@ export function repositoryRoute(
     async handler(req: RequestFacade) {
       const repoUri = req.params.uri as string;
       try {
+        await getReferenceHelper(req.getSavedObjectsClient()).ensureReference(repoUri);
         const repoObjectClient = new RepositoryObjectClient(new EsClientWithRequest(req));
         return await repoObjectClient.getRepositoryConfig(repoUri);
       } catch (error) {