[Fleet][EPM] Revert es-storage-related commits (#85942)

* Revert "[Fleet][EPM] Save installed package assets in ES (#83391)" This reverts commit 81a340e. * Revert 00c2e96 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Dec 15, 2020 · 7a39887 · 7a39887
1 parent 602410e
commit 7a39887
Show file tree

Hide file tree

Showing 14 changed files with 22 additions and 290 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts
@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
-export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -8,7 +8,6 @@
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
 import {
-  ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
   defaultPackages,
@@ -269,7 +268,6 @@ export type PackageInfo =
 export interface Installation extends SavedObjectAttributes {
   installed_kibana: KibanaAssetReference[];
   installed_es: EsAssetReference[];
-  package_assets: PackageAssetReference[];
   es_index_patterns: Record<string, string>;
   name: string;
   version: string;
@@ -299,10 +297,6 @@ export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: ElasticsearchAssetType;
 };
 
-export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
-  type: typeof ASSETS_SAVED_OBJECT_TYPE;
-};
-
 export type RequiredPackage = typeof requiredPackages;
 
 export type DefaultPackages = typeof defaultPackages;

diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts
@@ -41,7 +41,6 @@ export {
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   OUTPUT_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
-  ASSETS_SAVED_OBJECT_TYPE,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,

diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
@@ -41,13 +41,12 @@ import {
   removeInstallation,
   getLimitedPackages,
   getInstallationObject,
-  getInstallation,
 } from '../../services/epm/packages';
 import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors';
 import { splitPkgKey } from '../../services/epm/registry';
 import { licenseService } from '../../services';
 import { getArchiveEntry } from '../../services/epm/archive/cache';
-import { getAsset } from '../../services/epm/archive/storage';
+import { bufferToStream } from '../../services/epm/streams';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -108,51 +107,32 @@ export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.p
   try {
     const { pkgName, pkgVersion, filePath } = request.params;
     const savedObjectsClient = context.core.savedObjects.client;
-    const installation = await getInstallation({ savedObjectsClient, pkgName });
-    const useLocalFile = pkgVersion === installation?.version;
-
-    if (useLocalFile) {
-      const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
-      const fileBuffer = getArchiveEntry(assetPath);
-      // only pull local installation if we don't have it cached
-      const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
-
-      // error, if neither is available
-      if (!fileBuffer && !storedAsset) {
+    const savedObject = await getInstallationObject({ savedObjectsClient, pkgName });
+    const pkgInstallSource = savedObject?.attributes.install_source;
+    // TODO: when package storage is available, remove installSource check and check cache and storage, remove registry call
+    if (pkgInstallSource === 'upload' && pkgVersion === savedObject?.attributes.version) {
+      const headerContentType = mime.contentType(path.extname(filePath));
+      if (!headerContentType) {
         return response.custom({
-          body: `installed package file not found: ${filePath}`,
-          statusCode: 404,
+          body: `unknown content type for file: ${filePath}`,
+          statusCode: 400,
         });
       }
-
-      // if storedAsset is not available, fileBuffer *must* be
-      // b/c we error if we don't have at least one, and storedAsset is the least likely
-      const { buffer, contentType } = storedAsset
-        ? {
-            contentType: storedAsset.media_type,
-            buffer: storedAsset.data_utf8
-              ? Buffer.from(storedAsset.data_utf8, 'utf8')
-              : Buffer.from(storedAsset.data_base64, 'base64'),
-          }
-        : {
-            contentType: mime.contentType(path.extname(assetPath)),
-            buffer: fileBuffer,
-          };
-
-      if (!contentType) {
+      const archiveFile = getArchiveEntry(`${pkgName}-${pkgVersion}/${filePath}`);
+      if (!archiveFile) {
         return response.custom({
-          body: `unknown content type for file: ${filePath}`,
-          statusCode: 400,
+          body: `uploaded package file not found: ${filePath}`,
+          statusCode: 404,
         });
       }
-
+      const headers: ResponseHeaders = {
+        'cache-control': 'max-age=10, public',
+        'content-type': headerContentType,
+      };
       return response.custom({
-        body: buffer,
+        body: bufferToStream(archiveFile),
         statusCode: 200,
-        headers: {
-          'cache-control': 'max-age=10, public',
-          'content-type': contentType,
-        },
+        headers,
       });
     } else {
       const registryResponse = await getFile(pkgName, pkgVersion, filePath);

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -12,7 +12,6 @@ import {
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
-  ASSETS_SAVED_OBJECT_TYPE,
   AGENT_SAVED_OBJECT_TYPE,
   AGENT_EVENT_SAVED_OBJECT_TYPE,
   AGENT_ACTION_SAVED_OBJECT_TYPE,
@@ -305,39 +304,13 @@ const getSavedObjectTypes = (
             type: { type: 'keyword' },
           },
         },
-        package_assets: {
-          type: 'nested',
-          properties: {
-            id: { type: 'keyword' },
-            type: { type: 'keyword' },
-          },
-        },
         install_started_at: { type: 'date' },
         install_version: { type: 'keyword' },
         install_status: { type: 'keyword' },
         install_source: { type: 'keyword' },
       },
     },
   },
-  [ASSETS_SAVED_OBJECT_TYPE]: {
-    name: ASSETS_SAVED_OBJECT_TYPE,
-    hidden: false,
-    namespaceType: 'agnostic',
-    management: {
-      importableAndExportable: false,
-    },
-    mappings: {
-      properties: {
-        package_name: { type: 'keyword' },
-        package_version: { type: 'keyword' },
-        install_source: { type: 'keyword' },
-        asset_path: { type: 'keyword' },
-        media_type: { type: 'keyword' },
-        data_utf8: { type: 'text', index: false },
-        data_base64: { type: 'binary' },
-      },
-    },
-  },
 });
 
 export function registerSavedObjects(

diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -5,13 +5,7 @@
  */
 
 import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
-import {
-  InstallablePackage,
-  InstallSource,
-  PackageAssetReference,
-  MAX_TIME_COMPLETE_INSTALL,
-  ASSETS_SAVED_OBJECT_TYPE,
-} from '../../../../common';
+import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import {
   AssetReference,
@@ -30,7 +24,6 @@ import { deleteKibanaSavedObjectsAssets } from './remove';
 import { installTransform } from '../elasticsearch/transform/install';
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
 import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install';
-import { saveArchiveEntries } from '../archive/storage';
 import { ConcurrentInstallOperationError } from '../../../errors';
 
 // this is only exported for testing
@@ -195,26 +188,11 @@ export async function _installPackage({
     if (installKibanaAssetsError) throw installKibanaAssetsError;
     await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
 
-    const packageAssetResults = await saveArchiveEntries({
-      savedObjectsClient,
-      paths,
-      packageInfo,
-      installSource,
-    });
-    const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
-      (result) => ({
-        id: result.id,
-        type: ASSETS_SAVED_OBJECT_TYPE,
-      })
-    );
-
     // update to newly installed version when all assets are successfully installed
     if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
-
     await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
       install_version: pkgVersion,
       install_status: 'installed',
-      package_assets: packageAssetRefs,
     });
 
     return [

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts
@@ -43,7 +43,6 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
-    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test package',
     version: '1.0.0',

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts
@@ -15,7 +15,6 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
-    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',
@@ -33,7 +32,6 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
-    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',