Move s3 asset_storage settings to teraslice config (#3614)

This PR makes the following changes: - **\*\*Deprecated\*\*** all `asset_storage` configuration settings in terafoundation - These configuration settings now live in the teraslice config and will be removed from terafoundation in teraslice 2.0 - The teraslice `asset_storage` config settings will take priority of the terafoundation `asset_storage` settings if both are set. - Updated documentation with information on deprecation. - Bumps the following packages: - (patch) **@terascope/job-components** to `v0.74.2` - (patch) **@terascope/types** to `v0.17.2` - (patch) **@terascope/data-mate** to `v0.56.2` - (patch) **elasticsearch-store** to `v0.84.2` - (patch) **ts-transforms** to `v0.85.2` - (patch) **xlucene-parser** to `v0.58.2` - (patch) **xlucene-translator** to `v0.44.2` - (patch) **@terascope/elasticsearch-api** to `v3.20.2` - (patch) **@terascope/teraslice-state-storage** to `v0.53.2` - (minor) **terafoundation** to `v0.64.0` - Bumps **teraslice** to from `v1.5.1` to `v1.6.0` Ref to issue #3601 --------- Co-authored-by: Austin Godber <godber@terascope.io>
terascope · May 17, 2024 · 718ec56 · 718ec56
1 parent d79ff05
commit 718ec56
Show file tree

Hide file tree

Showing 34 changed files with 202 additions and 112 deletions.
diff --git a/docs/configuration/overview.md b/docs/configuration/overview.md
@@ -31,12 +31,13 @@ teraslice:
 ```
 
 ## Terafoundation Configuration Reference
+NOTE: All `asset_storage` related fields are deprecated. Please use the fields in the teraslice config instead. Also the `asset_storage` fields in the teraslice config will take precedence over the ones that are in terafoundation.
 
 |      Field      |    Type    |     Default     |                                      Description                                      |
 | :-------------: | :--------: | :-------------: | :-----------------------------------------------------------------------------------: |
-| **asset_storage_bucket** |  `String` | `ts-assets-<teraslice.name>` |       Name of S3 bucket if using S3 external asset storage.        |
-| **asset_storage_connection** |  `String`  | `"default"` |       Name of the connection of `asset_storage_connection_type` where asset bundles will be stored.        |
-| **asset_storage_connection_type** |  `String`  | `"elasticsearch-next"` |       Name of the connection type that will store asset bundles. options: `elasticsearch-next`, `s3`.        |
+| **[Deprecated]asset_storage_bucket** |  `String` | `ts-assets-<teraslice.name>` |       Name of S3 bucket if using S3 external asset storage.        |
+| **[Deprecated]asset_storage_connection** |  `String`  | `"default"` |       Name of the connection of `asset_storage_connection_type` where asset bundles will be stored.        |
+| **[Deprecated]asset_storage_connection_type** |  `String`  | `"elasticsearch-next"` |       Name of the connection type that will store asset bundles. options: `elasticsearch-next`, `s3`.        |
 | **connectors** |  `Object`  | none |       Required. An object whose keys are connection types and values are objects describing each connection of that type. See [Terafoundation Connectors](#terafoundation-connectors).        |
 | **environment** |  `String`  | `"development"` |       If set to `development` console logging will automatically be turned on.        |
 |  **log_level**  |  `String`  |    `"info"`     |                                Default logging levels                                 |
@@ -55,6 +56,9 @@ teraslice:
 |               **analytics_rate**                |             `duration`             |          `60000`           |                                           Rate in ms in which to push analytics to cluster master                                           |
 |            **api_response_timeout**             |             `duration`             |          `300000`          |    maximum time, in milliseconds, requests to the teraslice API will wait to complete a response without error (e.g. posting large assets)  |
 |              **assets_directory**               |              `String`              |      `"$PWD/assets"`       |                                                        directory to look for assets                                                         |
+| **asset_storage_bucket** |  `String` | `ts-assets-<teraslice.name>` |       Name of S3 bucket if using S3 external asset storage.        |
+| **asset_storage_connection** |  `String`  | `"default"` |       Name of the connection of `asset_storage_connection_type` where asset bundles will be stored.        |
+| **asset_storage_connection_type** |  `String`  | `"elasticsearch-next"` |       Name of the connection type that will store asset bundles. options: `elasticsearch-next`, `s3`.        |
 |                **assets_volume**                |              `String`              |             -              |                                                      name of shared asset volume (k8s)                                                      |
 |             **autoload_directory**              |              `String`              |     `"$PWD/autoload"`      |                                     directory to look for assets to auto deploy when teraslice boots up                                     |
 |            **cluster_manager_type**             |     `"native"`, `"kubernetes"`     |         `"native"`         |                                               determines which cluster system should be used                                                |

diff --git a/e2e/package.json b/e2e/package.json
@@ -42,9 +42,9 @@
         "ms": "^2.1.3"
     },
     "devDependencies": {
-        "@terascope/types": "^0.17.1",
+        "@terascope/types": "^0.17.2",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^0.84.1",
+        "elasticsearch-store": "^0.84.2",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "1.5.2",
+    "version": "1.6.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "0.56.1",
+    "version": "0.56.2",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -29,9 +29,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "^0.50.1",
-        "@terascope/types": "^0.17.1",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/data-types": "^0.50.2",
+        "@terascope/types": "^0.17.2",
+        "@terascope/utils": "^0.59.2",
         "@types/validator": "^13.11.9",
         "awesome-phonenumber": "^2.70.0",
         "date-fns": "^2.30.0",
@@ -46,7 +46,7 @@
         "uuid": "^9.0.1",
         "valid-url": "^1.0.9",
         "validator": "^13.11.0",
-        "xlucene-parser": "^0.58.1"
+        "xlucene-parser": "^0.58.2"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.6",

diff --git a/packages/data-types/package.json b/packages/data-types/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "0.50.1",
+    "version": "0.50.2",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -26,8 +26,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.1",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/types": "^0.17.2",
+        "@terascope/utils": "^0.59.2",
         "graphql": "^14.7.0",
         "lodash": "^4.17.21",
         "yargs": "^17.7.2"

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.20.1",
+    "version": "3.20.2",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -23,16 +23,16 @@
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.1",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/types": "^0.17.2",
+        "@terascope/utils": "^0.59.2",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
     "devDependencies": {
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.84.1",
+        "elasticsearch-store": "^0.84.2",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "0.84.1",
+    "version": "0.84.2",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -29,10 +29,10 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^0.56.1",
-        "@terascope/data-types": "^0.50.1",
-        "@terascope/types": "^0.17.1",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/data-mate": "^0.56.2",
+        "@terascope/data-types": "^0.50.2",
+        "@terascope/types": "^0.17.2",
+        "@terascope/utils": "^0.59.2",
         "ajv": "^6.12.6",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
@@ -41,7 +41,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1",
         "setimmediate": "^1.0.5",
         "uuid": "^9.0.1",
-        "xlucene-translator": "^0.44.1"
+        "xlucene-translator": "^0.44.2"
     },
     "devDependencies": {
         "@types/uuid": "^9.0.8"

diff --git a/packages/generator-teraslice/package.json b/packages/generator-teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "generator-teraslice",
     "displayName": "Generator Teraslice",
-    "version": "0.38.1",
+    "version": "0.38.2",
     "description": "Generate teraslice related packages and code",
     "keywords": [
         "teraslice",
@@ -24,7 +24,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.59.1",
+        "@terascope/utils": "^0.59.2",
         "chalk": "^4.1.2",
         "lodash": "^4.17.21",
         "yeoman-generator": "^5.8.0",

diff --git a/packages/job-components/package.json b/packages/job-components/package.json
@@ -31,7 +31,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.59.1",
+        "@terascope/utils": "^0.59.2",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",

diff --git a/packages/job-components/src/interfaces/context.ts b/packages/job-components/src/interfaces/context.ts
@@ -22,6 +22,9 @@ export interface TerasliceConfig {
     analytics_rate: number|60000;
     api_response_timeout?: number|300000;
     assets_directory?: string[] | string;
+    asset_storage_connection_type?: string;
+    asset_storage_connection?: string;
+    asset_storage_bucket?: string;
     assets_volume?: string;
     cluster_manager_type: ClusterManagerType;
     /** This will only be available in the context of k8s */
@@ -64,8 +67,8 @@ export interface TerasliceConfig {
 
 export interface TerafoundationConfig {
     connectors: Record<string, any>;
-    asset_storage_connection_type: string;
-    asset_storage_connection: string;
+    asset_storage_connection_type?: string;
+    asset_storage_connection?: string;
     asset_storage_bucket?: string;
     prom_metrics_enabled: boolean;
     prom_metrics_port: number;

diff --git a/packages/job-components/src/test-helpers.ts b/packages/job-components/src/test-helpers.ts
@@ -221,6 +221,9 @@ export class TestContext implements i.Context {
                 action_timeout: 10000,
                 analytics_rate: 10000,
                 assets_directory: path.join(process.cwd(), 'assets'),
+                asset_storage_connection_type: 'elasticsearch-next',
+                asset_storage_connection: 'default',
+                asset_storage_bucket: '',
                 cluster_manager_type: options.cluster_manager_type || 'native',
                 hostname: 'localhost',
                 index_rollover_frequency: {

diff --git a/packages/job-components/test/config-validators-spec.ts b/packages/job-components/test/config-validators-spec.ts
@@ -54,12 +54,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {
@@ -88,13 +88,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
-
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
             const schema = jobSchema(context);
             const job = {
                 operations: [{ _op: 'noop' }],
@@ -118,12 +117,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {
@@ -156,12 +155,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {
@@ -193,12 +192,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {
@@ -229,12 +228,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {
@@ -272,12 +271,12 @@ describe('when using native clustering', () => {
                         },
                     },
                 },
-                asset_storage_connection_type: 'elasticsearch-next',
-                asset_storage_connection: 'default',
                 prom_metrics_enabled: false,
                 prom_metrics_port: 3333,
                 prom_metrics_add_default: true,
             };
+            context.sysconfig.teraslice.asset_storage_connection_type = 'elasticsearch-next';
+            context.sysconfig.teraslice.asset_storage_connection = 'default';
 
             const schema = jobSchema(context);
             const job = {

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.77.1",
+    "version": "0.77.2",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {
@@ -32,7 +32,7 @@
     },
     "dependencies": {
         "@kubernetes/client-node": "^0.20.0",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/utils": "^0.59.2",
         "codecov": "^3.8.3",
         "execa": "^5.1.0",
         "fs-extra": "^11.2.0",

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "0.63.2",
+    "version": "0.64.0",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -28,15 +28,15 @@
     },
     "dependencies": {
         "@terascope/file-asset-apis": "^0.13.0",
-        "@terascope/types": "^0.17.1",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/types": "^0.17.2",
+        "@terascope/utils": "^0.59.2",
         "bluebird": "^3.7.2",
         "bunyan": "^1.8.15",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.84.1",
+        "elasticsearch-store": "^0.84.2",
         "express": "^4.19.2",
         "js-yaml": "^4.1.0",
         "nanoid": "^3.3.4",