From 448a810bcc4dc7ff700eff793f182fbf3cd1f272 Mon Sep 17 00:00:00 2001 From: sotojn Date: Wed, 31 Jul 2024 08:33:29 -0700 Subject: [PATCH 1/4] update minio to RELEASE.2022-06-11T19-55-32Z --- packages/scripts/src/helpers/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 705c3e3ad62..b2a5b249984 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -66,7 +66,7 @@ export const MINIO_HOSTNAME = process.env.MINIO_HOSTNAME || HOST_IP; export const MINIO_PORT = process.env.MINIO_PORT || '49000'; export const ENCRYPT_MINIO = toBoolean(process.env.ENCRYPT_MINIO ?? false); export const MINIO_HOST = `http${ENCRYPT_MINIO ? 's' : ''}://${MINIO_HOSTNAME}:${MINIO_PORT}`; -export const MINIO_VERSION = process.env.MINIO_VERSION || 'RELEASE.2020-02-07T23-28-16Z'; +export const MINIO_VERSION = process.env.MINIO_VERSION || 'RELEASE.2022-06-11T19-55-32Z'; export const MINIO_DOCKER_IMAGE = process.env.MINIO_DOCKER_IMAGE || 'minio/minio'; export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || 'minioadmin'; export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || 'minioadmin'; From dbbfe6eb218486a658b8a49f20d62b5009955aa2 Mon Sep 17 00:00:00 2001 From: sotojn Date: Wed, 31 Jul 2024 08:59:40 -0700 Subject: [PATCH 2/4] add MINIO_VOLUME to config in scripts --- packages/scripts/src/helpers/config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index b2a5b249984..9dc709b86d6 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -1,4 +1,5 @@ import { address } from 'ip'; +import fs from 'node:fs'; import { toBoolean, toSafeString, isCI, toIntegerOrThrow } from '@terascope/utils'; @@ -70,6 +71,7 @@ export const MINIO_VERSION = process.env.MINIO_VERSION || 'RELEASE.2022-06-11T19 export const MINIO_DOCKER_IMAGE = process.env.MINIO_DOCKER_IMAGE || 'minio/minio'; export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || 'minioadmin'; export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || 'minioadmin'; +export const MINIO_VOLUME = fs.mkdtempSync('minio'); export const RABBITMQ_VERSION = process.env.RABBITMQ_VERSION || '3.8.16-management-alpine'; export const RABBITMQ_DOCKER_IMAGE = process.env.RABBITMQ_DOCKER_IMAGE || 'rabbitmq'; From 1222522a9709074d52fb299239f5134a9a17870d Mon Sep 17 00:00:00 2001 From: sotojn Date: Wed, 31 Jul 2024 09:37:06 -0700 Subject: [PATCH 3/4] set tmp dir for minio volume --- packages/scripts/src/helpers/config.ts | 4 +++- packages/scripts/src/helpers/scripts.ts | 6 ++++-- .../scripts/src/helpers/test-runner/services.ts | 17 ++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 9dc709b86d6..79bb4a59fff 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -1,5 +1,7 @@ import { address } from 'ip'; import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { toBoolean, toSafeString, isCI, toIntegerOrThrow } from '@terascope/utils'; @@ -71,7 +73,7 @@ export const MINIO_VERSION = process.env.MINIO_VERSION || 'RELEASE.2022-06-11T19 export const MINIO_DOCKER_IMAGE = process.env.MINIO_DOCKER_IMAGE || 'minio/minio'; export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || 'minioadmin'; export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || 'minioadmin'; -export const MINIO_VOLUME = fs.mkdtempSync('minio'); +export const MINIO_VOLUME = fs.mkdtempSync(path.join(os.tmpdir(), 'ts-minio')); export const RABBITMQ_VERSION = process.env.RABBITMQ_VERSION || '3.8.16-management-alpine'; export const RABBITMQ_DOCKER_IMAGE = process.env.RABBITMQ_DOCKER_IMAGE || 'rabbitmq'; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index fc2a454cfca..88f122346d5 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -255,7 +255,7 @@ export type DockerRunOptions = { env?: ExecEnv; network?: string; args?: string[]; - mount?: string + mount?: string[] }; export async function dockerRun( @@ -271,7 +271,9 @@ export async function dockerRun( } if (opt.mount && !ignoreMount) { - args.push('--mount', opt.mount); + for (const mount of opt.mount) { + args.push('--mount', mount); + } } if (opt.ports && opt.ports.length) { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index f4a2092c3cd..025ef1905bc 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -54,7 +54,7 @@ const services: Readonly>> = { [Service.RestrainedElasticsearch]: { image: config.ELASTICSEARCH_DOCKER_IMAGE, name: `${config.TEST_NAMESPACE}_${config.ELASTICSEARCH_NAME}`, - mount: `type=bind,source=${restrainedElasticsearchConfigPath},target=/usr/share/elasticsearch/config/elasticsearch.yml`, + mount: [`type=bind,source=${restrainedElasticsearchConfigPath},target=/usr/share/elasticsearch/config/elasticsearch.yml`], ports: [`${config.RESTRAINED_ELASTICSEARCH_PORT}:${config.RESTRAINED_ELASTICSEARCH_PORT}`], env: { ES_JAVA_OPTS: config.SERVICE_HEAP_OPTS, @@ -131,13 +131,16 @@ const services: Readonly>> = { [Service.Minio]: { image: config.MINIO_DOCKER_IMAGE, name: `${config.TEST_NAMESPACE}_${config.MINIO_NAME}`, - tmpfs: config.SERVICES_USE_TMPFS - ? ['/data'] - : undefined, + /// We will be able to go back and use this on a later version of minio + // tmpfs: config.SERVICES_USE_TMPFS + // ? ['/data'] + // : undefined, ports: [`${config.MINIO_PORT}:${config.MINIO_PORT}`], mount: config.ENCRYPT_MINIO - ? `type=bind,source=${path.join(getRootDir(), '/e2e/test/certs')},target=/opt/certs` - : '', + ? [`type=bind,source=${path.join(getRootDir(), '/e2e/test/certs')},target=/opt/certs`, + `type=bind,source=${config.MINIO_VOLUME},target=/data` + ] + : [`type=bind,source=${config.MINIO_VOLUME},target=/data`], env: { MINIO_ACCESS_KEY: config.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: config.MINIO_SECRET_KEY, @@ -151,7 +154,7 @@ const services: Readonly>> = { image: config.RABBITMQ_DOCKER_IMAGE, name: `${config.TEST_NAMESPACE}_${config.RABBITMQ_NAME}`, ports: [`${config.RABBITMQ_MANAGEMENT_PORT}:15672`, `${config.RABBITMQ_PORT}:5672`], - mount: `type=bind,source=${rabbitConfigPath},target=/etc/rabbitmq/rabbitmq.conf`, + mount: [`type=bind,source=${rabbitConfigPath},target=/etc/rabbitmq/rabbitmq.conf`], env: { RABBITMQ_HOSTNAME: '0.0.0.0', RABBITMQ_USER: config.RABBITMQ_USER, From cc84808a341bcdd4735b529bdbbca87abf705799 Mon Sep 17 00:00:00 2001 From: sotojn Date: Wed, 31 Jul 2024 15:26:48 -0700 Subject: [PATCH 4/4] add reference to minio issue in comments --- packages/scripts/src/helpers/test-runner/services.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 025ef1905bc..a000de17c51 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -132,6 +132,7 @@ const services: Readonly>> = { image: config.MINIO_DOCKER_IMAGE, name: `${config.TEST_NAMESPACE}_${config.MINIO_NAME}`, /// We will be able to go back and use this on a later version of minio + /// Minio issue ref: https://github.com/minio/minio/issues/15733 // tmpfs: config.SERVICES_USE_TMPFS // ? ['/data'] // : undefined,