feat: add local docker compose

Author: solomonope

docker-compose-gcs-distributed-test.yaml

@@ -13,18 +13,15 @@ services:
     ports:
       - "8000:8000"
     environment:
-      - P_S3_URL=http://minio:9000
-      - P_S3_ACCESS_KEY=parseable
-      - P_S3_SECRET_KEY=supersecret
-      - P_S3_REGION=us-east-1
-      - P_S3_BUCKET=parseable
+      - P_GCS_BUCKET=parseable-test-gcs-local
       - P_STAGING_DIR=/tmp/data
       - P_USERNAME=parseableadmin
       - P_PASSWORD=parseableadmin
       - P_CHECK_UPDATE=false
       - P_PARQUET_COMPRESSION_ALGO=snappy
       - P_MODE=query
       - RUST_LOG=warn
+      - GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/mpt-randd-8217aef869fd.json
     networks:
       - parseable-internal
     healthcheck:
@@ -36,7 +33,9 @@ services:
       restart_policy:
         condition: on-failure
         delay: 20s
-        max_attempts: 3
+        max_attempts:
+    volumes:
+      - "/home/opeyemi/Downloads/:/parseable/svc/:ro,z"
   # ingest server one
   parseable-ingest-one:
     container_name: parseable-ingest-one
@@ -48,11 +47,7 @@ services:
     ports:
       - "8000"
     environment:
-      - P_S3_URL=http://minio:9000
-      - P_S3_ACCESS_KEY=parseable
-      - P_S3_SECRET_KEY=supersecret
-      - P_S3_REGION=us-east-1
-      - P_S3_BUCKET=parseable
+      - P_GCS_BUCKET=parseable-test-gcs-local
       - P_STAGING_DIR=/tmp/data
       - P_USERNAME=parseableadmin
       - P_PASSWORD=parseableadmin
@@ -61,6 +56,7 @@ services:
       - P_MODE=ingest
       - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
       - RUST_LOG=warn
+      - GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/mpt-randd-8217aef869fd.json
     networks:
       - parseable-internal
     healthcheck:
@@ -76,6 +72,8 @@ services:
         condition: on-failure
         delay: 20s
         max_attempts: 3
+    volumes:
+      - "/home/opeyemi/Downloads/:/parseable/svc/:ro,z"
 
   quest:
     platform: linux/amd64
@@ -90,10 +88,10 @@ services:
         "20",
         "10",
         "5m",
-        "minio:9000",
-        "parseable",
-        "supersecret",
-        "parseable",
+        "storage.googleapis.com",
+        "GOOG1ESWXFI3QEE6X2SEYKIQDK6PONOW3QPKDD3HJZOBE34H6PWQWWQHYYBSI",
+        "1Cfhir7cep9ej56UlLX6RKmkuV2BMcC2+BDFoSoa",
+        "parseable-test-gcs-local",
         "http://parseable-ingest-one:8000",
         "parseableadmin",
         "parseableadmin",
@@ -105,8 +103,6 @@ services:
         condition: service_healthy
       parseable-ingest-one:
         condition: service_healthy
-      minio:
-        condition: service_healthy
     deploy:
       restart_policy:
         condition: on-failure

src/parseable/mod.rs

@@ -249,6 +249,8 @@ impl Parseable {
             return "S3 bucket";
         } else if self.storage.name() == "blob_store" {
             return "Azure Blob Storage";
+        } else if self.storage.name() == "gcs" {
+            return "Google Object Store";
         }
         "Unknown"
     }

src/storage/gcs.rs

@@ -69,7 +69,13 @@ use super::{
 
 pub struct GCSConfig {
     /// The endpoint to GCS or compatible object storage platform
-    #[arg(long, env = "P_S3_URL", value_name = "url", required = true)]
+    #[arg(
+        long,
+        env = "P_GCS_URL",
+        value_name = "url",
+        default_value = "https://storage.googleapis.com",
+        required = false
+    )]
     pub endpoint_url: String,
 
     /// The GCS or compatible object storage bucket to be used for storage
@@ -107,7 +113,7 @@ impl GCSConfig {
             backoff: BackoffConfig::default(),
         };
 
-        let builder = GoogleCloudStorageBuilder::new()
+        let builder = GoogleCloudStorageBuilder::from_env()
             .with_bucket_name(&self.bucket_name)
             .with_retry(retry_config);