Implement S3 archiver (#1055)

This closes #290 as the second required archiver.
iggy-rs · Jul 13, 2024 · fd0e80a · fd0e80a
1 parent d6e5a90
commit fd0e80a
Show file tree

Hide file tree

Showing 16 changed files with 466 additions and 45 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/configs/server.json b/configs/server.json
@@ -8,9 +8,11 @@
  },
  "s3": {
  "key_id": "123",
- "access_key": "secret",
+ "key_secret": "secret",
  "bucket": "iggy",
- "region": "eu-west-1"
+ "endpoint": "http://localhost:9000",
+ "region": "eu-west-1",
+ "tmp_upload_dir": "local_data/s3_tmp"
  }
  },
  "messages": {

diff --git a/configs/server.toml b/configs/server.toml
@@ -12,12 +12,21 @@ path = "local_data/archive"
 [data_maintenance.archiver.s3]
 # Access key ID for the S3 bucket.
 key_id = "123"
+
 # Secret access key for the S3 bucket
-access_key = "secret"
+key_secret = "secret"
+
+# Name of the S3 bucket.
+bucket = "iggy"
+
+# Endpoint of the S3 region.
+endpoint = "http://localhost:9000"
+
 # Region of the S3 bucket.
 region = "eu-west-1"
-# Endpoint of the S3 bucket.
-bucket = "iggy"
+
+# Temporary directory for storing the data before uploading to S3.
+tmp_upload_dir = "local_data/s3_tmp"
 
 [data_maintenance.messages]
 # Enables or disables the archiver process for closed segments containing messages.