Skip to content

Commit

Permalink
feat: add minio_s3_bucket_replication resource
Browse files Browse the repository at this point in the history
  • Loading branch information
acolombier committed Oct 16, 2023
1 parent c06ddba commit ae97089
Show file tree
Hide file tree
Showing 20 changed files with 2,989 additions and 44 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ plan.bin
#go packages
vendor/*
.vscode
.idea
.idea
dev.tfrc
12 changes: 12 additions & 0 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,18 @@ tasks:
MINIO_USER: minio
MINIO_PASSWORD: minio123
MINIO_ENABLE_HTTPS: false
SECOND_MINIO_ENDPOINT: 172.17.0.1:9002
SECOND_MINIO_USER: minio
SECOND_MINIO_PASSWORD: minio321
SECOND_MINIO_ENABLE_HTTPS: false
THIRD_MINIO_ENDPOINT: 172.17.0.1:9004
THIRD_MINIO_USER: minio
THIRD_MINIO_PASSWORD: minio456
THIRD_MINIO_ENABLE_HTTPS: false
FOURTH_MINIO_ENDPOINT: 172.17.0.1:9006
FOURTH_MINIO_USER: minio
FOURTH_MINIO_PASSWORD: minio654
FOURTH_MINIO_ENABLE_HTTPS: false
cmds:
- go test -v -cover ./minio
silent: true
99 changes: 96 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: "3"
services:
minio:
image: minio/minio:RELEASE.2023-03-13T19-46-17Z
image: quay.io/minio/minio:RELEASE.2023-08-31T15-31-16Z
ports:
- "9000:9000"
- "9001:9001"
Expand All @@ -12,12 +12,66 @@ services:
MINIO_NOTIFY_WEBHOOK_ENABLE_primary: "on"
MINIO_NOTIFY_WEBHOOK_ENDPOINT_primary: https://webhook.example.com
command: server --console-address :9001 /data{0...3}
secondminio: # This is used to test bucket replication
image: quay.io/minio/minio:RELEASE.2023-08-31T15-31-16Z
ports:
- "9002:9000"
- "9003:9001"
environment:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: minio321
MINIO_CI_CD: "1"
MINIO_NOTIFY_WEBHOOK_ENABLE_primary: "on"
MINIO_NOTIFY_WEBHOOK_ENDPOINT_primary: https://webhook.example.com
command: server --console-address :9001 /data{0...3}
thirdminio: # This is used to test bucket replication
image: quay.io/minio/minio:RELEASE.2023-08-31T15-31-16Z
ports:
- "9004:9000"
- "9005:9001"
environment:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: minio456
MINIO_CI_CD: "1"
MINIO_NOTIFY_WEBHOOK_ENABLE_primary: "on"
MINIO_NOTIFY_WEBHOOK_ENDPOINT_primary: https://webhook.example.com
command: server --console-address :9001 /data{0...3}
fourthminio: # This is used to test bucket replication
image: quay.io/minio/minio:RELEASE.2023-08-31T15-31-16Z
ports:
- "9006:9000"
- "9007:9001"
environment:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: minio654
MINIO_CI_CD: "1"
MINIO_NOTIFY_WEBHOOK_ENABLE_primary: "on"
MINIO_NOTIFY_WEBHOOK_ENDPOINT_primary: https://webhook.example.com
command: server --console-address :9001 /data{0...3}
adminio-ui:
image: rzrbld/adminio-ui:v1.93-210123
environment:
API_BASE_URL: "http://localhost:8080"
ADMINIO_MULTI_BACKEND: "false"
ADMINIO_BACKENDS: '[{"name":"myminio","url":"http://localhost:8080"},{"name":"localhost","url":"http://localhost:8081"},{"name":"error","url":"http://localhost:8082"}]'
ADMINIO_MULTI_BACKEND: "true"
ADMINIO_BACKENDS: |-
[
{
"name": "minio",
"url": "http://localhost:8080"
},
{
"name": "secondminio",
"url": "http://localhost:8081"
},
{
"name": "thirdminio",
"url": "http://localhost:8082"
},
{
"name": "fourthminio",
"url": "http://localhost:8083"
}
]
NGX_ROOT_PATH: "/"
ports:
- "8000:80"
Expand All @@ -34,3 +88,42 @@ services:
- adminio-ui
ports:
- "8080:8080"
secondadminio-api:
image: rzrbld/adminio-api:v1.84-210123
environment:
MINIO_ACCESS: minio
MINIO_SECRET: minio321
MINIO_HOST_PORT: secondminio:9000
MINIO_KMS_MASTER_KEY: terraform-key:da2f4cfa32bed76507dcd44b42872328a8e14f25cd2a1ec0fb85d299a192a447
ADMINIO_HOST_PORT: :8080
depends_on:
- secondminio
- adminio-ui
ports:
- "8081:8080"
thirdadminio-api:
image: rzrbld/adminio-api:v1.84-210123
environment:
MINIO_ACCESS: minio
MINIO_SECRET: minio456
MINIO_HOST_PORT: thirdminio:9000
MINIO_KMS_MASTER_KEY: terraform-key:da2f4cfa32bed76507dcd44b42872328a8e14f25cd2a1ec0fb85d299a192a447
ADMINIO_HOST_PORT: :8082
depends_on:
- thirdminio
- adminio-ui
ports:
- "8082:8080"
fourthadminio-api:
image: rzrbld/adminio-api:v1.84-210123
environment:
MINIO_ACCESS: minio
MINIO_SECRET: minio654
MINIO_HOST_PORT: fourthminio:9000
MINIO_KMS_MASTER_KEY: terraform-key:da2f4cfa32bed76507dcd44b42872328a8e14f25cd2a1ec0fb85d299a192a447
ADMINIO_HOST_PORT: :8083
depends_on:
- fourthminio
- adminio-ui
ports:
- "8083:8080"
24 changes: 24 additions & 0 deletions examples/resources/minio_s3_bucket_replication/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
terraform {
required_providers {
minio = {
source = "aminueza/minio"
version = ">= 1.19.0"
}
}
}

provider "minio" {
minio_server = var.minio_server_a
minio_region = var.minio_region_a
minio_user = var.minio_user_a
minio_password = var.minio_password_a
}

provider "minio" {
alias = "deployment_b"
minio_server = var.minio_server_b
minio_region = var.minio_region_b
minio_user = var.minio_user_b
minio_password = var.minio_password_b
}

178 changes: 178 additions & 0 deletions examples/resources/minio_s3_bucket_replication/resource.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
resource "minio_s3_bucket" "my_bucket_in_a" {
bucket = "my-bucket"
}

resource "minio_s3_bucket" "my_bucket_in_b" {
provider = minio.deployment_b
bucket = "my-bucket"
}

resource "minio_s3_bucket_versioning" "my_bucket_in_a" {
bucket = minio_s3_bucket.my_bucket_in_a.bucket

versioning_configuration {
status = "Enabled"
}
}

resource "minio_s3_bucket_versioning" "my_bucket_in_b" {
provider = minio.deployment_b
bucket = minio_s3_bucket.my_bucket_in_b.bucket

versioning_configuration {
status = "Enabled"
}
}

data "minio_iam_policy_document" "replication_policy" {
statement {
sid = "ReadBuckets"
effect = "Allow"
resources = ["arn:aws:s3:::*"]

actions = [
"s3:ListBucket",
]
}

statement {
sid = "EnableReplicationOnBucket"
effect = "Allow"
resources = ["arn:aws:s3:::my-bucket"]

actions = [
"s3:GetReplicationConfiguration",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:GetBucketLocation",
"s3:GetBucketVersioning",
"s3:GetBucketObjectLockConfiguration",
"s3:GetEncryptionConfiguration",
]
}

statement {
sid = "EnableReplicatingDataIntoBucket"
effect = "Allow"
resources = ["arn:aws:s3:::my-bucket/*"]

actions = [
"s3:GetReplicationConfiguration",
"s3:ReplicateTags",
"s3:AbortMultipartUpload",
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectVersionTagging",
"s3:PutObject",
"s3:PutObjectRetention",
"s3:PutBucketObjectLockConfiguration",
"s3:PutObjectLegalHold",
"s3:DeleteObject",
"s3:ReplicateObject",
"s3:ReplicateDelete",
]
}
}

# One-Way replication (A -> B)
resource "minio_iam_policy" "replication_in_b" {
provider = minio.deployment_b
name = "ReplicationToMyBucketPolicy"
policy = data.minio_iam_policy_document.replication_policy.json
}

resource "minio_iam_user" "replication_in_b" {
provider = minio.deployment_b
name = "my-user"
force_destroy = true
}

resource "minio_iam_user_policy_attachment" "replication_in_b" {
provider = minio.deployment_b
user_name = minio_iam_user.replication_in_b.name
policy_name = minio_iam_policy.replication_in_b.id
}

resource "minio_iam_service_account" "replication_in_b" {
provider = minio.deployment_b
target_user = minio_iam_user.replication_in_b.name

depends_on = [
minio_iam_user_policy_attachment.replication_in_b
]
}

resource "minio_s3_bucket_replication" "replication_in_b" {
bucket = minio_s3_bucket.my_bucket_in_a.bucket

rule {
delete_replication = true
delete_marker_replication = true
existing_object_replication = true
metadata_sync = true # Must be true for two-way

target {
bucket = minio_s3_bucket.my_bucket_in_b.bucket
secure = false
host = var.minio_server_b
bandwidth_limt = "100M"
access_key = minio_iam_service_account.replication_in_b.access_key
secret_key = minio_iam_service_account.replication_in_b.secret_key
}
}

depends_on = [
minio_s3_bucket_versioning.my_bucket_in_a,
minio_s3_bucket_versioning.my_bucket_in_b
]
}

# Two-Way replication (A <-> B)
resource "minio_iam_policy" "replication_in_a" {
name = "ReplicationToMyBucketPolicy"
policy = data.minio_iam_policy_document.replication_policy.json
}

resource "minio_iam_user" "replication_in_a" {
name = "my-user"
force_destroy = true
}

resource "minio_iam_user_policy_attachment" "replication_in_a" {
user_name = minio_iam_user.replication_in_a.name
policy_name = minio_iam_policy.replication_in_a.id
}

resource "minio_iam_service_account" "replication_in_a" {
target_user = minio_iam_user.replication_in_a.name

depends_on = [
minio_iam_user_policy_attachment.replication_in_b
]
}

resource "minio_s3_bucket_replication" "replication_in_a" {
bucket = minio_s3_bucket.my_bucket_in_b.bucket
provider = minio.deployment_b

rule {
delete_replication = true
delete_marker_replication = true
existing_object_replication = true
metadata_sync = true

target {
bucket = minio_s3_bucket.my_bucket_in_a.bucket
host = var.minio_server_a
secure = false
bandwidth_limt = "100M"
access_key = minio_iam_service_account.replication_in_a.access_key
secret_key = minio_iam_service_account.replication_in_a.secret_key
}
}

depends_on = [
minio_s3_bucket_versioning.my_bucket_in_a,
minio_s3_bucket_versioning.my_bucket_in_b,
]
}
39 changes: 39 additions & 0 deletions examples/resources/minio_s3_bucket_replication/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
variable "minio_region_a" {
description = "Default MINIO region"
default = "us-east-1"
}

variable "minio_server_a" {
description = "Default MINIO host and port"
default = "localhost:9000"
}

variable "minio_user_a" {
description = "MINIO user"
default = "minio"
}

variable "minio_password_a" {
description = "MINIO password"
default = "minio123"
}

variable "minio_region_b" {
description = "Default MINIO region"
default = "eu-west-1"
}

variable "minio_server_b" {
description = "Default MINIO host and port"
default = "localhost:9002"
}

variable "minio_user_b" {
description = "MINIO user"
default = "minio"
}

variable "minio_password_b" {
description = "MINIO password"
default = "minio321"
}
Loading

0 comments on commit ae97089

Please sign in to comment.