feat: add retention policy to gcs module [DA-3815] (#29)

* feat: add retention policy to gcs module

* fix: retention lock documentation

* fix: terratest

* fix: permissions on workflow

* fix: workflow

* fix: go version in workflow

* fix: rention lock confliction with object versionning

* fix: test case

* feat: add semetic release and PR

* fix: semantic worflow

* fix: public workflow for semantic PR

.github/workflows/semantic-pr.yaml

@@ -4,7 +4,7 @@
 # Use this workflow for public repos, since public repos cannot access our internal
 # workflows repo.
 ---
-name: public-semantic-pr
+name: repository-semantic-pr
 permissions:
   contents: write
   pull-requests: write
@@ -18,8 +18,8 @@ on:
       - synchronize
 
 jobs:
-  public-semantic-pr:
-    name: public-semantic-pr
+  repository-semantic-pr:
+    name: repository-semantic-pr
     runs-on: ubuntu-latest
     steps:
       - uses: amannn/action-semantic-pull-request@v4

.github/workflows/terratest.yml

@@ -26,10 +26,10 @@ jobs:
         uses: actions/checkout@v2
         with:
           submodules: true
-      - name: Set up Go (1.17)
+      - name: Set up Go (1.22)
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.22
         id: go
       - name: Login to Google Cloud
         uses: google-github-actions/auth@v0
@@ -40,7 +40,7 @@ jobs:
         run: gcloud config set project $GOOGLE_PROJECT
       - name: Run terratest
         run: |
-          make test_and_cover
+          make tests
       - name: Release
         uses: cycjimmy/semantic-release-action@v3
         env:

examples/bucket/main.tf

@@ -24,3 +24,18 @@ module "test_bucket" {
 
   force_destroy = true
 }
+
+module "test_bucket_with_retention" {
+  source = "../../modules/gcp_gcs_bucket"
+
+  location   = "asia-southeast2"
+  name       = "test-bucket-with-retention"
+  project_id = "storage-0994"
+
+  object_versioning_enabled = false
+  retention_lock_enabled    = true
+
+  soft_delete_retention_duration_seconds = 0
+
+  force_destroy = true
+}

examples/bucket/outputs.tf

@@ -5,3 +5,11 @@ output "test_bucket_id" {
 output "test_bucket_name" {
   value = module.test_bucket.name
 }
+
+output "test_bucket_with_retention_id" {
+  value = module.test_bucket_with_retention.id
+}
+
+output "test_bucket_with_retention_name" {
+  value = module.test_bucket_with_retention.name
+}

modules/gcp_gcs_bucket/README.md

@@ -7,17 +7,17 @@ This module will create bucket in GCP with enable server-side encryption and log
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_google"></a> [google](#requirement\_google) | >= 4.51.0 |
-| <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.1.2 |
-| <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.7.2 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.22 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0, < 4.0 |
+| <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.11, < 1.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_google"></a> [google](#provider\_google) | 4.65.2 |
-| <a name="provider_random"></a> [random](#provider\_random) | 3.5.1 |
-| <a name="provider_time"></a> [time](#provider\_time) | 0.9.1 |
+| <a name="provider_google"></a> [google](#provider\_google) | ~> 5.22 |
+| <a name="provider_random"></a> [random](#provider\_random) | >= 3.0, < 4.0 |
+| <a name="provider_time"></a> [time](#provider\_time) | >= 0.11, < 1.0 |
 
 ## Modules
 
@@ -42,9 +42,15 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_default_event_based_hold"></a> [default\_event\_based\_hold](#input\_default\_event\_based\_hold) | (Optional) Whether or not to automatically apply an eventBasedHold to new objects added to the bucket. | `bool` | `false` | no |
 | <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | When deleting a bucket, this boolean option will delete all contained objects. If you try to delete a bucket that contains objects, Terraform will fail that run. | `bool` | `false` | no |
+| <a name="input_lifecycle_rules"></a> [lifecycle\_rules](#input\_lifecycle\_rules) | List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches\_storage\_class should be a comma delimited string. | <pre>list(object({<br>    # Object with keys:<br>    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.<br>    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.<br>    action = map(string)<br><br>    # Object with keys:<br>    # - age - (Optional) Minimum age of an object in days to satisfy this condition.<br>    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.<br>    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".<br>    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, STANDARD, DURABLE_REDUCED_AVAILABILITY.<br>    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.<br>    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.<br>    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.<br>    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.<br>    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.<br>    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.<br>    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.<br>    condition = map(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_location"></a> [location](#input\_location) | (Required) The GCS location | `string` | n/a | yes |
 | <a name="input_name"></a> [name](#input\_name) | (Required) The name of the bucket. | `string` | n/a | yes |
+| <a name="input_object_versioning_enabled"></a> [object\_versioning\_enabled](#input\_object\_versioning\_enabled) | If set to true, the bucket will be versioned. | `bool` | `true` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | (Required) Id of the project in which the bucket is created | `string` | n/a | yes |
+| <a name="input_retention_lock_bucket_enabled"></a> [retention\_lock\_bucket\_enabled](#input\_retention\_lock\_bucket\_enabled) | Bucket will be locked and cannot be deleted from retention policy | `bool` | `false` | no |
+| <a name="input_retention_lock_duration_seconds"></a> [retention\_lock\_duration\_seconds](#input\_retention\_lock\_duration\_seconds) | The duration in seconds that objects in the bucket must be retained and cannot be deleted or replaced. The value must be in between 0 and 3155695200 (100 years). | `number` | `86400` | no |
+| <a name="input_retention_lock_enabled"></a> [retention\_lock\_enabled](#input\_retention\_lock\_enabled) | If set to true, the bucket will be locked and objects in the bucket will be protected from deletion. Note that retention\_policy cannot be used with object versioning. They are mutually exclusive. | `bool` | `false` | no |
+| <a name="input_soft_delete_retention_duration_seconds"></a> [soft\_delete\_retention\_duration\_seconds](#input\_soft\_delete\_retention\_duration\_seconds) | The duration in seconds that soft-deleted objects in the bucket will be retained and cannot be permanently deleted. Default value is 2678400 (30 days). The value must be in between 604800(7 days) and 7776000(90 days). Note: To disable the soft delete policy on a bucket, This field must be set to 0. | `number` | `2678400` | no |
 | <a name="input_storage_class"></a> [storage\_class](#input\_storage\_class) | (Optional, Default: 'STANDARD') The Storage Class of the new bucket. Supported values include: STANDARD, MULTI\_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE. | `string` | `"STANDARD"` | no |
 
 ## Outputs

modules/gcp_gcs_bucket/inputs.tf

@@ -56,6 +56,35 @@ variable "lifecycle_rules" {
   default     = []
 }
 
+variable "object_versioning_enabled" {
+  type        = bool
+  description = "If set to true, the bucket will be versioned."
+  default     = true
+}
+
+variable "retention_lock_bucket_enabled" {
+  type        = bool
+  description = "Bucket will be locked and cannot be deleted from retention policy"
+  default     = false
+}
+
+variable "retention_lock_enabled" {
+  type        = bool
+  description = "If set to true, the bucket will be locked and objects in the bucket will be protected from deletion. Note that retention_policy cannot be used with object versioning. They are mutually exclusive."
+  default     = false
+}
+
+variable "retention_lock_duration_seconds" {
+  type        = number
+  description = "The duration in seconds that objects in the bucket must be retained and cannot be deleted or replaced. The value must be in between 0 and 3155695200 (100 years)."
+  default     = 86400 # 1 day
+
+  validation {
+    condition     = var.retention_lock_duration_seconds >= 0 && var.retention_lock_duration_seconds <= 3155695200
+    error_message = "The retention_lock_duration_seconds must be between 0 and 3155695200 (100 years)."
+  }
+}
+
 variable "soft_delete_retention_duration_seconds" {
   type        = number
   description = "The duration in seconds that soft-deleted objects in the bucket will be retained and cannot be permanently deleted. Default value is 2678400 (30 days). The value must be in between 604800(7 days) and 7776000(90 days). Note: To disable the soft delete policy on a bucket, This field must be set to 0."

modules/gcp_gcs_bucket/main.tf

@@ -74,6 +74,7 @@ resource "google_storage_bucket" "google_storage_bucket_logging" {
 }
 
 resource "google_storage_bucket" "google_storage_bucket" {
+  #checkov:skip=CKV_GCP_78: Bucket versioning should be enabled by default however skipping the Checkov rule as it is not a requirement for all buckets with retention policy enabled.
   name          = "${var.name}_${random_id.random_id.hex}"
   location      = var.location
   force_destroy = var.force_destroy
@@ -87,7 +88,7 @@ resource "google_storage_bucket" "google_storage_bucket" {
   public_access_prevention = "enforced"
 
   versioning {
-    enabled = true
+    enabled = var.object_versioning_enabled
   }
 
   logging {
@@ -121,6 +122,14 @@ resource "google_storage_bucket" "google_storage_bucket" {
     }
   }
 
+  dynamic "retention_policy" {
+    for_each = var.retention_lock_enabled ? [1] : []
+    content {
+      is_locked        = var.retention_lock_bucket_enabled
+      retention_period = var.retention_lock_duration_seconds
+    }
+  }
+
   soft_delete_policy {
     retention_duration_seconds = var.soft_delete_retention_duration_seconds
   }

test/gcp_gcs_bucket_test.go

@@ -40,6 +40,12 @@ func TestGCSBucketCreation(t *testing.T) {
 		name := terraform.Output(t, options, "test_bucket_name")
 		assert.NotEmpty(t, name)
 
+		id_retention := terraform.Output(t, options, "test_bucket_with_retention_id")
+		assert.NotEmpty(t, id_retention)
+
+		name_retention := terraform.Output(t, options, "test_bucket_with_retention_name")
+		assert.NotEmpty(t, name_retention)
+
 		ctx := context.Background()
 		client, err := storage.NewClient(ctx, option.WithCredentialsJSON([]byte(credentials)))
 		if err != nil {