[WIP] Add additional checksum algorithms in aws_s3_object resource and data source

internal/service/s3/object.go

@@ -3,8 +3,12 @@ package s3
 import (
 	"bytes"
 	"context"
+	"crypto/sha1"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/binary"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"log"
 	"net/http"
@@ -70,6 +74,27 @@ func ResourceObject() *schema.Resource {
 				Type:     schema.TypeString,
 				Optional: true,
 			},
+			"checksum_algorithm": {
+				Type:         schema.TypeString,
+				Optional:     true,
+				ValidateFunc: validation.StringInSlice(s3.ChecksumAlgorithm_Values(), false),
+			},
+			"checksum_crc32": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_crc32c": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_sha1": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_sha256": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
 			"content": {
 				Type:          schema.TypeString,
 				Optional:      true,
@@ -199,8 +224,9 @@ func resourceObjectRead(d *schema.ResourceData, meta interface{}) error {
 	key := d.Get("key").(string)
 
 	input := &s3.HeadObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
+		Bucket:       aws.String(bucket),
+		Key:          aws.String(key),
+		ChecksumMode: aws.String(s3.ChecksumModeEnabled),
 	}
 
 	var resp *s3.HeadObjectOutput
@@ -239,6 +265,10 @@ func resourceObjectRead(d *schema.ResourceData, meta interface{}) error {
 
 	d.Set("bucket_key_enabled", resp.BucketKeyEnabled)
 	d.Set("cache_control", resp.CacheControl)
+	d.Set("checksum_crc32", resp.ChecksumCRC32)
+	d.Set("checksum_crc32c", resp.ChecksumCRC32C)
+	d.Set("checksum_sha1", resp.ChecksumSHA1)
+	d.Set("checksum_sha256", resp.ChecksumSHA256)
 	d.Set("content_disposition", resp.ContentDisposition)
 	d.Set("content_encoding", resp.ContentEncoding)
 	d.Set("content_language", resp.ContentLanguage)
@@ -532,6 +562,35 @@ func resourceObjectUpload(d *schema.ResourceData, meta interface{}) error {
 		input.ObjectLockRetainUntilDate = expandObjectDate(v.(string))
 	}
 
+	if v, ok := d.GetOk("checksum_algorithm"); ok {
+		input.ChecksumAlgorithm = aws.String(v.(string))
+
+		buf := new(bytes.Buffer)
+		_, err := io.Copy(buf, body)
+		if err != nil {
+			return fmt.Errorf("copying s3 object body to bytes buffer: %s", err)
+		}
+		// 'body' changes empty after 'io.Copy' called. Set 'buf' instead of 'body' to 'input.Body'.
+		input.Body = buf
+
+		switch v.(string) {
+		case s3.ChecksumAlgorithmCrc32:
+			rawHash := make([]byte, 4)
+			binary.BigEndian.PutUint32(rawHash, crc32.Checksum(buf.Bytes(), crc32.MakeTable(crc32.IEEE)))
+			input.ChecksumCRC32 = aws.String(base64.StdEncoding.EncodeToString(rawHash))
+		case s3.ChecksumAlgorithmCrc32c:
+			rawHash := make([]byte, 4)
+			binary.BigEndian.PutUint32(rawHash, crc32.Checksum(buf.Bytes(), crc32.MakeTable(crc32.Castagnoli)))
+			input.ChecksumCRC32C = aws.String(base64.StdEncoding.EncodeToString(rawHash))
+		case s3.ChecksumAlgorithmSha1:
+			rawHash := sha1.Sum(buf.Bytes())
+			input.ChecksumSHA1 = aws.String(base64.StdEncoding.EncodeToString(rawHash[:]))
+		case s3.ChecksumAlgorithmSha256:
+			rawHash := sha256.Sum256(buf.Bytes())
+			input.ChecksumSHA256 = aws.String(base64.StdEncoding.EncodeToString(rawHash[:]))
+		}
+	}
+
 	if _, err := uploader.Upload(input); err != nil {
 		return fmt.Errorf("uploading object to S3 bucket (%s): %s", bucket, err)
 	}
@@ -589,6 +648,7 @@ func hasObjectContentChanges(d verify.ResourceDiffer) bool {
 	for _, key := range []string{
 		"bucket_key_enabled",
 		"cache_control",
+		"checksum_algorithm",
 		"content_base64",
 		"content_disposition",
 		"content_encoding",

internal/service/s3/object_data_source.go

@@ -37,6 +37,22 @@ func DataSourceObject() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+			"checksum_crc32": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_crc32c": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_sha1": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
+			"checksum_sha256": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
 			"content_disposition": {
 				Type:     schema.TypeString,
 				Computed: true,
@@ -133,8 +149,9 @@ func dataSourceObjectRead(d *schema.ResourceData, meta interface{}) error {
 	key := d.Get("key").(string)
 
 	input := s3.HeadObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
+		Bucket:       aws.String(bucket),
+		Key:          aws.String(key),
+		ChecksumMode: aws.String(s3.ChecksumModeEnabled),
 	}
 	if v, ok := d.GetOk("range"); ok {
 		input.Range = aws.String(v.(string))
@@ -165,6 +182,10 @@ func dataSourceObjectRead(d *schema.ResourceData, meta interface{}) error {
 
 	d.Set("bucket_key_enabled", out.BucketKeyEnabled)
 	d.Set("cache_control", out.CacheControl)
+	d.Set("checksum_crc32", out.ChecksumCRC32)
+	d.Set("checksum_crc32c", out.ChecksumCRC32C)
+	d.Set("checksum_sha1", out.ChecksumSHA1)
+	d.Set("checksum_sha256", out.ChecksumSHA256)
 	d.Set("content_disposition", out.ContentDisposition)
 	d.Set("content_encoding", out.ContentEncoding)
 	d.Set("content_language", out.ContentLanguage)

internal/service/s3/object_data_source_test.go

@@ -230,6 +230,11 @@ func TestAccS3ObjectDataSource_allParams(t *testing.T) {
 					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_legal_hold_status", resourceName, "object_lock_legal_hold_status"),
 					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_mode", resourceName, "object_lock_mode"),
 					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_retain_until_date", resourceName, "object_lock_retain_until_date"),
+					// Additional checksum algorithm
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_crc32", resourceName, "checksum_crc32"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_crc32c", resourceName, "checksum_crc32c"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_sha1", resourceName, "checksum_sha1"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_sha256", resourceName, "checksum_sha256"),
 				),
 			},
 		},
@@ -429,6 +434,150 @@ func TestAccS3ObjectDataSource_singleSlashAsKey(t *testing.T) {
 	})
 }
 
+func TestAccS3ObjectDataSource_checksumCRC32(t *testing.T) {
+	rInt := sdkacctest.RandInt()
+
+	var rObj s3.GetObjectOutput
+	var dsObj s3.GetObjectOutput
+
+	resourceName := "aws_s3_object.object"
+	dataSourceName := "data.aws_s3_object.obj"
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:                  func() { acctest.PreCheck(t) },
+		ErrorCheck:                acctest.ErrorCheck(t, s3.EndpointsID),
+		ProtoV5ProviderFactories:  acctest.ProtoV5ProviderFactories,
+		PreventPostDestroyRefresh: true,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccObjectDataSourceConfig_checksum(rInt, s3.ChecksumAlgorithmCrc32),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckObjectExists(resourceName, &rObj),
+					testAccCheckObjectExistsDataSource(dataSourceName, &dsObj),
+					resource.TestCheckResourceAttr(dataSourceName, "content_length", "11"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "content_type", resourceName, "content_type"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "etag", resourceName, "etag"),
+					resource.TestMatchResourceAttr(dataSourceName, "last_modified", regexp.MustCompile(rfc1123RegexPattern)),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_legal_hold_status", resourceName, "object_lock_legal_hold_status"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_mode", resourceName, "object_lock_mode"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_retain_until_date", resourceName, "object_lock_retain_until_date"),
+					resource.TestCheckNoResourceAttr(dataSourceName, "body"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_crc32", resourceName, "checksum_crc32"),
+					resource.TestCheckResourceAttr(dataSourceName, "checksum_crc32", "ShexVg=="),
+				),
+			},
+		},
+	})
+}
+
+func TestAccS3ObjectDataSource_checksumCRC32C(t *testing.T) {
+	rInt := sdkacctest.RandInt()
+
+	var rObj s3.GetObjectOutput
+	var dsObj s3.GetObjectOutput
+
+	resourceName := "aws_s3_object.object"
+	dataSourceName := "data.aws_s3_object.obj"
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:                  func() { acctest.PreCheck(t) },
+		ErrorCheck:                acctest.ErrorCheck(t, s3.EndpointsID),
+		ProtoV5ProviderFactories:  acctest.ProtoV5ProviderFactories,
+		PreventPostDestroyRefresh: true,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccObjectDataSourceConfig_checksum(rInt, s3.ChecksumAlgorithmCrc32c),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckObjectExists(resourceName, &rObj),
+					testAccCheckObjectExistsDataSource(dataSourceName, &dsObj),
+					resource.TestCheckResourceAttr(dataSourceName, "content_length", "11"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "content_type", resourceName, "content_type"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "etag", resourceName, "etag"),
+					resource.TestMatchResourceAttr(dataSourceName, "last_modified", regexp.MustCompile(rfc1123RegexPattern)),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_legal_hold_status", resourceName, "object_lock_legal_hold_status"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_mode", resourceName, "object_lock_mode"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_retain_until_date", resourceName, "object_lock_retain_until_date"),
+					resource.TestCheckNoResourceAttr(dataSourceName, "body"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_crc32c", resourceName, "checksum_crc32c"),
+					resource.TestCheckResourceAttr(dataSourceName, "checksum_crc32c", "aR2qLw=="),
+				),
+			},
+		},
+	})
+}
+
+func TestAccS3ObjectDataSource_checksumSHA1(t *testing.T) {
+	rInt := sdkacctest.RandInt()
+
+	var rObj s3.GetObjectOutput
+	var dsObj s3.GetObjectOutput
+
+	resourceName := "aws_s3_object.object"
+	dataSourceName := "data.aws_s3_object.obj"
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:                  func() { acctest.PreCheck(t) },
+		ErrorCheck:                acctest.ErrorCheck(t, s3.EndpointsID),
+		ProtoV5ProviderFactories:  acctest.ProtoV5ProviderFactories,
+		PreventPostDestroyRefresh: true,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccObjectDataSourceConfig_checksum(rInt, s3.ChecksumAlgorithmSha1),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckObjectExists(resourceName, &rObj),
+					testAccCheckObjectExistsDataSource(dataSourceName, &dsObj),
+					resource.TestCheckResourceAttr(dataSourceName, "content_length", "11"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "content_type", resourceName, "content_type"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "etag", resourceName, "etag"),
+					resource.TestMatchResourceAttr(dataSourceName, "last_modified", regexp.MustCompile(rfc1123RegexPattern)),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_legal_hold_status", resourceName, "object_lock_legal_hold_status"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_mode", resourceName, "object_lock_mode"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_retain_until_date", resourceName, "object_lock_retain_until_date"),
+					resource.TestCheckNoResourceAttr(dataSourceName, "body"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_sha1", resourceName, "checksum_sha1"),
+					resource.TestCheckResourceAttr(dataSourceName, "checksum_sha1", "Ck1VqNd45QIvq3AZd8XYQLvEhtA="),
+				),
+			},
+		},
+	})
+}
+
+func TestAccS3ObjectDataSource_checksumSHA256(t *testing.T) {
+	rInt := sdkacctest.RandInt()
+
+	var rObj s3.GetObjectOutput
+	var dsObj s3.GetObjectOutput
+
+	resourceName := "aws_s3_object.object"
+	dataSourceName := "data.aws_s3_object.obj"
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:                  func() { acctest.PreCheck(t) },
+		ErrorCheck:                acctest.ErrorCheck(t, s3.EndpointsID),
+		ProtoV5ProviderFactories:  acctest.ProtoV5ProviderFactories,
+		PreventPostDestroyRefresh: true,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccObjectDataSourceConfig_checksum(rInt, s3.ChecksumAlgorithmSha256),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckObjectExists(resourceName, &rObj),
+					testAccCheckObjectExistsDataSource(dataSourceName, &dsObj),
+					resource.TestCheckResourceAttr(dataSourceName, "content_length", "11"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "content_type", resourceName, "content_type"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "etag", resourceName, "etag"),
+					resource.TestMatchResourceAttr(dataSourceName, "last_modified", regexp.MustCompile(rfc1123RegexPattern)),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_legal_hold_status", resourceName, "object_lock_legal_hold_status"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_mode", resourceName, "object_lock_mode"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "object_lock_retain_until_date", resourceName, "object_lock_retain_until_date"),
+					resource.TestCheckNoResourceAttr(dataSourceName, "body"),
+					resource.TestCheckResourceAttrPair(dataSourceName, "checksum_sha256", resourceName, "checksum_sha256"),
+					resource.TestCheckResourceAttr(dataSourceName, "checksum_sha256", "pZGm1Av0IEBKARczz7exkNYsZb8LzaMrV7J32a2fFG4="),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckObjectExistsDataSource(n string, obj *s3.GetObjectOutput) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		rs, ok := s.RootModule().Resources[n]
@@ -771,3 +920,23 @@ data "aws_s3_object" "test" {
 }
 `, rName)
 }
+
+func testAccObjectDataSourceConfig_checksum(randInt int, algorithm string) string {
+	return fmt.Sprintf(`
+resource "aws_s3_bucket" "object_bucket" {
+  bucket = "tf-object-test-bucket-%[1]d"
+}
+
+resource "aws_s3_object" "object" {
+  bucket             = aws_s3_bucket.object_bucket.bucket
+  key                = "tf-testing-obj-%[1]d"
+  content            = "Hello World"
+  checksum_algorithm = %[2]q
+}
+
+data "aws_s3_object" "obj" {
+  bucket = aws_s3_bucket.object_bucket.bucket
+  key    = aws_s3_object.object.key
+}
+`, randInt, algorithm)
+}