Base64 encode response body (#158)

* Adding response_body_base64_std attribute to hold standard base64 encoding of the response body (#157)

* Updating CHANGELOG.md and docs (#157)

* Renaming attribute from response_body_base64_std to response_body_base64 (#157)

* Clean up imports from merge

* Add version check to x509cert test to skip Terraform `v0.14.0`

* Add Changie entries

* Add version check to `TestDataSource_ResponseBodyBinary` to skip TF `v0.14.0`

* Update version checks to skip all TF versions `0.14.x`

* Update version checks to use tfversion variables

* Rename `ResponseBodyBase64` field

* Clean up `go.mod`

---------

Co-authored-by: SBGoods <github@simplebox.anonaddy.com>

.changes/unreleased/ENHANCEMENTS-20230613-133335.yaml

@@ -0,0 +1,6 @@
+kind: ENHANCEMENTS
+body: 'data-source/http: `response_body_base64` has been added and contains a standard
+  base64 encoding of the response body'
+time: 2023-06-13T13:33:35.77003-04:00
+custom:
+  Issue: "158"

.changes/unreleased/ENHANCEMENTS-20230613-133401.yaml

@@ -0,0 +1,6 @@
+kind: ENHANCEMENTS
+body: 'data-source/http: Replaced issuing warning on the basis of possible non-text
+  `Content-Type` with issuing warning if response body does not contain valid UTF-8.'
+time: 2023-06-13T13:34:01.499384-04:00
+custom:
+  Issue: "158"

docs/data-sources/http.md

@@ -165,6 +165,7 @@ resource "null_resource" "example" {
 - `body` (String, Deprecated) The response body returned as a string. **NOTE**: This is deprecated, use `response_body` instead.
 - `id` (String) The URL used for the request.
 - `response_body` (String) The response body returned as a string.
+- `response_body_base64` (String) The response body encoded as base64 (standard) as defined in [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-4).
 - `response_headers` (Map of String) A map of response header field names and values. Duplicate headers are concatenated according to [RFC2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2).
 - `status_code` (Number) The HTTP response status code.
 

internal/provider/data_source_http.go

@@ -7,15 +7,15 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/hashicorp/go-retryablehttp"
 	"github.com/hashicorp/terraform-plugin-framework-validators/int64validator"
@@ -125,6 +125,11 @@ a 5xx-range (except 501) status code is received. For further details see
 				DeprecationMessage: "Use response_body instead",
 			},
 
+			"response_body_base64": schema.StringAttribute{
+				Description: "The response body encoded as base64 (standard) as defined in [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-4).",
+				Computed:    true,
+			},
+
 			"ca_cert_pem": schema.StringAttribute{
 				Description: "Certificate data of the Certificate Authority (CA) " +
 					"in [PEM (RFC 1421)](https://datatracker.ietf.org/doc/html/rfc1421) format.",
@@ -329,14 +334,6 @@ func (d *httpDataSource) Read(ctx context.Context, req datasource.ReadRequest, r
 
 	defer response.Body.Close()
 
-	contentType := response.Header.Get("Content-Type")
-	if !isContentTypeText(contentType) {
-		resp.Diagnostics.AddWarning(
-			fmt.Sprintf("Content-Type is not recognized as a text type, got %q", contentType),
-			"If the content is binary data, Terraform may not properly handle the contents of the response.",
-		)
-	}
-
 	bytes, err := io.ReadAll(response.Body)
 	if err != nil {
 		resp.Diagnostics.AddError(
@@ -346,7 +343,15 @@ func (d *httpDataSource) Read(ctx context.Context, req datasource.ReadRequest, r
 		return
 	}
 
+	if !utf8.Valid(bytes) {
+		resp.Diagnostics.AddWarning(
+			"Response body is not recognized as UTF-8",
+			"Terraform may not properly handle the response_body if the contents are binary.",
+		)
+	}
+
 	responseBody := string(bytes)
+	responseBodyBase64Std := base64.StdEncoding.EncodeToString(bytes)
 
 	responseHeaders := make(map[string]string)
 	for k, v := range response.Header {
@@ -364,52 +369,28 @@ func (d *httpDataSource) Read(ctx context.Context, req datasource.ReadRequest, r
 	model.ResponseHeaders = respHeadersState
 	model.ResponseBody = types.StringValue(responseBody)
 	model.Body = types.StringValue(responseBody)
+	model.ResponseBodyBase64 = types.StringValue(responseBodyBase64Std)
 	model.StatusCode = types.Int64Value(int64(response.StatusCode))
 
 	diags = resp.State.Set(ctx, model)
 	resp.Diagnostics.Append(diags...)
 }
 
-// This is to prevent potential issues w/ binary files
-// and generally unprintable characters
-// See https://github.com/hashicorp/terraform/pull/3858#issuecomment-156856738
-func isContentTypeText(contentType string) bool {
-
-	parsedType, params, err := mime.ParseMediaType(contentType)
-	if err != nil {
-		return false
-	}
-
-	allowedContentTypes := []*regexp.Regexp{
-		regexp.MustCompile("^text/.+"),
-		regexp.MustCompile("^application/json$"),
-		regexp.MustCompile(`^application/samlmetadata\+xml`),
-	}
-
-	for _, r := range allowedContentTypes {
-		if r.MatchString(parsedType) {
-			charset := strings.ToLower(params["charset"])
-			return charset == "" || charset == "utf-8" || charset == "us-ascii"
-		}
-	}
-
-	return false
-}
-
 type modelV0 struct {
-	ID              types.String `tfsdk:"id"`
-	URL             types.String `tfsdk:"url"`
-	Method          types.String `tfsdk:"method"`
-	RequestHeaders  types.Map    `tfsdk:"request_headers"`
-	RequestBody     types.String `tfsdk:"request_body"`
-	RequestTimeout  types.Int64  `tfsdk:"request_timeout_ms"`
-	Retry           types.Object `tfsdk:"retry"`
-	ResponseHeaders types.Map    `tfsdk:"response_headers"`
-	CaCertificate   types.String `tfsdk:"ca_cert_pem"`
-	Insecure        types.Bool   `tfsdk:"insecure"`
-	ResponseBody    types.String `tfsdk:"response_body"`
-	Body            types.String `tfsdk:"body"`
-	StatusCode      types.Int64  `tfsdk:"status_code"`
+	ID                 types.String `tfsdk:"id"`
+	URL                types.String `tfsdk:"url"`
+	Method             types.String `tfsdk:"method"`
+	RequestHeaders     types.Map    `tfsdk:"request_headers"`
+	RequestBody        types.String `tfsdk:"request_body"`
+	RequestTimeout     types.Int64  `tfsdk:"request_timeout_ms"`
+	Retry              types.Object `tfsdk:"retry"`
+	ResponseHeaders    types.Map    `tfsdk:"response_headers"`
+	CaCertificate      types.String `tfsdk:"ca_cert_pem"`
+	Insecure           types.Bool   `tfsdk:"insecure"`
+	ResponseBody       types.String `tfsdk:"response_body"`
+	Body               types.String `tfsdk:"body"`
+	ResponseBodyBase64 types.String `tfsdk:"response_body_base64"`
+	StatusCode         types.Int64  `tfsdk:"status_code"`
 }
 
 type retryModel struct {

internal/provider/data_source_http_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
+	"github.com/hashicorp/terraform-plugin-testing/tfversion"
 )
 
 func TestDataSource_200(t *testing.T) {
@@ -230,32 +231,37 @@ func TestDataSource_utf16_200(t *testing.T) {
 	})
 }
 
-// TODO: This test fails under Terraform 0.14. It should be uncommented when we
-// are able to include Terraform version logic within acceptance tests
-// (see https://github.com/hashicorp/terraform-plugin-sdk/issues/776), or when
-// 0.14 is removed from the test matrix (see
-// https://github.com/hashicorp/terraform-provider-http/pull/74).
-//
-//func TestDataSource_x509cert(t *testing.T) {
-//	testHttpMock := setUpMockHttpServer()
-//	defer testHttpMock.server.Close()
-//
-//	resource.UnitTest(t, resource.TestCase{
-//		ProtoV5ProviderFactories: protoV5ProviderFactories(),
-//		Steps: []resource.TestStep{
-//			{
-//				Config: fmt.Sprintf(`
-//							data "http" "http_test" {
-//  								url = "%s/x509-ca-cert/200"
-//							}`, testHttpMock.server.URL),
-//				Check: resource.ComposeTestCheckFunc(
-//					resource.TestCheckResourceAttr("data.http.http_test", "response_body", "pem"),
-//					resource.TestCheckResourceAttr("data.http.http_test", "status_code", "200"),
-//				),
-//			},
-//		},
-//	})
-//}
+func TestDataSource_x509cert(t *testing.T) {
+	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/x-x509-ca-cert")
+		w.WriteHeader(http.StatusOK)
+		_, err := w.Write([]byte("pem"))
+		if err != nil {
+			t.Errorf("error writing body: %s", err)
+		}
+	}))
+	defer testServer.Close()
+
+	resource.ParallelTest(t, resource.TestCase{
+		ProtoV5ProviderFactories: protoV5ProviderFactories(),
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			//test fails in TF 0.14.x due to https://github.com/hashicorp/terraform-provider-http/issues/58
+			tfversion.SkipBetween(tfversion.Version0_14_0, tfversion.Version0_15_0),
+		},
+		Steps: []resource.TestStep{
+			{
+				Config: fmt.Sprintf(`
+							data "http" "http_test" {
+ 								url = "%s/x509-ca-cert/200"
+							}`, testServer.URL),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("data.http.http_test", "response_body", "pem"),
+					resource.TestCheckResourceAttr("data.http.http_test", "status_code", "200"),
+				),
+			},
+		},
+	})
+}
 
 func TestDataSource_UpgradeFromVersion2_2_0(t *testing.T) {
 	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -793,6 +799,64 @@ func TestDataSource_MaxDelayAtLeastEqualToMinDelay(t *testing.T) {
 	})
 }
 
+func TestDataSource_ResponseBodyText(t *testing.T) {
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`你好世界`)) // Hello world
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer svr.Close()
+
+	resource.ParallelTest(t, resource.TestCase{
+		ProtoV5ProviderFactories: protoV5ProviderFactories(),
+		Steps: []resource.TestStep{
+			{
+				Config: fmt.Sprintf(`
+							data "http" "http_test" {
+								url = "%s"
+							}`, svr.URL),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("data.http.http_test", "response_body", "你好世界"),
+					resource.TestCheckResourceAttr("data.http.http_test", "response_body_base64", "5L2g5aW95LiW55WM"),
+				),
+			},
+		},
+	})
+}
+
+func TestDataSource_ResponseBodyBinary(t *testing.T) {
+	// 1 x 1 transparent gif pixel.
+	const transPixel = "\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x21\xF9\x04\x01\x00\x00\x00\x00\x2C\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02\x44\x01\x00\x3B"
+
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/gif")
+		_, _ = w.Write([]byte(transPixel))
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer svr.Close()
+
+	resource.ParallelTest(t, resource.TestCase{
+		ProtoV5ProviderFactories: protoV5ProviderFactories(),
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			//test fails in TF 0.14.x due to quirk in behavior
+			//where a warning results in nothing being written to output.
+			tfversion.SkipBetween(tfversion.Version0_14_0, tfversion.Version0_15_0),
+		},
+		Steps: []resource.TestStep{
+			{
+				Config: fmt.Sprintf(`
+							data "http" "http_test" {
+								url = "%s"
+							}`, svr.URL),
+				Check: resource.ComposeTestCheckFunc(
+					// Note the replacement character in the string representation in `response_body`.
+					resource.TestCheckResourceAttr("data.http.http_test", "response_body", "GIF89a\x01\x00\x01\x00�\x00\x00\x00\x00\x00\x00\x00\x00!�\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"),
+					resource.TestCheckResourceAttr("data.http.http_test", "response_body_base64", "R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="),
+				),
+			},
+		},
+	})
+}
+
 func checkServerAndProxyRequestCount(proxyRequestCount, serverRequestCount *int) resource.TestCheckFunc {
 	return func(_ *terraform.State) error {
 		if *proxyRequestCount != *serverRequestCount {