diff --git a/command/plugin.go b/command/plugin.go index 8f8b0903df0..50699654918 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -17,6 +17,7 @@ import ( nullbuilder "github.com/hashicorp/packer/builder/null" hcppackerimagedatasource "github.com/hashicorp/packer/datasource/hcp-packer-image" hcppackeriterationdatasource "github.com/hashicorp/packer/datasource/hcp-packer-iteration" + httpdatasource "github.com/hashicorp/packer/datasource/http" nulldatasource "github.com/hashicorp/packer/datasource/null" artificepostprocessor "github.com/hashicorp/packer/post-processor/artifice" checksumpostprocessor "github.com/hashicorp/packer/post-processor/checksum" @@ -64,6 +65,7 @@ var PostProcessors = map[string]packersdk.PostProcessor{ var Datasources = map[string]packersdk.Datasource{ "hcp-packer-image": new(hcppackerimagedatasource.Datasource), "hcp-packer-iteration": new(hcppackeriterationdatasource.Datasource), + "http": new(httpdatasource.Datasource), "null": new(nulldatasource.Datasource), } diff --git a/datasource/http/data.go b/datasource/http/data.go new file mode 100644 index 00000000000..f31172148c3 --- /dev/null +++ b/datasource/http/data.go @@ -0,0 +1,157 @@ +//go:generate packer-sdc struct-markdown +//go:generate packer-sdc mapstructure-to-hcl2 -type DatasourceOutput,Config +package http + +import ( + "context" + "fmt" + "io/ioutil" + "mime" + "net/http" + "regexp" + "strings" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer-plugin-sdk/common" + "github.com/hashicorp/packer-plugin-sdk/hcl2helper" + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/config" + "github.com/zclconf/go-cty/cty" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + // The URL to request data from. This URL must respond with a `200 OK` response and a `text/*` or `application/json` Content-Type + Url string `mapstructure:"url" required:"true"` + // A map of strings representing additional HTTP headers to include in the request. + Request_headers map[string]string `mapstructure:"request_headers" required:"false"` +} + +type Datasource struct { + config Config +} + +type DatasourceOutput struct { + // The URL the data was requested from. + Url string `mapstructure:"url"` + // The raw body of the HTTP response. + Response_body string `mapstructure:"body"` + // A map of strings representing the response HTTP headers. + // Duplicate headers are contatenated with , according to [RFC2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2) + Response_headers map[string]string `mapstructure:"request_headers"` +} + +func (d *Datasource) ConfigSpec() hcldec.ObjectSpec { + return d.config.FlatMapstructure().HCL2Spec() +} + +func (d *Datasource) Configure(raws ...interface{}) error { + err := config.Decode(&d.config, nil, raws...) + if err != nil { + return err + } + + var errs *packersdk.MultiError + + if d.config.Url == "" { + errs = packersdk.MultiErrorAppend( + errs, + fmt.Errorf("the `url` must be specified")) + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + return nil +} + +func (d *Datasource) OutputSpec() hcldec.ObjectSpec { + return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec() +} + +// This is to prevent potential issues w/ binary files +// and generally unprintable characters +// See https://github.com/hashicorp/terraform/pull/3858#issuecomment-156856738 +func isContentTypeText(contentType string) bool { + + parsedType, params, err := mime.ParseMediaType(contentType) + if err != nil { + return false + } + + allowedContentTypes := []*regexp.Regexp{ + regexp.MustCompile("^text/.+"), + regexp.MustCompile("^application/json$"), + regexp.MustCompile("^application/samlmetadata\\+xml"), + } + + for _, r := range allowedContentTypes { + if r.MatchString(parsedType) { + charset := strings.ToLower(params["charset"]) + return charset == "" || charset == "utf-8" || charset == "us-ascii" + } + } + + return false +} + +// Most of this code comes from http terraform provider data source +// https://github.com/hashicorp/terraform-provider-http/blob/main/internal/provider/data_source.go +func (d *Datasource) Execute() (cty.Value, error) { + ctx := context.TODO() + url, headers := d.config.Url, d.config.Request_headers + client := &http.Client{} + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + // TODO: How to make a test case for this? + if err != nil { + fmt.Println("Error creating http request") + return cty.NullVal(cty.EmptyObject), err + } + + for name, value := range headers { + req.Header.Set(name, value) + } + + resp, err := client.Do(req) + // TODO: How to make test case for this + if err != nil { + fmt.Println("Error making performing http request") + return cty.NullVal(cty.EmptyObject), err + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return cty.NullVal(cty.EmptyObject), fmt.Errorf("HTTP request error. Response code: %d", resp.StatusCode) + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" || isContentTypeText(contentType) == false { + fmt.Println(fmt.Sprintf( + "Content-Type is not recognized as a text type, got %q", + contentType)) + fmt.Println("If the content is binary data, Packer may not properly handle the contents of the response.") + } + + bytes, err := ioutil.ReadAll(resp.Body) + // TODO: How to make test case for this? + if err != nil { + fmt.Println("Error processing response body of call") + return cty.NullVal(cty.EmptyObject), err + } + + responseHeaders := make(map[string]string) + for k, v := range resp.Header { + // Concatenate according to RFC2616 + // cf. https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 + responseHeaders[k] = strings.Join(v, ", ") + } + + output := DatasourceOutput{ + Url: d.config.Url, + Response_headers: responseHeaders, + Response_body: string(bytes), + } + return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil +} diff --git a/datasource/http/data.hcl2spec.go b/datasource/http/data.hcl2spec.go new file mode 100644 index 00000000000..07d3b6b81dd --- /dev/null +++ b/datasource/http/data.hcl2spec.go @@ -0,0 +1,76 @@ +// Code generated by "packer-sdc mapstructure-to-hcl2"; DO NOT EDIT. + +package http + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` + PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` + Url *string `mapstructure:"url" required:"true" cty:"url" hcl:"url"` + Request_headers map[string]string `mapstructure:"request_headers" required:"false" cty:"request_headers" hcl:"request_headers"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "url": &hcldec.AttrSpec{Name: "url", Type: cty.String, Required: false}, + "request_headers": &hcldec.AttrSpec{Name: "request_headers", Type: cty.Map(cty.String), Required: false}, + } + return s +} + +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatDatasourceOutput struct { + Url *string `mapstructure:"url" cty:"url" hcl:"url"` + Response_body *string `mapstructure:"body" cty:"body" hcl:"body"` + Response_headers map[string]string `mapstructure:"request_headers" cty:"request_headers" hcl:"request_headers"` +} + +// FlatMapstructure returns a new FlatDatasourceOutput. +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*DatasourceOutput) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatDatasourceOutput) +} + +// HCL2Spec returns the hcl spec of a DatasourceOutput. +// This spec is used by HCL to read the fields of DatasourceOutput. +// The decoded values from this spec will then be applied to a FlatDatasourceOutput. +func (*FlatDatasourceOutput) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "url": &hcldec.AttrSpec{Name: "url", Type: cty.String, Required: false}, + "body": &hcldec.AttrSpec{Name: "body", Type: cty.String, Required: false}, + "request_headers": &hcldec.AttrSpec{Name: "request_headers", Type: cty.Map(cty.String), Required: false}, + } + return s +} diff --git a/datasource/http/data_acc_test.go b/datasource/http/data_acc_test.go new file mode 100644 index 00000000000..b6b1f83ac27 --- /dev/null +++ b/datasource/http/data_acc_test.go @@ -0,0 +1,109 @@ +package http + +import ( + _ "embed" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "testing" + + "github.com/hashicorp/packer-plugin-sdk/acctest" +) + +//go:embed test-fixtures/basic.pkr.hcl +var testDatasourceBasic string + +//go:embed test-fixtures/empty_url.pkr.hcl +var testDatasourceEmptyUrl string + +//go:embed test-fixtures/404_url.pkr.hcl +var testDatasource404Url string + +func TestHttpDataSource(t *testing.T) { + tests := []struct { + Name string + Path string + Error bool + Outputs map[string]string + }{ + { + Name: "basic_test", + Path: testDatasourceBasic, + Error: false, + Outputs: map[string]string{ + "url": "url is https://www.packer.io/", + // Check that body is not empty + "body": "body is true", + }, + }, + { + Name: "url_is_empty", + Path: testDatasourceEmptyUrl, + Error: true, + Outputs: map[string]string{ + "error": "the `url` must be specified", + }, + }, + { + Name: "404_url", + Path: testDatasource404Url, + Error: true, + }, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + testCase := &acctest.PluginTestCase{ + Name: tt.Name, + Setup: func() error { + return nil + }, + Teardown: func() error { + return nil + }, + Template: tt.Path, + Type: "http", + Check: func(buildCommand *exec.Cmd, logfile string) error { + if buildCommand.ProcessState != nil { + if buildCommand.ProcessState.ExitCode() != 0 && !tt.Error { + return fmt.Errorf("Bad exit code. Logfile: %s", logfile) + } + if tt.Error && buildCommand.ProcessState.ExitCode() == 0 { + return fmt.Errorf("Expected Bad exit code.") + } + } + + if tt.Outputs != nil { + logs, err := os.Open(logfile) + if err != nil { + return fmt.Errorf("Unable find %s", logfile) + } + defer logs.Close() + + logsBytes, err := ioutil.ReadAll(logs) + if err != nil { + return fmt.Errorf("Unable to read %s", logfile) + } + logsString := string(logsBytes) + + for key, val := range tt.Outputs { + if matched, _ := regexp.MatchString(val+".*", logsString); !matched { + t.Fatalf( + "logs doesn't contain expected log %v with value %v in %q", + key, + val, + logsString) + } + } + + } + + return nil + }, + } + acctest.TestPlugin(t, testCase) + }) + } + +} diff --git a/datasource/http/test-fixtures/404_url.pkr.hcl b/datasource/http/test-fixtures/404_url.pkr.hcl new file mode 100644 index 00000000000..398af371d30 --- /dev/null +++ b/datasource/http/test-fixtures/404_url.pkr.hcl @@ -0,0 +1,24 @@ + +source "null" "example" { + communicator = "none" +} + +data "http" "basic" { + url = "https://www.packer.io/thisWillFail" +} + +locals { + url = "${data.http.basic.url}" +} + +build { + name = "mybuild" + sources = [ + "source.null.example" + ] + provisioner "shell-local" { + inline = [ + "echo data is ${local.url}", + ] + } +} diff --git a/datasource/http/test-fixtures/basic.pkr.hcl b/datasource/http/test-fixtures/basic.pkr.hcl new file mode 100644 index 00000000000..1e7e4a46db3 --- /dev/null +++ b/datasource/http/test-fixtures/basic.pkr.hcl @@ -0,0 +1,25 @@ +source "null" "example" { + communicator = "none" +} + +data "http" "basic" { + url = "https://www.packer.io/" +} + +locals { + url = "${data.http.basic.url}" + body = "${data.http.basic.body}" != "" +} + +build { + name = "mybuild" + sources = [ + "source.null.example" + ] + provisioner "shell-local" { + inline = [ + "echo url is ${local.url}", + "echo body is ${local.body}" + ] + } +} diff --git a/datasource/http/test-fixtures/empty_url.pkr.hcl b/datasource/http/test-fixtures/empty_url.pkr.hcl new file mode 100644 index 00000000000..7810218beda --- /dev/null +++ b/datasource/http/test-fixtures/empty_url.pkr.hcl @@ -0,0 +1,23 @@ +source "null" "example" { + communicator = "none" +} + +data "http" "basic" { + url = "" +} + +locals { + url = "${data.http.basic.url}" +} + +build { + name = "mybuild" + sources = [ + "source.null.example" + ] + provisioner "shell-local" { + inline = [ + "echo data is ${local.url}", + ] + } +} diff --git a/website/content/docs/datasources/http.mdx b/website/content/docs/datasources/http.mdx new file mode 100644 index 00000000000..e5140f9fb55 --- /dev/null +++ b/website/content/docs/datasources/http.mdx @@ -0,0 +1,49 @@ +--- +description: | + The http Data Source retrieves information from an http endpoint to be used + during Packer builds +page_title: Http - Data Sources +--- + + + + + + +# Http Data Source + +Type: `http` + +The `http` data source makes an HTTP GET request to the given URL and exports information about the response. + + +## Basic Example + +```hcl +data "http" "example" { + url = "https://checkpoint-api.hashicorp.com/v1/check/terraform" + + # Optional request headers + request_headers = { + Accept = "application/json" + } +} + +## Configuration Reference + +Configuration options are organized below into two categories: required and +optional. Within each category, the available options are alphabetized and +described. + +### Required: + +@include 'datasource/http/Config-required.mdx' + +### Not Required: +@include 'datasource/http/Config-not-required.mdx' + +## Datasource outputs + +The outputs for this datasource are as follows: + +@include 'datasource/http/DatasourceOutput.mdx' diff --git a/website/content/partials/datasource/http/Config-not-required.mdx b/website/content/partials/datasource/http/Config-not-required.mdx new file mode 100644 index 00000000000..1344f2aae9c --- /dev/null +++ b/website/content/partials/datasource/http/Config-not-required.mdx @@ -0,0 +1,5 @@ + + +- `request_headers` (map[string]string) - Request headers for call + + diff --git a/website/content/partials/datasource/http/Config-required.mdx b/website/content/partials/datasource/http/Config-required.mdx new file mode 100644 index 00000000000..f40d0df69dc --- /dev/null +++ b/website/content/partials/datasource/http/Config-required.mdx @@ -0,0 +1,5 @@ + + +- `url` (string) - Url where should be getting things from + + diff --git a/website/content/partials/datasource/http/DatasourceOutput.mdx b/website/content/partials/datasource/http/DatasourceOutput.mdx new file mode 100644 index 00000000000..bdafc1249f8 --- /dev/null +++ b/website/content/partials/datasource/http/DatasourceOutput.mdx @@ -0,0 +1,9 @@ + + +- `url` (string) - Url + +- `body` (string) - Response _ body + +- `request_headers` (map[string]string) - Response _ headers + + diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 1e4b0fbae37..f116a27fe8a 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -718,6 +718,10 @@ "hidden": true } ] + }, + { + "title": "Http", + "path": "datasources/http" } ] },