From 81b769bd93af9853118bbef7ac1f48b435058787 Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Mon, 10 Apr 2017 15:02:06 -0400 Subject: [PATCH 1/6] PAAS-5611 Adding docs --- .../providers/fastly/r/service_v1.html.markdown | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/website/source/docs/providers/fastly/r/service_v1.html.markdown b/website/source/docs/providers/fastly/r/service_v1.html.markdown index 3429c86aa27c..4df1ac6fce8f 100644 --- a/website/source/docs/providers/fastly/r/service_v1.html.markdown +++ b/website/source/docs/providers/fastly/r/service_v1.html.markdown @@ -155,6 +155,8 @@ Defined below. Defined below. * `sumologic` - (Optional) A Sumologic endpoint to send streaming logs too. Defined below. +* `gcslogging` - (Optional) A gcs endpoint to send streaming logs too. +Defined below. * `response_object` - (Optional) Allows you to create synthetic responses that exist entirely on the varnish machine. Useful for creating error or maintenance pages that exists outside the scope of your datacenter. Best when used with Condition objects. * `vcl` - (Optional) A set of custom VCL configuration blocks. The ability to upload custom VCL code is not enabled by default for new Fastly @@ -322,10 +324,21 @@ The `sumologic` block supports: * `name` - (Required) A unique name to identify this Sumologic endpoint. * `url` - (Required) The URL to Sumologic collector endpoint * `format` - (Optional) Apache-style string or VCL variables to use for log formatting. Defaults to Apache Common Log format (`%h %l %u %t %r %>s`) -* `format_version` - (Optional) The version of the custom logging format used for the configured endpoint. Can be either 1 (the default, version 1 log format) or 2 (the version 2 log format). +* `format_version` - (Optional) The version of the custom logging format used for the configured endpoint. Can be either 1 (the default, version 1 log format) or 2 (the version 2 log format). * `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. * `message_type` - (Optional) How the message should be formatted. One of: classic, loggly, logplex, blank. See [Fastly's Documentation on Sumologic][fastly-sumologic] +The `gcslogging` block supports: + +* `name` - (Required) A unique name to identify this GCS endpoint. +* `email` - (Required) The email address associated with the target GCS bucket on your account. +* `bucket_name` - (Required) The name of the bucket in which to store the logs. +* `secret_key` - (Required) The secret key associated with the target gcs bucket on your account. +* `format` - (Optional) Apache-style string or VCL variables to use for log formatting. Defaults to Apache Common Log format (`%h %l %u %t %r %>s`) +* `format_version` - (Optional) The version of the custom logging format used for the configured endpoint. Can be either 1 (the default, version 1 log format) or 2 (the version 2 log format). +* `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. +* `message_type` - (Optional) How the message should be formatted. One of: classic, loggly, logplex, blank. See [Fastly's Documentation on GCS][fastly-gcs] + The `response_object` block supports: * `name` - (Required) A unique name to identify this Response Object. @@ -369,3 +382,4 @@ Service. [fastly-cname]: https://docs.fastly.com/guides/basic-setup/adding-cname-records [fastly-conditionals]: https://docs.fastly.com/guides/conditions/using-conditions [fastly-sumologic]: https://docs.fastly.com/api/logging#logging_sumologic +[fastly-gcs]: https://docs.fastly.com/api/logging#logging_gcs From 83e9e9a74f8261cd166e80a3be0854ad6026e258 Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Tue, 11 Apr 2017 12:42:01 -0400 Subject: [PATCH 2/6] PAAS-5611 adding gcs logging resource --- .../fastly/resource_fastly_service_v1.go | 96 +++++++++++++++++++ .../fastly/r/service_v1.html.markdown | 2 - 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/builtin/providers/fastly/resource_fastly_service_v1.go b/builtin/providers/fastly/resource_fastly_service_v1.go index e3b4a7c7b368..d11e8402a95e 100644 --- a/builtin/providers/fastly/resource_fastly_service_v1.go +++ b/builtin/providers/fastly/resource_fastly_service_v1.go @@ -641,6 +641,49 @@ func resourceServiceV1() *schema.Resource { }, }, + "gcslogging": { + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // Required fields + "name": { + Type: schema.TypeString, + Required: true, + Description: "Unique name to refer to this logging setup", + }, + "email": { + Type: schema.TypeString, + Required: true, + Description: "The email address associated with the target GCS bucket on your account.", + }, + "bucket_name": { + Type: schema.TypeString, + Required: true, + Description: "The name of the bucket in which to store the logs.", + }, + "secret_key": { + Type: schema.TypeString, + Required: true, + Description: "The secret key associated with the target gcs bucket on your account.", + }, + // Optional fields + "format": { + Type: schema.TypeString, + Optional: true, + Default: "%h %l %u %t %r %>s", + Description: "Apache-style string or VCL variables to use for log formatting", + }, + "response_condition": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "Name of a condition to apply this logging.", + }, + }, + }, + }, + "response_object": { Type: schema.TypeSet, Optional: true, @@ -1443,6 +1486,59 @@ func resourceServiceV1Update(d *schema.ResourceData, meta interface{}) error { } } + // find difference in gcslogging + if d.HasChange("gcslogging") { + os, ns := d.GetChange("gcslogging") + if os == nil { + os = new(schema.Set) + } + if ns == nil { + ns = new(schema.Set) + } + + oss := os.(*schema.Set) + nss := ns.(*schema.Set) + removeGcslogging := oss.Difference(nss).List() + addGcslogging := nss.Difference(oss).List() + + // DELETE old gcslogging configurations + for _, pRaw := range removeGcslogging { + sf := pRaw.(map[string]interface{}) + opts := gofastly.DeleteGCSInput{ + Service: d.Id(), + Version: latestVersion, + Name: sf["name"].(string), + } + + log.Printf("[DEBUG] Fastly gcslogging removal opts: %#v", opts) + err := conn.DeleteGCS(&opts) + if err != nil { + return err + } + } + + // POST new/updated gcslogging + for _, pRaw := range addGcslogging { + sf := pRaw.(map[string]interface{}) + opts := gofastly.CreateGCSInput{ + Service: d.Id(), + Version: latestVersion, + Name: sf["name"].(string), + User: sf["email"].(string), + Bucket: sf["bucket_name"].(string), + SecretKey: sf["secret_key"].(string), + Format: sf["format"].(string), + ResponseCondition: sf["response_condition"].(string), + } + + log.Printf("[DEBUG] Create Sumologic Opts: %#v", opts) + _, err := conn.CreateGCS(&opts) + if err != nil { + return err + } + } + } + // find difference in Response Object if d.HasChange("response_object") { or, nr := d.GetChange("response_object") diff --git a/website/source/docs/providers/fastly/r/service_v1.html.markdown b/website/source/docs/providers/fastly/r/service_v1.html.markdown index 4df1ac6fce8f..b6ebbd45cc7a 100644 --- a/website/source/docs/providers/fastly/r/service_v1.html.markdown +++ b/website/source/docs/providers/fastly/r/service_v1.html.markdown @@ -335,9 +335,7 @@ The `gcslogging` block supports: * `bucket_name` - (Required) The name of the bucket in which to store the logs. * `secret_key` - (Required) The secret key associated with the target gcs bucket on your account. * `format` - (Optional) Apache-style string or VCL variables to use for log formatting. Defaults to Apache Common Log format (`%h %l %u %t %r %>s`) -* `format_version` - (Optional) The version of the custom logging format used for the configured endpoint. Can be either 1 (the default, version 1 log format) or 2 (the version 2 log format). * `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. -* `message_type` - (Optional) How the message should be formatted. One of: classic, loggly, logplex, blank. See [Fastly's Documentation on GCS][fastly-gcs] The `response_object` block supports: From 1d3ffea0618c5c36a2be3d90250b716b1dc542d0 Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Tue, 11 Apr 2017 13:39:47 -0400 Subject: [PATCH 3/6] PAAS-5611 adding gcs testing --- .../fastly/resource_fastly_service_v1.go | 68 +++++++++++++++++++ .../fastly/r/service_v1.html.markdown | 7 ++ 2 files changed, 75 insertions(+) diff --git a/builtin/providers/fastly/resource_fastly_service_v1.go b/builtin/providers/fastly/resource_fastly_service_v1.go index d11e8402a95e..0f9d94c026f5 100644 --- a/builtin/providers/fastly/resource_fastly_service_v1.go +++ b/builtin/providers/fastly/resource_fastly_service_v1.go @@ -668,12 +668,35 @@ func resourceServiceV1() *schema.Resource { Description: "The secret key associated with the target gcs bucket on your account.", }, // Optional fields + "path": { + Type: schema.TypeString, + Optional: true, + Description: "Path to store the files. Must end with a trailing slash", + }, + "gzip_level": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + Description: "Gzip Compression level", + }, + "period": { + Type: schema.TypeInt, + Optional: true, + Default: 3600, + Description: "How frequently the logs should be transferred, in seconds (Default 3600)", + }, "format": { Type: schema.TypeString, Optional: true, Default: "%h %l %u %t %r %>s", Description: "Apache-style string or VCL variables to use for log formatting", }, + "timestamp_format": { + Type: schema.TypeString, + Optional: true, + Default: "%Y-%m-%dT%H:%M:%S.000", + Description: "specified timestamp formatting (default `%Y-%m-%dT%H:%M:%S.000`)", + }, "response_condition": { Type: schema.TypeString, Optional: true, @@ -1972,6 +1995,22 @@ func resourceServiceV1Read(d *schema.ResourceData, meta interface{}) error { log.Printf("[WARN] Error setting Sumologic for (%s): %s", d.Id(), err) } + // refresh GCS Logging + log.Printf("[DEBUG] Refreshing GCS for (%s)", d.Id()) + GCSList, err := conn.ListGCSs(&gofastly.ListGCSsInput{ + Service: d.Id(), + Version: s.ActiveVersion.Number, + }) + + if err != nil { + return fmt.Errorf("[ERR] Error looking up GCS for (%s), version (%s): %s", d.Id(), s.ActiveVersion.Number, err) + } + + gcsl := flattenGCS(GCSList) + if err := d.Set("gcs", gcsl); err != nil { + log.Printf("[WARN] Error setting gcs for (%s): %s", d.Id(), err) + } + // refresh Response Objects log.Printf("[DEBUG] Refreshing Response Object for (%s)", d.Id()) responseObjectList, err := conn.ListResponseObjects(&gofastly.ListResponseObjectsInput{ @@ -2438,6 +2477,35 @@ func flattenSumologics(sumologicList []*gofastly.Sumologic) []map[string]interfa return l } +func flattenGCS(gcsList []*gofastly.GCS) []map[string]interface{} { + var gl []map[string]interface{} + for _, p := range gcsList { + // Convert gcs to a map for saving to state. + ns := map[string]interface{}{ + "name": p.Name, + "email": p.User, + "bucket_name": p.Bucket, + "secret_key": p.SecretKey, + "path": p.Path, + "period": int(p.Period), + "gzip_level": int(p.GzipLevel), + "response_condition": p.ResponseCondition, + "format": p.Format, + } + + // prune any empty values that come from the default string value in structs + for k, v := range ns { + if v == "" { + delete(ns, k) + } + } + + gl = append(gl, ns) + } + + return gl +} + func flattenResponseObjects(responseObjectList []*gofastly.ResponseObject) []map[string]interface{} { var rol []map[string]interface{} for _, ro := range responseObjectList { diff --git a/website/source/docs/providers/fastly/r/service_v1.html.markdown b/website/source/docs/providers/fastly/r/service_v1.html.markdown index b6ebbd45cc7a..4d0bcbfb340f 100644 --- a/website/source/docs/providers/fastly/r/service_v1.html.markdown +++ b/website/source/docs/providers/fastly/r/service_v1.html.markdown @@ -334,6 +334,13 @@ The `gcslogging` block supports: * `email` - (Required) The email address associated with the target GCS bucket on your account. * `bucket_name` - (Required) The name of the bucket in which to store the logs. * `secret_key` - (Required) The secret key associated with the target gcs bucket on your account. +* `path` - (Optional) Path to store the files. Must end with a trailing slash. +If this field is left empty, the files will be saved in the bucket's root path. +* `period` - (Optional) How frequently the logs should be transferred, in +seconds. Default `3600`. +* `gzip_level` - (Optional) Level of GZIP compression, from `0-9`. `0` is no +compression. `1` is fastest and least compressed, `9` is slowest and most +compressed. Default `0`. * `format` - (Optional) Apache-style string or VCL variables to use for log formatting. Defaults to Apache Common Log format (`%h %l %u %t %r %>s`) * `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. From 607a3f9c7470325a6a991671087fc6c8ebf53d24 Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Tue, 11 Apr 2017 13:41:35 -0400 Subject: [PATCH 4/6] Adding testing file --- ...ource_fastly_service_v1_gcslogging_test.go | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go diff --git a/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go b/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go new file mode 100644 index 000000000000..04745ac6d304 --- /dev/null +++ b/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go @@ -0,0 +1,133 @@ +package fastly + +import ( + "fmt" + "reflect" + "testing" + + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" + gofastly "github.com/sethvargo/go-fastly" +) + +func TestResourceFastlyFlattenGCS(t *testing.T) { + cases := []struct { + remote []*gofastly.GCS + local []map[string]interface{} + }{ + { + remote: []*gofastly.GCS{ + &gofastly.GCS{ + Name: "GCS collector", + User: "email@example.com", + Bucket: "bucketName", + SecretKey: "secretKey", + Format: "log format", + ResponseCondition: "condition 1", + Period: 3600, + GzipLevel: 0, + }, + }, + local: []map[string]interface{}{ + map[string]interface{}{ + "name": "GCS collector", + "email": "email@example.com", + "bucket_name": "bucketName", + "secret_key": "secretKey", + "format": "log format", + "response_condition": "condition 1", + "period": 3600, + "gzip_level": 0, + }, + }, + }, + } + + for _, c := range cases { + out := flattenGCS(c.remote) + if !reflect.DeepEqual(out, c.local) { + t.Fatalf("Error matching:\nexpected: %#v\ngot: %#v", c.local, out) + } + } +} + +func TestAccFastlyServiceV1_gcslogging(t *testing.T) { + var service gofastly.ServiceDetail + name := fmt.Sprintf("tf-test-%s", acctest.RandString(10)) + gcsName := fmt.Sprintf("gcs %s", acctest.RandString(10)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckServiceV1Destroy, + Steps: []resource.TestStep{ + { + Config: testAccServiceV1Config_gcs(name, gcsName), + Check: resource.ComposeTestCheckFunc( + testAccCheckServiceV1Exists("fastly_service_v1.foo", &service), + testAccCheckFastlyServiceV1Attributes_gcs(&service, name, gcsName), + ), + }, + }, + }) +} + +func testAccCheckFastlyServiceV1Attributes_gcs(service *gofastly.ServiceDetail, name, gcsName string) resource.TestCheckFunc { + return func(s *terraform.State) error { + + if service.Name != gcsName { + return fmt.Errorf("Bad name, expected (%s), got (%s)", name, service.Name) + } + + conn := testAccProvider.Meta().(*FastlyClient).conn + gcsList, err := conn.ListGCSs(&gofastly.ListGCSsInput{ + Service: service.ID, + Version: service.ActiveVersion.Number, + }) + + if err != nil { + return fmt.Errorf("[ERR] Error looking up GCSs for (%s), version (%s): %s", service.Name, service.ActiveVersion.Number, err) + } + + if len(gcsList) != 1 { + return fmt.Errorf("GCS missing, expected: 1, got: %d", len(gcsList)) + } + + if gcsList[0].Name != gcsName { + return fmt.Errorf("GCS name mismatch, expected: %s, got: %#v", gcsName, gcsList[0].Name) + } + + return nil + } +} + +func testAccServiceV1Config_gcs(name, gcsName string) string { + backendName := fmt.Sprintf("%s.aws.amazon.com", acctest.RandString(3)) + + return fmt.Sprintf(` +resource "fastly_service_v1" "foo" { + name = "%s" + + domain { + name = "test.notadomain.com" + comment = "tf-testing-domain" + } + + backend { + address = "%s" + name = "tf -test backend" + } + + gcslogging { + name = "%s" + email = "email@example.com", + bucket_name = "bucketName", + secret_key = "secretKey", + format = "log format", + response_condition = "condition 1", + } + + force_destroy = true +}`, name, backendName, gcsName) +} From 989193281558ce4f281beb02b6e3dfa3b3ec0e1d Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Tue, 11 Apr 2017 14:20:10 -0400 Subject: [PATCH 5/6] Cleaning up type and making flatten array clearer --- .../fastly/resource_fastly_service_v1.go | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/builtin/providers/fastly/resource_fastly_service_v1.go b/builtin/providers/fastly/resource_fastly_service_v1.go index 0f9d94c026f5..1057b9056c44 100644 --- a/builtin/providers/fastly/resource_fastly_service_v1.go +++ b/builtin/providers/fastly/resource_fastly_service_v1.go @@ -1554,7 +1554,7 @@ func resourceServiceV1Update(d *schema.ResourceData, meta interface{}) error { ResponseCondition: sf["response_condition"].(string), } - log.Printf("[DEBUG] Create Sumologic Opts: %#v", opts) + log.Printf("[DEBUG] Create GCS Opts: %#v", opts) _, err := conn.CreateGCS(&opts) if err != nil { return err @@ -2478,32 +2478,32 @@ func flattenSumologics(sumologicList []*gofastly.Sumologic) []map[string]interfa } func flattenGCS(gcsList []*gofastly.GCS) []map[string]interface{} { - var gl []map[string]interface{} - for _, p := range gcsList { + var GCSList []map[string]interface{} + for _, currentGCS := range gcsList { // Convert gcs to a map for saving to state. - ns := map[string]interface{}{ - "name": p.Name, - "email": p.User, - "bucket_name": p.Bucket, - "secret_key": p.SecretKey, - "path": p.Path, - "period": int(p.Period), - "gzip_level": int(p.GzipLevel), - "response_condition": p.ResponseCondition, - "format": p.Format, + GCSMapString := map[string]interface{}{ + "name": currentGCS.Name, + "email": currentGCS.User, + "bucket_name": currentGCS.Bucket, + "secret_key": currentGCS.SecretKey, + "path": currentGCS.Path, + "period": int(currentGCS.Period), + "gzip_level": int(currentGCS.GzipLevel), + "response_condition": currentGCS.ResponseCondition, + "format": currentGCS.Format, } // prune any empty values that come from the default string value in structs - for k, v := range ns { + for k, v := range GCSMapString { if v == "" { - delete(ns, k) + delete(GCSMapString, k) } } - gl = append(gl, ns) + GCSList = append(GCSList, GCSMapString) } - return gl + return GCSList } func flattenResponseObjects(responseObjectList []*gofastly.ResponseObject) []map[string]interface{} { From 56974aba95f3cf7716d801421d228673fa3bef89 Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Fri, 21 Apr 2017 09:54:58 -0400 Subject: [PATCH 6/6] Fixing issue with tests --- ...ource_fastly_service_v1_gcslogging_test.go | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go b/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go index 04745ac6d304..ed777bf02d76 100644 --- a/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go +++ b/builtin/providers/fastly/resource_fastly_service_v1_gcslogging_test.go @@ -19,26 +19,24 @@ func TestResourceFastlyFlattenGCS(t *testing.T) { { remote: []*gofastly.GCS{ &gofastly.GCS{ - Name: "GCS collector", - User: "email@example.com", - Bucket: "bucketName", - SecretKey: "secretKey", - Format: "log format", - ResponseCondition: "condition 1", - Period: 3600, - GzipLevel: 0, + Name: "GCS collector", + User: "email@example.com", + Bucket: "bucketName", + SecretKey: "secretKey", + Format: "log format", + Period: 3600, + GzipLevel: 0, }, }, local: []map[string]interface{}{ map[string]interface{}{ - "name": "GCS collector", - "email": "email@example.com", - "bucket_name": "bucketName", - "secret_key": "secretKey", - "format": "log format", - "response_condition": "condition 1", - "period": 3600, - "gzip_level": 0, + "name": "GCS collector", + "email": "email@example.com", + "bucket_name": "bucketName", + "secret_key": "secretKey", + "format": "log format", + "period": 3600, + "gzip_level": 0, }, }, }, @@ -76,7 +74,7 @@ func TestAccFastlyServiceV1_gcslogging(t *testing.T) { func testAccCheckFastlyServiceV1Attributes_gcs(service *gofastly.ServiceDetail, name, gcsName string) resource.TestCheckFunc { return func(s *terraform.State) error { - if service.Name != gcsName { + if service.Name != name { return fmt.Errorf("Bad name, expected (%s), got (%s)", name, service.Name) } @@ -125,7 +123,7 @@ resource "fastly_service_v1" "foo" { bucket_name = "bucketName", secret_key = "secretKey", format = "log format", - response_condition = "condition 1", + response_condition = "", } force_destroy = true