diff --git a/integration/v4_to_v5/testdata/logpush_job/expected/logpush_job.tf b/integration/v4_to_v5/testdata/logpush_job/expected/logpush_job.tf new file mode 100644 index 0000000..c61082f --- /dev/null +++ b/integration/v4_to_v5/testdata/logpush_job/expected/logpush_job.tf @@ -0,0 +1,96 @@ +# Minimal logpush job +resource "cloudflare_logpush_job" "minimal" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" +} + +# Job with logpull_options only (no output_options) +resource "cloudflare_logpush_job" "with_logpull_options" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + logpull_options = "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" +} + +# Job with output_options block +resource "cloudflare_logpush_job" "with_output_options" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + + output_options = { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp"] + output_type = "ndjson" + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +} + +# Job with cve20214428 field (should be renamed) +resource "cloudflare_logpush_job" "with_cve_field" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + + output_options = { + output_type = "ndjson" + cve_2021_44228 = true + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +} + +# Job with instant-logs kind (should be removed) +resource "cloudflare_logpush_job" "instant_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" +} + +# Job with edge kind (should be preserved) +resource "cloudflare_logpush_job" "edge_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "edge" +} + +# Job with "" kind (should be preserved) +resource "cloudflare_logpush_job" "edge_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "" +} + +# Full featured job with all transformations +resource "cloudflare_logpush_job" "full" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + enabled = true + name = "my-logpush-job" + frequency = "high" + + output_options = { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp", "RayID"] + output_type = "ndjson" + sample_rate = 1.0 + timestamp_format = "unixnano" + cve_2021_44228 = true + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + } +} diff --git a/integration/v4_to_v5/testdata/logpush_job/expected/terraform.tfstate b/integration/v4_to_v5/testdata/logpush_job/expected/terraform.tfstate new file mode 100644 index 0000000..4d8a3bc --- /dev/null +++ b/integration/v4_to_v5/testdata/logpush_job/expected/terraform.tfstate @@ -0,0 +1,174 @@ +{ + "version": 4, + "terraform_version": "1.5.0", + "serial": 1, + "lineage": "test-logpush-job-lineage", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "minimal", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_logpull_options", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "logpull_options": "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_output_options", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "output_options": { + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "output_type": "ndjson", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1.0 + } + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_cve_field", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "4", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "output_options": { + "cve_2021_44228": true, + "output_type": "ndjson", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1.0 + } + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "instant_logs", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "5", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "edge_logs", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "6", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "kind": "edge" + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "full", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "7", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "name": "my-logpush-job", + "frequency": "high", + "max_upload_bytes": 5000000.0, + "max_upload_records": 1000.0, + "max_upload_interval_seconds": 30.0, + "output_options": { + "cve_2021_44228": true, + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp", "RayID"], + "output_type": "ndjson", + "sample_rate": 1.0, + "timestamp_format": "unixnano", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n" + } + } + } + ] + } + ] +} diff --git a/integration/v4_to_v5/testdata/logpush_job/input/logpush_job.tf b/integration/v4_to_v5/testdata/logpush_job/input/logpush_job.tf new file mode 100644 index 0000000..50e2dff --- /dev/null +++ b/integration/v4_to_v5/testdata/logpush_job/input/logpush_job.tf @@ -0,0 +1,85 @@ +# Minimal logpush job +resource "cloudflare_logpush_job" "minimal" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" +} + +# Job with logpull_options only (no output_options) +resource "cloudflare_logpush_job" "with_logpull_options" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + logpull_options = "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" +} + +# Job with output_options block +resource "cloudflare_logpush_job" "with_output_options" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + + output_options { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp"] + output_type = "ndjson" + } +} + +# Job with cve20214428 field (should be renamed) +resource "cloudflare_logpush_job" "with_cve_field" { + account_id = var.cloudflare_account_id + dataset = "audit_logs" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + + output_options { + cve20214428 = true + output_type = "ndjson" + } +} + +# Job with instant-logs kind (should be removed) +resource "cloudflare_logpush_job" "instant_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "instant-logs" +} + +# Job with edge kind (should be preserved) +resource "cloudflare_logpush_job" "edge_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "edge" +} + +# Job with "" kind (should be preserved) +resource "cloudflare_logpush_job" "edge_logs" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "" +} + +# Full featured job with all transformations +resource "cloudflare_logpush_job" "full" { + zone_id = var.cloudflare_zone_id + dataset = "http_requests" + destination_conf = "https://logpush-receiver.sd.cfplat.com" + kind = "instant-logs" + enabled = true + name = "my-logpush-job" + frequency = "high" + + output_options { + cve20214428 = true + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp", "RayID"] + output_type = "ndjson" + sample_rate = 1.0 + timestamp_format = "unixnano" + } +} diff --git a/integration/v4_to_v5/testdata/logpush_job/input/terraform.tfstate b/integration/v4_to_v5/testdata/logpush_job/input/terraform.tfstate new file mode 100644 index 0000000..1ed18cd --- /dev/null +++ b/integration/v4_to_v5/testdata/logpush_job/input/terraform.tfstate @@ -0,0 +1,172 @@ +{ + "version": 4, + "terraform_version": "1.5.0", + "serial": 1, + "lineage": "test-logpush-job-lineage", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "minimal", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_logpull_options", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "logpull_options": "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_output_options", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "output_options": [ + { + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "output_type": "ndjson" + } + ] + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "with_cve_field", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "4", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "output_options": [ + { + "cve20214428": true, + "output_type": "ndjson" + } + ] + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "instant_logs", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "5", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "kind": "instant-logs" + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "edge_logs", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "6", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "kind": "edge" + } + } + ] + }, + { + "mode": "managed", + "type": "cloudflare_logpush_job", + "name": "full", + "provider": "provider[\"registry.terraform.io/cloudflare/cloudflare\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "7", + "account_id": "f037e56e89293a057740de681ac9abbe", + "dataset": "http_requests", + "destination_conf": "https://logpush-receiver.sd.cfplat.com", + "enabled": true, + "kind": "instant-logs", + "name": "my-logpush-job", + "frequency": "high", + "max_upload_bytes": 5000000, + "max_upload_records": 1000, + "max_upload_interval_seconds": 30, + "output_options": [ + { + "cve20214428": true, + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp", "RayID"], + "output_type": "ndjson", + "sample_rate": 1.0, + "timestamp_format": "unixnano" + } + ], + "error_message": "Some error", + "last_complete": "2024-01-01T00:00:00Z", + "last_error": "2024-01-01T00:00:00Z" + } + } + ] + } + ] +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go index f11b069..5371f65 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -5,14 +5,15 @@ import ( "github.com/cloudflare/tf-migrate/internal/resources/api_token" "github.com/cloudflare/tf-migrate/internal/resources/dns_record" "github.com/cloudflare/tf-migrate/internal/resources/logpull_retention" + "github.com/cloudflare/tf-migrate/internal/resources/logpush_job" "github.com/cloudflare/tf-migrate/internal/resources/notification_policy_webhooks" "github.com/cloudflare/tf-migrate/internal/resources/r2_bucket" "github.com/cloudflare/tf-migrate/internal/resources/workers_kv" "github.com/cloudflare/tf-migrate/internal/resources/workers_kv_namespace" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_access_service_token" + "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_device_posture_rule" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_dlp_custom_profile" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_gateway_policy" - "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_device_posture_rule" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_list" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_tunnel_cloudflared" "github.com/cloudflare/tf-migrate/internal/resources/zero_trust_tunnel_cloudflared_route" @@ -30,6 +31,7 @@ func RegisterAllMigrations() { zone.NewV4ToV5Migrator() zone_dnssec.NewV4ToV5Migrator() logpull_retention.NewV4ToV5Migrator() + logpush_job.NewV4ToV5Migrator() notification_policy_webhooks.NewV4ToV5Migrator() r2_bucket.NewV4ToV5Migrator() workers_kv.NewV4ToV5Migrator() diff --git a/internal/resources/logpush_job/v4_to_v5.go b/internal/resources/logpush_job/v4_to_v5.go new file mode 100644 index 0000000..99c3984 --- /dev/null +++ b/internal/resources/logpush_job/v4_to_v5.go @@ -0,0 +1,225 @@ +package logpush_job + +import ( + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/cloudflare/tf-migrate/internal" + "github.com/cloudflare/tf-migrate/internal/hcl" + "github.com/cloudflare/tf-migrate/internal/transform" + tfhcl "github.com/cloudflare/tf-migrate/internal/transform/hcl" + "github.com/cloudflare/tf-migrate/internal/transform/state" +) + +type V4ToV5Migrator struct { +} + +func NewV4ToV5Migrator() transform.ResourceTransformer { + migrator := &V4ToV5Migrator{} + // Register with the OLD (v4) resource name - same as v5 in this case + internal.RegisterMigrator("cloudflare_logpush_job", "v4", "v5", migrator) + return migrator +} + +func (m *V4ToV5Migrator) GetResourceType() string { + return "cloudflare_logpush_job" +} + +func (m *V4ToV5Migrator) CanHandle(resourceType string) bool { + return resourceType == "cloudflare_logpush_job" +} + +func (m *V4ToV5Migrator) Preprocess(content string) string { + return content +} + +// This resource does not rename, so we return the same name for both old and new +func (m *V4ToV5Migrator) GetResourceRename() (string, string) { + return "cloudflare_logpush_job", "cloudflare_logpush_job" +} + +func (m *V4ToV5Migrator) TransformConfig(ctx *transform.Context, block *hclwrite.Block) (*transform.TransformResult, error) { + body := block.Body() + + // 1. Convert output_options block to attribute (block → attribute syntax) + // This handles: output_options { ... } → output_options = { ... } + if outputBlock := tfhcl.FindBlockByType(body, "output_options"); outputBlock != nil { + outputBody := outputBlock.Body() + + // Rename cve20214428 → cve_2021_44228 BEFORE conversion + tfhcl.RenameAttribute(outputBody, "cve20214428", "cve_2021_44228") + + // Add v4 schema defaults if not already present (to preserve v4 behavior in v5) + // v5 does not have defaults for these fields, so we must make them explicit + m.ensureV4SchemaDefaults(outputBody) + + tfhcl.ConvertSingleBlockToAttribute(body, "output_options", "output_options") + } + + // 2. Handle kind = "instant-logs" → remove attribute + // "instant-logs" is no longer valid in v5, remove the attribute entirely + if kindAttr := body.GetAttribute("kind"); kindAttr != nil { + kindValue := tfhcl.ExtractStringFromAttribute(kindAttr) + if kindValue == "instant-logs" { + // Remove the attribute entirely since instant-logs is not valid in v5 + body.RemoveAttribute("kind") + } + } + + return &transform.TransformResult{ + Blocks: []*hclwrite.Block{block}, + RemoveOriginal: false, + }, nil +} + +// ensureV4SchemaDefaults adds v4 schema defaults to output_options if not present +// This preserves v4 behavior in v5, which has no defaults for these fields +func (m *V4ToV5Migrator) ensureV4SchemaDefaults(body *hclwrite.Body) { + // Use a slice to ensure deterministic ordering of defaults + type defaultPair struct { + field string + value interface{} + } + + v4Defaults := []defaultPair{ + {"field_delimiter", ","}, + {"record_prefix", "{"}, + {"record_suffix", "}\n"}, + {"timestamp_format", "unixnano"}, + {"sample_rate", 1.0}, + } + + for _, pair := range v4Defaults { + if body.GetAttribute(pair.field) == nil { + // Field not present, add the v4 default + tokens := hcl.TokensForSimpleValue(pair.value) + if tokens != nil { + body.SetAttributeRaw(pair.field, tokens) + } + } + } +} + +func (m *V4ToV5Migrator) TransformState(ctx *transform.Context, stateJSON gjson.Result, resourcePath string, resourceName string) (string, error) { + result := stateJSON.String() + + if !stateJSON.Exists() || !stateJSON.Get("attributes").Exists() { + return result, nil + } + + attrs := stateJSON.Get("attributes") + + // 1. Convert integer fields to float64 for int64 compatibility + result = m.convertNumericFields(result, attrs) + + // 2. Transform output_options array to object + result = m.transformOutputOptions(result, attrs) + + // 3. Remove computed-only fields (these should not be in state) + result = state.RemoveFields(result, "attributes", attrs, + "error_message", "last_complete", "last_error") + + // 4. Handle kind value change: "instant-logs" → remove attribute + // "instant-logs" is no longer valid in v5, remove it entirely + if kind := attrs.Get("kind"); kind.Exists() && kind.String() == "instant-logs" { + result, _ = sjson.Delete(result, "attributes.kind") + } + + return result, nil +} + +// convertNumericFields converts integer fields to float64 for int64 compatibility +func (m *V4ToV5Migrator) convertNumericFields(result string, attrs gjson.Result) string { + numericFields := []string{ + "max_upload_bytes", + "max_upload_records", + "max_upload_interval_seconds", + } + + for _, field := range numericFields { + if val := attrs.Get(field); val.Exists() { + // Convert to float64 for int64 compatibility + result, _ = sjson.Set(result, "attributes."+field, state.ConvertToFloat64(val)) + } + } + + return result +} + +// transformOutputOptions transforms output_options from array to object and renames fields +// Preserves v4 schema defaults in state to match config (v5 has no defaults for these fields) +func (m *V4ToV5Migrator) transformOutputOptions(result string, attrs gjson.Result) string { + outputOpts := attrs.Get("output_options") + + if !outputOpts.Exists() { + return result + } + + // Handle array → object transformation + if outputOpts.IsArray() { + array := outputOpts.Array() + if len(array) == 0 { + // Empty array → remove field + result, _ = sjson.Delete(result, "attributes.output_options") + } else { + // Take first element and convert to object + firstElem := array[0] + obj := make(map[string]interface{}) + + firstElem.ForEach(func(key, value gjson.Result) bool { + k := key.String() + + // Rename cve20214428 → cve_2021_44228 + if k == "cve20214428" { + k = "cve_2021_44228" + } + + // Keep all values including v4 schema defaults (they're now in migrated config) + obj[k] = state.ConvertGjsonValue(value) + return true + }) + + // Add v4 schema defaults if not present (to match migrated config) + m.addV4SchemaDefaultsToState(obj) + + result, _ = sjson.Set(result, "attributes.output_options", obj) + } + } else if outputOpts.IsObject() { + // Already an object, just rename field if needed + result = state.RenameField(result, "attributes.output_options", outputOpts, + "cve20214428", "cve_2021_44228") + + // Add v4 schema defaults if not present (to match migrated config) + // First, get the current object from result + updatedOpts := gjson.Get(result, "attributes.output_options") + if updatedOpts.Exists() { + obj := make(map[string]interface{}) + updatedOpts.ForEach(func(key, value gjson.Result) bool { + obj[key.String()] = state.ConvertGjsonValue(value) + return true + }) + m.addV4SchemaDefaultsToState(obj) + result, _ = sjson.Set(result, "attributes.output_options", obj) + } + } + + return result +} + +// addV4SchemaDefaultsToState adds v4 schema defaults to state object if not present +func (m *V4ToV5Migrator) addV4SchemaDefaultsToState(obj map[string]interface{}) { + v4Defaults := map[string]interface{}{ + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1.0, + } + + for field, defaultValue := range v4Defaults { + if _, exists := obj[field]; !exists { + obj[field] = defaultValue + } + } +} diff --git a/internal/resources/logpush_job/v4_to_v5_test.go b/internal/resources/logpush_job/v4_to_v5_test.go new file mode 100644 index 0000000..56a6965 --- /dev/null +++ b/internal/resources/logpush_job/v4_to_v5_test.go @@ -0,0 +1,661 @@ +package logpush_job + +import ( + "testing" + + "github.com/cloudflare/tf-migrate/internal/testhelpers" +) + +func TestV4ToV5Transformation(t *testing.T) { + migrator := NewV4ToV5Migrator() + + t.Run("ConfigTransformation", func(t *testing.T) { + tests := []testhelpers.ConfigTestCase{ + { + Name: "Minimal resource", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" +}`, + }, + { + Name: "Resource with logpull_options only (no output_options)", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + logpull_options = "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + logpull_options = "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" +}`, + }, + { + Name: "Convert output_options block to attribute and add v4 defaults", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp"] + output_type = "ndjson" + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options = { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp"] + output_type = "ndjson" + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +}`, + }, + { + Name: "Rename cve20214428 to cve_2021_44228 and add v4 defaults", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options { + cve20214428 = true + output_type = "ndjson" + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options = { + output_type = "ndjson" + cve_2021_44228 = true + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +}`, + }, + { + Name: "Add v4 defaults when some fields are user-configured", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options { + output_type = "ndjson" + field_delimiter = "|" + sample_rate = 0.5 + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options = { + output_type = "ndjson" + field_delimiter = "|" + sample_rate = 0.5 + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + } +}`, + }, + { + Name: "Preserve all user-configured values over defaults", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options { + output_type = "csv" + field_delimiter = "|" + record_prefix = "[" + record_suffix = "]" + timestamp_format = "rfc3339" + sample_rate = 0.1 + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + + output_options = { + output_type = "csv" + field_delimiter = "|" + record_prefix = "[" + record_suffix = "]" + timestamp_format = "rfc3339" + sample_rate = 0.1 + } +}`, + }, + { + Name: "Handle kind instant-logs by removing attribute", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "instant-logs" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" +}`, + }, + { + Name: "Preserve kind edge value", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "edge" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "edge" +}`, + }, + { + Name: "Preserve kind empty string", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "" +}`, + }, + { + Name: "Full transformation with all changes", + Input: `resource "cloudflare_logpush_job" "example" { + account_id = "abc123" + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + kind = "instant-logs" + enabled = true + + output_options { + cve20214428 = true + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp", "RayID"] + output_type = "ndjson" + sample_rate = 1.0 + timestamp_format = "unixnano" + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + account_id = "abc123" + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + enabled = true + + output_options = { + batch_prefix = "{" + batch_suffix = "}" + field_names = ["ClientIP", "EdgeStartTimestamp", "RayID"] + output_type = "ndjson" + sample_rate = 1.0 + timestamp_format = "unixnano" + cve_2021_44228 = true + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + } +}`, + }, + { + Name: "Multiple resources in one file", + Input: `resource "cloudflare_logpush_job" "job1" { + dataset = "http_requests" + destination_conf = "s3://bucket1/logs?region=us-west-2" + kind = "instant-logs" + + output_options { + cve20214428 = true + output_type = "ndjson" + } +} + +resource "cloudflare_logpush_job" "job2" { + dataset = "firewall_events" + destination_conf = "s3://bucket2/logs?region=us-east-1" + + output_options { + output_type = "csv" + } +}`, + Expected: `resource "cloudflare_logpush_job" "job1" { + dataset = "http_requests" + destination_conf = "s3://bucket1/logs?region=us-west-2" + + output_options = { + output_type = "ndjson" + cve_2021_44228 = true + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +} + +resource "cloudflare_logpush_job" "job2" { + dataset = "firewall_events" + destination_conf = "s3://bucket2/logs?region=us-east-1" + + output_options = { + output_type = "csv" + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +}`, + }, + { + Name: "With variable references", + Input: `resource "cloudflare_logpush_job" "example" { + account_id = var.account_id + dataset = var.dataset + destination_conf = var.destination_conf + + output_options { + field_names = var.field_names + output_type = "ndjson" + } +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + account_id = var.account_id + dataset = var.dataset + destination_conf = var.destination_conf + + output_options = { + field_names = var.field_names + output_type = "ndjson" + field_delimiter = "," + record_prefix = "{" + record_suffix = "}\n" + timestamp_format = "unixnano" + sample_rate = 1 + } +}`, + }, + { + Name: "With deprecated frequency field", + Input: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + frequency = "high" +}`, + Expected: `resource "cloudflare_logpush_job" "example" { + dataset = "http_requests" + destination_conf = "s3://mybucket/logs?region=us-west-2" + frequency = "high" +}`, + }, + } + + testhelpers.RunConfigTransformTests(t, tests, migrator) + }) + + t.Run("StateTransformation", func(t *testing.T) { + tests := []testhelpers.StateTestCase{ + { + Name: "Minimal state", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2" + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2" + } +}`, + }, + { + Name: "State with logpull_options only (no output_options)", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "logpull_options": "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "logpull_options": "fields=ClientIP,EdgeStartTimestamp×tamps=unixnano" + } +}`, + }, + { + Name: "Transform output_options array to object", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": [ + { + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "output_type": "ndjson" + } + ] + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "output_type": "ndjson", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1 + } + } +}`, + }, + { + Name: "Rename cve20214428 in output_options array", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": [ + { + "cve20214428": true, + "output_type": "ndjson" + } + ] + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "cve_2021_44228": true, + "output_type": "ndjson", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1 + } + } +}`, + }, + { + Name: "Rename cve20214428 in output_options object", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "cve20214428": true, + "output_type": "ndjson" + } + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "cve_2021_44228": true, + "output_type": "ndjson", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "timestamp_format": "unixnano", + "sample_rate": 1 + } + } +}`, + }, + { + Name: "Convert numeric fields to float64", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "max_upload_bytes": 5000000, + "max_upload_records": 1000, + "max_upload_interval_seconds": 30 + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "max_upload_bytes": 5000000.0, + "max_upload_records": 1000.0, + "max_upload_interval_seconds": 30.0 + } +}`, + }, + { + Name: "Preserve v4 schema defaults in output_options state", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": [ + { + "cve20214428": true, + "output_type": "ndjson", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "sample_rate": 1.0, + "timestamp_format": "unixnano" + } + ] + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "cve_2021_44228": true, + "output_type": "ndjson", + "field_names": ["ClientIP", "EdgeStartTimestamp"], + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n", + "sample_rate": 1.0, + "timestamp_format": "unixnano" + } + } +}`, + }, + { + Name: "Preserve user-configured values over v4 defaults in state", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": [ + { + "output_type": "csv", + "field_delimiter": "|", + "record_prefix": "[", + "sample_rate": 0.5, + "timestamp_format": "rfc3339" + } + ] + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": { + "output_type": "csv", + "field_delimiter": "|", + "record_prefix": "[", + "sample_rate": 0.5, + "timestamp_format": "rfc3339", + "record_suffix": "}\n" + } + } +}`, + }, + { + Name: "Handle kind instant-logs by removing attribute", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "kind": "instant-logs" + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2" + } +}`, + }, + { + Name: "Preserve kind edge value", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "kind": "edge" + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "kind": "edge" + } +}`, + }, + { + Name: "Remove computed-only fields", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "error_message": "Some error", + "last_complete": "2024-01-01T00:00:00Z", + "last_error": "2024-01-01T00:00:00Z" + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2" + } +}`, + }, + { + Name: "Empty output_options array removed", + Input: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "output_options": [] + } +}`, + Expected: `{ + "attributes": { + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2" + } +}`, + }, + { + Name: "Full state transformation", + Input: `{ + "attributes": { + "account_id": "abc123", + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "kind": "instant-logs", + "enabled": true, + "max_upload_bytes": 5000000, + "max_upload_records": 1000, + "max_upload_interval_seconds": 30, + "output_options": [ + { + "cve20214428": true, + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp", "RayID"], + "output_type": "ndjson", + "sample_rate": 1.0, + "timestamp_format": "unixnano" + } + ], + "error_message": "Some error", + "last_complete": "2024-01-01T00:00:00Z", + "last_error": "2024-01-01T00:00:00Z" + } +}`, + Expected: `{ + "attributes": { + "account_id": "abc123", + "dataset": "http_requests", + "destination_conf": "s3://mybucket/logs?region=us-west-2", + "enabled": true, + "max_upload_bytes": 5000000.0, + "max_upload_records": 1000.0, + "max_upload_interval_seconds": 30.0, + "output_options": { + "cve_2021_44228": true, + "batch_prefix": "{", + "batch_suffix": "}", + "field_names": ["ClientIP", "EdgeStartTimestamp", "RayID"], + "output_type": "ndjson", + "sample_rate": 1.0, + "timestamp_format": "unixnano", + "field_delimiter": ",", + "record_prefix": "{", + "record_suffix": "}\n" + } + } +}`, + }, + } + + testhelpers.RunStateTransformTests(t, tests, migrator) + }) +}