feat: Added support for VPC Flow Logs in Parquet format

README.md

@@ -120,6 +120,14 @@ You can add additional tags with `intra_subnet_tags` as with other subnet types.
 
 VPC Flow Log allows to capture IP traffic for a specific network interface (ENI), subnet, or entire VPC. This module supports enabling or disabling VPC Flow Logs for entire VPC. If you need to have VPC Flow Logs for subnet or ENI, you have to manage it outside of this module with [aws_flow_log resource](https://www.terraform.io/docs/providers/aws/r/flow_log.html).
 
+### VPC Flow Log Examples
+
+By default `file_format` is `plain-text`. You can also specify `parquet` to have logs written in Apache Parquet format.
+
+```
+log_file_format = "parquet"
+```
+
 ### Permissions Boundary
 
 If your organization requires a permissions boundary to be attached to the VPC Flow Log role, make sure that you specify an ARN of the permissions boundary policy as `vpc_flow_log_permissions_boundary` argument. Read more about required [IAM policy for publishing flow logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-cwl.html#flow-logs-iam).

examples/vpc-flow-logs/main.tf

@@ -31,6 +31,25 @@ module "vpc_with_flow_logs_s3_bucket" {
   }
 }
 
+module "vpc_with_flow_logs_s3_bucket_parquet" {
+  source = "../../"
+
+  name = "vpc-flow-logs-s3-bucket"
+  cidr = "10.30.0.0/16"
+
+  azs            = ["${local.region}a"]
+  public_subnets = ["10.30.101.0/24"]
+
+  enable_flow_log           = true
+  flow_log_destination_type = "s3"
+  flow_log_destination_arn  = module.s3_bucket.this_s3_bucket_arn
+  log_file_format           = "parquet"
+
+  vpc_flow_log_tags = {
+    Name = "vpc-flow-logs-s3-bucket"
+  }
+}
+
 # CloudWatch Log Group and IAM role created automatically
 module "vpc_with_flow_logs_cloudwatch_logs_default" {
   source = "../../"

variables.tf

@@ -1151,3 +1151,26 @@ variable "outpost_az" {
   type        = string
   default     = null
 }
+
+variable "log_file_format" {
+  description = "(Optional) The format for the flow log. Valid values: `plain-text`, `parquet`."
+  type        = string
+  default     = "plain-text"
+  validation {
+    condition = can(regex("^(plain-text|parquet)$",
+    var.log_file_format))
+    error_message = "ERROR valid values: plain-text, parquet."
+  }
+}
+
+variable "hive_compatible_partitions" {
+  description = "(Optional) Indicates whether to use Hive-compatible prefixes for flow logs stored in Amazon S3."
+  type        = bool
+  default     = false
+}
+
+variable "per_hour_partition" {
+  description = "(Optional) Indicates whether to partition the flow log per hour. This reduces the cost and response time for queries."
+  type        = bool
+  default     = false
+}

versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.38"
+      version = ">= 3.63"
     }
   }
 }

vpc-flow-logs.tf

@@ -23,6 +23,11 @@ resource "aws_flow_log" "this" {
   traffic_type             = var.flow_log_traffic_type
   vpc_id                   = local.vpc_id
   max_aggregation_interval = var.flow_log_max_aggregation_interval
+  destination_options {
+    file_format                = var.log_file_format
+    hive_compatible_partitions = var.hive_compatible_partitions
+    per_hour_partition         = var.per_hour_partition
+  }
 
   tags = merge(var.tags, var.vpc_flow_log_tags)
 }