elb_s3_elasticsearch.conf

# Parse AWS ELB logs stored in an S3 Bucket 

input {
       	s3 {
            	bucket => "bucket.name"
                region => "us-west-2"
                type => "elblogs"
                prefix => "elb.logs"
                sincedb_path => "./last-s3-file"
        }
}


filter {
	if [type] == "elblogs" {
		grok {
			match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb_name} %{IP:elb_client_ip}:%{INT:elb_client_port:int} (?:%{IP:elb_backend_ip}:%{NUMBER:elb_backend_port:int}|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_proce$
			match => ["message", "%{GREEDYDATA:event_name} for ELB: %{NOTSPACE:elb_name} at %{TIMESTAMP_ISO8601:timestamp}"]
		}
		
		if [elb_request] =~ /.+/ {
			grok {
				match => ["elb_request", "(?:%{WORD:http_method}) (?:%{DATA:http_path})? (?:%{DATA:http_type}/%{NUMBER:http_version:float})?|%{GREEDYDATA:rawrequest}"]
			}
		}

		if [http_path] =~ /.+/ {
			grok {
				match => ["http_path", "(?:%{WORD:http_path_protocol}://)?(%{NOTSPACE:http_path_site}:)?(?:%{NUMBER:http_path_port:int})?(?:%{GREEDYDATA:http_path_url})?"]
			}
		}
		
		geoip {
			source => "elb_client_ip"
		}
	}

	date {
		match => [ "timestamp", "ISO8601" ]
	}

	useragent {
		source => "userAgent"
		prefix => "browser_"
	}
}

output {
  elasticsearch {
        hosts => ["https://endpoint:port"]
        user =>"username"
        password =>"password"
        action => "index"
        index => "elb-%{+YYYY.MM.dd}"
  }
}