Initial commit

.editorconfig

@@ -0,0 +1,37 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+max_line_length = 80
+trim_trailing_whitespace = true
+
+[*.{tf,tfvars,hcl,tpl}]
+indent_size = 2
+indent_style = space
+
+[*.md]
+max_line_length = 0
+trim_trailing_whitespace = false
+
+[*.js]
+indent_size = 2
+indent_style = space
+
+[*.conf]
+indent_size = 2
+indent_style = space
+
+[Makefile]
+indent_style = tab
+indent_size = 2
+
+[*.mk]
+indent_style = tab
+indent_size = 2
+
+[COMMIT_EDITMSG]
+max_line_length = 0

.envrc

@@ -0,0 +1 @@
+.envrc.local

.envrc.local

@@ -0,0 +1,15 @@
+export PROJECT_NAME=ziwon
+export APP_NAME=fluentd-aggregator
+
+export STACK_NAME=swarm
+export SWARM_MASTER=node-m1
+
+export ES_TAG=7.6.2
+export ES_JVM_HEAP=512m
+
+export ES_HOST=elasticsearch
+export ES_PORT=9200
+export ES_USER=elastic
+export ES_PASSWORD=changeme
+export ES_SCHEME=http
+export ES_SSL_VERIFY=false

.envrc.prod

@@ -0,0 +1,17 @@
+export PROJECT_NAME=ziwon
+export APP_NAME=fluentd-aggregator
+
+export AWS_ACCOUNT_ID=57xxxxxxxxxx
+export AWS_REGION=ap-northeast-2
+export AWS_PROFILE=default
+export VPC_ID=vpc-xxxxxxxxx
+
+export ECR_IMAGE_URL=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${APP_NAME}
+export TAG=latest
+
+export ES_HOST=elastic.ap-northeast-1.aws.found.io
+export ES_PORT=9243
+export ES_USER=elastic
+export ES_PASSWORD=changeme
+export ES_SCHEME=https
+export ES_SSL_VERIFY=true

.gitignore

@@ -0,0 +1,38 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
+# .tfvars files are managed as part of configuration and so should be included in
+# version control.
+#
+# example.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+# configs generated with makefile
+*/config/*[a-z.].tflint
+*/config/*[a-z_].hcl
+Brewfile.lock.json
+*.zip

.labels

@@ -0,0 +1,3 @@
+docker node update --label-add worker=01 node-w1
+docker node update --label-add worker=02 node-w2
+docker node update --label-add worker=03 node-w3

.virtualboxrc

@@ -0,0 +1,8 @@
+STACK_NAME=swarm
+
+# Settings for VM variables
+SWARM_MASTER=node-m1
+MANAGERS=2
+WORKERS=4
+VBOX_MEMORY=2048
+VBOX_CPUS=2

Dockerfile

@@ -0,0 +1,17 @@
+FROM fluent/fluentd:v1.10-debian
+
+USER root
+RUN apt-get update \
+    && apt-get upgrade -y -qq \
+    && apt-get install \
+        -y --no-install-recommends net-tools curl \
+    && gem install \
+        fluent-plugin-elasticsearch \
+        fluent-plugin-s3 \
+        fluent-plugin-record-modifier \
+        fluent-plugin-rewrite-tag-filter \
+        fluent-plugin-prometheus \
+    && gem sources --clear-all
+
+COPY conf/ /fluentd/etc/
+COPY entrypoint.sh /bin/

Makefile

@@ -0,0 +1,24 @@
+HOME_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+.SILENT: ; # no need for @
+include .envrc
+export
+
+include $(HOME_DIR)/swarm.mk
+include $(HOME_DIR)/docker.mk
+
+PHONY: envrc-local
+envrc-local: # Change environment to local development: # make envrc-local
+	ln -fs .envrc.local .envrc
+	direnv allow
+
+PHONY: envrc-prod
+envrc-prod: # Change environment to prod development: # make envrc-prod
+	ln -fs .envrc.prod .envrc
+	direnv allow
+
+PHONY: help
+help: # Show this help message: # make help
+	echo "Usage: make [command] [args]"
+	grep -E '^[a-zA-Z_-]+:.*?# .*$$' $(MAKEFILE_LIST) | sort | sed -e 's/: \([a-z\.\-][^ ]*\) #/: #/g' | awk 'BEGIN {FS = ": # "}; {printf "\t\033[36m%-32s\033[0m \033[33m%-45s\033[0m (e.g. \033[32m%s\033[0m)\n", $$1, $$2, $$3}'
+
+.DEFAULT_GOAL := help

README.md

@@ -0,0 +1,194 @@
+# terraform-fluentd-aggregator
+
+This repo was created for the purpose of storing logs collected from multiple swarm clusters on Elastic Search.
+
+You may utilize my [fluent-bit-docker-metadata](https://github.com/ziwon/fluent-bit-docker-metadata) to ship logs to the aggregator from edge swarm cluster.
+
+## Usage
+
+Fluentd Aggregator runs as one ECS Fargate application, is deployed in a blue-green deployment, and routes the traffic load to the Network Load Balancer.
+
+```
+module "app" {
+  source = "./modules/app"
+
+  internal_load_balancer = false
+
+  vpc_id                = var.vpc_id
+  app                   = local.app_id
+  region                = var.region
+  environment           = var.environment
+  service_replicas      = var.service_replicas
+  task_definition       = var.task_definition
+  pipeline_image_tag    = var.pipeline_image_tag
+  primary_domain        = var.primary_domain
+  dns_name              = "${var.project}-${var.app}.${var.primary_domain}"
+  certificate_arn       = var.certificate_arn
+  kms_arn               = local.kms_arn
+  repository_name       = var.repository_name
+  enable_cpu_high_alarm = true
+  enable_cpu_low_alarm  = false
+
+  tags = merge(
+    map("Last Updated", "${module.global.build_time}"),
+    var.tags
+  )
+}
+
+```
+
+## Fluentd Configuration
+
+As shown in the following configuration file, the collected logs are dynamically indexed to ElasticSearch according to the log properties using `record_transformer` filter and `elasticsearch dynamic` plugin. e.g.) `shinhan-aiagent`-2020.08.15
+
+```
+<filter docker.**>
+  @type record_transformer
+  <record>
+    service_name ${tag_parts[1]}
+  </record>
+</filter>
+
+<match docker.**>
+  @type elasticsearch_dynamic
+
+  host "#{ENV['ES_HOST'] || 'elasticsearch'}"
+  port "#{ENV['ES_PORT'] || '9200'}"
+  user "#{ENV['ES_USER'] || 'elastic'}"
+  password "#{ENV['ES_PASSWORD'] || 'changeme'}"
+  scheme "#{ENV['ES_SCHEME'] || 'http'}"
+  ssl_verify "#{ENV['ES_SSL_VERIFY'] || 'false'}"
+  ssl_version TLSv1_2
+
+  logstash_format true
+  logstash_dateformat %Y.%m.%d
+  logstash_prefix ${record['biz_client']}-${record['service_name']}
+
+  ...
+```
+
+## Local Swarm Cluster
+
+On your mac, you can test the following pipeline by deploying a swarm cluster of four docker machines. Note that ES in your swarm dose run as standalone for testing.
+
+```
+fluent bit -> fluentd -> elastic search -> kibana
+```
+
+To deploy a swarm cluster:
+
+```
+$ make node-up
+```
+
+Build fluent-aggregator in docker-machine:
+
+```
+$ eval $(docker-machine env node-m1)
+$ make docker-build
+```
+
+To start services:
+
+```
+$ make stack-start
+```
+
+You can access kibana with `http://192.168.128.101` since the host IP is always keep the same values even if the docker machine is deployed multiple times.
+
+To change a certification with a customized domain:
+
+```
+$ DOMAIN=awesome.domain make node-gen-cert
+```
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | ~> 0.12.0 |
+| aws | ~> 2.57.0 |
+| null | ~> 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | ~> 2.57.0 |
+| null | ~> 2.0 |
+
+## Environment Variables
+
+We create and manage per-project isolated development environments according to the **12factor** using **`direnv`**
+
+Chanege to local env:
+
+```
+$ make envrc-local
+```
+
+Change to prod env:
+
+```
+$ make envrc-prod
+```
+
+You may need to set up the following values in production with AWS Cloud and Elastic Cloud:
+
+```
+export PROJECT_NAME=ziwon
+export APP_NAME=fluentd-aggregator
+
+export AWS_ACCOUNT_ID=57xxxxxxxxxx
+export AWS_REGION=ap-northeast-2
+export AWS_PROFILE=default
+export VPC_ID=vpc-xxxxxxxxx
+
+export ECR_IMAGE_URL=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${APP_NAME}
+export TAG=latest
+
+export ES_HOST=elastic.ap-northeast-1.aws.found.io
+export ES_PORT=9243
+export ES_USER=elastic
+export ES_PASSWORD=changeme
+export ES_SCHEME=https
+export ES_SSL_VERIFY=true
+```
+
+## Makefile Targets
+
+```
+$ make
+Usage: make [command] [args]
+        Makefile:envrc-local             Change environment to local development       (e.g. make envrc-local)
+        Makefile:envrc-prod              Change environment to prod development        (e.g. make envrc-prod)
+        Makefile:help                    Show this help message                        (e.g. make help)
+        docker.mk:docker-build           Build docker image                            (e.g. make docker-build)
+        docker.mk:docker-commit          Commit current container using killed tag     (e.g. make docker-commit)
+        docker.mk:docker-history         Show the history of an image                  (e.g. make docker-history)
+        docker.mk:docker-push            Push an image to Amazon ECR registry          (e.g. make docker-push)
+        docker.mk:docker-run             Run a command in a new container              (e.g. make docker-run)
+        swarm.mk:node-add-cert           Add certification into the local machine      (e.g. make node-add-cert)
+        swarm.mk:node-cleanup            Clean up the docker volume                    (e.g. make node-cleanup)
+        swarm.mk:node-down               Terminate swarm nodes                         (e.g. make node-down)
+        swarm.mk:node-gen-cert           Generate SSL certification                    (e.g. make node-gen-cert)
+        swarm.mk:node-ip                 Show the address of node                      (e.g. make node-ip)
+        swarm.mk:node-list               Show node list                                (e.g. make node-list)
+        swarm.mk:node-up                 Bootstrap swarm nodes                         (e.g. make node-up)
+        swarm.mk:stack-exec              Get executed the given command into container (e.g. make stack-exec sh)
+        swarm.mk:stack-logs              Show logs from the service                    (e.g. make stack-logs elasticsearch)
+        swarm.mk:stack-ps                List stack process                            (e.g. make stack-ps)
+        swarm.mk:stack-reload            Reload the stack from swarm                   (e.g. make stack-reload)
+        swarm.mk:stack-service           List stack services                           (e.g. make stack-service)
+        swarm.mk:stack-start             Start the stack onto swarm                    (e.g. make stack-start)
+        swarm.mk:stack-stop              Remove the stack from swarm                   (e.g. make stack-stop)
+        swarm.mk:stack-viz               Visualize the swarm stack                     (e.g. make stack-viz)
+```
+
+## Inputs
+(TBD)
+
+
+
+## Outputs
+(TBD)

certs/ca/ca-cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRDCCAiygAwIBAgIUNHKf5dojgXHfPqov7icDW72WmLQwDQYJKoZIhvcNAQEL
+BQAwFzEVMBMGA1UEAwwMZG9ja2VyLmxvY2FsMB4XDTIwMDgxNzA1NDEzNVoXDTIx
+MDgxNzA1NDEzNVowFzEVMBMGA1UEAwwMZG9ja2VyLmxvY2FsMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxAsReH/WByqeWqoC3L8cIMZdiRhW+0nIEa/+
+HR7KB4JaYHQpsJ0B3aywGIqsmUnw7JwMM07KqXUg8zu+WpV2XzKjH+33xLR8s2j/
+478+FXA5pzcPr07ZOV7/z2njRmgTOwlNd1MGEFVIg7t4w5P6iRGe9Dm2W7levg1o
+73Bj/ykWFg+kcI9CsJEYZ83yMIFezpBdHmpehHviX4XC2dW4w3AvwwxaQd6RPdi1
+LaGemDefhZLV05by8YOHS3i373L307ZAjrlKjqYTXAXNF78euooaJNT87bZGbBnz
+ltaGNPqwkG7fSLJQfUblc+4f2tI9+E3ZX/UfSBBwFCURhzH5NQIDAQABo4GHMIGE
+MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCsVGklVTvZ9mDK7gs1unLKKDm3H
+MFIGA1UdIwRLMEmAFCsVGklVTvZ9mDK7gs1unLKKDm3HoRukGTAXMRUwEwYDVQQD
+DAxkb2NrZXIubG9jYWyCFDRyn+XaI4Fx3z6qL+4nA1u9lpi0MA0GCSqGSIb3DQEB
+CwUAA4IBAQClOHshvVAytIhkHmY8hFyaFlGpt02oKi7tsSpcQJgEb5XkYQh64z1I
+5ph1FSjurJIbuzRp96c1yXpyY2qYERqkKD6yntKFND+PJFmMHo7pnDq2yh8jmVoP
+NAw+1iuF9QTI1x5TE2lSoqiYu5as+9CvCND6oRTs81UXNzNK0yl78wogTuQv3YA/
+OxepDwosqP6/VeUmHlvd+f8RyCw7fpGdJMJKRzMx+NvJC45XiKfEbxKqFAl0WhWe
+ND4+ABdDztV9s2FQVFlcs3k2F1ypILjUiqtAHZkLZ2euKt6p/YkXUdVWH4zLe9c9
+in8HGjsuYDWcuzOUisi4j3Dtf/OFDwdS
+-----END CERTIFICATE-----

certs/ca/ca-cert.srl

@@ -0,0 +1 @@
+35D50FA1C260C2D31BB63AD0CC71A0F5BAC59936