Skip to content

Commit

Permalink
terraform cleanup: tags, formatting, better defaults (#1080)
Browse files Browse the repository at this point in the history 
Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
  • Loading branch information
@oavdeev
oavdeev authored Oct 21, 2020
1 parent fc361af commit 54b0104
Show file tree
Hide file tree
Showing 8 changed files with 130 additions and 111 deletions.
26 changes: 13 additions & 13 deletions infra/terraform/aws/eks.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ resource "random_string" "suffix" {
resource "aws_security_group" "all_worker_mgmt" {
name_prefix = "${var.name_prefix}-worker"
vpc_id = module.vpc.vpc_id

tags = var.tags
}

module "vpc" {
Expand All @@ -78,40 +80,38 @@ module "vpc" {
"kubernetes.io/cluster/${local.cluster_name}" = "shared"
"kubernetes.io/role/internal-elb" = "1"
}

tags = var.tags
}

module "eks" {
source = "terraform-aws-modules/eks/aws"
source = "terraform-aws-modules/eks/aws"
version = "12.2.0"

cluster_name = local.cluster_name
cluster_version = "1.17"
subnets = module.vpc.private_subnets

tags = {
Environment = "test"
GithubRepo = "terraform-aws-eks"
GithubOrg = "terraform-aws-modules"
}
tags = var.tags

vpc_id = module.vpc.vpc_id

worker_groups = [
{
name = "worker-group-1"
instance_type = "r3.large"
asg_desired_capacity = 2
name = "worker-group-1"
instance_type = "r3.large"
asg_desired_capacity = 2
},
{
name = "worker-group-2"
instance_type = "r3.large"
asg_desired_capacity = 1
name = "worker-group-2"
instance_type = "r3.large"
asg_desired_capacity = 1
},
]

worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id]
map_roles = var.map_roles
map_accounts = var.map_accounts

workers_additional_policies = [aws_iam_policy.worker_policy.id]
workers_additional_policies = [aws_iam_policy.worker_policy.id]
}
2 changes: 2 additions & 0 deletions infra/terraform/aws/emr.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,6 @@ resource "aws_emr_cluster" "persistent_cluster" {
step_concurrency_level = 256

log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/"

tags = var.tags
}
124 changes: 62 additions & 62 deletions infra/terraform/aws/helm.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,81 +12,81 @@ provider "helm" {
# RDS password is stored in a configmap which is not awesome but that RDS instance is not routable
# from the outside anyways so that'll do.
locals {
feast_core_config = {
redis = {
enabled = false
}
postgresql = {
enabled = false
}
kafka = {
enabled = false
}
feast_core_config = {
redis = {
enabled = false
}
postgresql = {
enabled = false
}
kafka = {
enabled = false
}

"feast-core" = {
"application-generated.yaml" = {
enabled = false
}
"feast-core" = {
"application-generated.yaml" = {
enabled = false
}

"application-override.yaml" = {
spring = {
datasource = {
url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}"
username = "${module.rds_cluster.master_username}"
password = "${random_password.db_password.result}"
}
}
feast = {
stream = {
type = "kafka"
options = {
bootstrapServers = ${aws_msk_cluster.msk.bootstrap_brokers}
topic = "feast"
}
}
}
server = {
port = "8080"
}
"application-override.yaml" = {
spring = {
datasource = {
url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}"
username = "${module.rds_cluster.master_username}"
password = "${random_password.db_password.result}"
}
}
feast = {
stream = {
type = "kafka"
options = {
bootstrapServers = aws_msk_cluster.msk.bootstrap_brokers
topic = "feast"
}
}
}
server = {
port = "8080"
}
}
}

"feast-online-serving" = {
"application-override.yaml" = {
enabled = true
feast = {
stores = [
{
name = "online"
type = "REDIS"
config = {
host = module.redis.endpoint
port = 6379
}
subscriptions = [
{
name= "*"
project= "*"
version= "*"
}
]
}
]
job_store = {
redis_host = module.redis.endpoint
redis_port = 6379
}
"feast-online-serving" = {
"application-override.yaml" = {
enabled = true
feast = {
stores = [
{
name = "online"
type = "REDIS"
config = {
host = module.redis.endpoint
port = 6379
}
subscriptions = [
{
name = "*"
project = "*"
version = "*"
}
]
}
]
job_store = {
redis_host = module.redis.endpoint
redis_port = 6379
}
}
}
}
}
}

resource "helm_release" "feast" {
name = "feast"
chart = "../../charts/feast"
name = "feast"
chart = "../../charts/feast"

wait = false
wait = false

values = [
yamlencode(local.feast_core_config)
Expand Down
13 changes: 8 additions & 5 deletions infra/terraform/aws/msk.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ resource "aws_security_group" "broker" {
vpc_id = module.vpc.vpc_id

ingress {
description = "Allow connections from the worker group"
description = "Allow connections from the worker group"
security_groups = [aws_security_group.all_worker_mgmt.id]
protocol = "tcp"
from_port = 0
to_port = 65535
protocol = "tcp"
from_port = 0
to_port = 65535
}

egress {
Expand All @@ -16,6 +16,8 @@ resource "aws_security_group" "broker" {
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}

tags = var.tags
}

resource "aws_msk_cluster" "msk" {
Expand All @@ -26,7 +28,7 @@ resource "aws_msk_cluster" "msk" {
broker_node_group_info {
instance_type = "kafka.t3.small"
ebs_volume_size = 100
client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]
client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]]
security_groups = [aws_security_group.broker.id]
}

Expand All @@ -46,4 +48,5 @@ resource "aws_msk_cluster" "msk" {
}
}

tags = var.tags
}
18 changes: 10 additions & 8 deletions infra/terraform/aws/online_store.tf
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
module "redis" {
source = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0"
subnets = module.vpc.private_subnets
name = "${var.name_prefix}-online"
vpc_id = module.vpc.vpc_id
allowed_security_groups = [aws_security_group.all_worker_mgmt.id]
availability_zones = module.vpc.azs
}
module "redis" {
source = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0"
subnets = module.vpc.private_subnets
name = "${var.name_prefix}-online"
vpc_id = module.vpc.vpc_id
allowed_security_groups = [aws_security_group.all_worker_mgmt.id]
availability_zones = module.vpc.azs

tags = var.tags
}
36 changes: 19 additions & 17 deletions infra/terraform/aws/rds.tf
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
resource "random_password" "db_password" {
length = 16
special = true
length = 16
special = true
override_special = "!#()-[]<>"
}

module "rds_cluster" {
source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0"
name = "${var.name_prefix}-db"
engine = "aurora-postgresql"
engine_mode = "serverless"
engine_version = "10.7"
cluster_family = "aurora-postgresql10"
cluster_size = 0
admin_user = var.postgres_db_user
admin_password = random_password.db_password.result
db_name = var.postgres_db_name
db_port = 5432
instance_type = "db.t2.small"
vpc_id = module.vpc.vpc_id
security_groups = [aws_security_group.all_worker_mgmt.id]
subnets = module.vpc.private_subnets
source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0"
name = "${var.name_prefix}-db"
engine = "aurora-postgresql"
engine_mode = "serverless"
engine_version = "10.7"
cluster_family = "aurora-postgresql10"
cluster_size = 0
admin_user = var.postgres_db_user
admin_password = random_password.db_password.result
db_name = var.postgres_db_name
db_port = 5432
instance_type = "db.t2.small"
vpc_id = module.vpc.vpc_id
security_groups = [aws_security_group.all_worker_mgmt.id]
subnets = module.vpc.private_subnets

scaling_configuration = [
{
Expand All @@ -30,4 +30,6 @@ module "rds_cluster" {
timeout_action = "ForceApplyCapacityChange"
}
]

tags = var.tags
}
8 changes: 5 additions & 3 deletions infra/terraform/aws/s3.tf
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
resource "random_string" "s3_suffix" {
length = 8
lower = true
upper = false
length = 8
lower = true
upper = false
special = false
}

Expand All @@ -17,4 +17,6 @@ resource "aws_s3_bucket" "feast_bucket" {
}
}
}

tags = var.tags
}
14 changes: 11 additions & 3 deletions infra/terraform/aws/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ variable "name_prefix" {
}

variable "postgres_db_name" {
default = "feast"
default = "feast"
}

variable "postgres_db_user" {
default = "feast"
default = "feast"
}

variable "map_accounts" {
Expand All @@ -34,5 +34,13 @@ variable "map_roles" {
}

variable "use_persistent_emr_cluster" {
default = false
description = "Create a persistent EMR cluster."
default = true
}

variable "tags" {
description = "Tags"
type = map(string)

default = {}
}

0 comments on commit 54b0104

Please sign in to comment.