Deploy elasticsearch with terraform

environment

@@ -71,12 +71,16 @@ DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_ARRAY=(
 DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS=${DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_ARRAY[*]}
 DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_TEST="human-cell-atlas-travis-test.iam.gserviceaccount.com"
 DSS_ES_DOMAIN="dss-index-$DSS_DEPLOYMENT_STAGE"
+DSS_ES_INSTANCE_TYPE="t2.small.elasticsearch"
+DSS_ES_VOLUME_SIZE="10"
+DSS_ES_INSTANCE_COUNT="1"
 PYTHONWARNINGS=ignore:ResourceWarning,ignore::UserWarning:zipfile:
 DSS_SECRETS_STORE="dcp/dss"
 EVENT_RELAY_AWS_USERNAME="dss-event-relay-${DSS_DEPLOYMENT_STAGE}"
 EVENT_RELAY_AWS_ACCESS_KEY_SECRETS_NAME="event-relay-user-aws-access-key"
 GOOGLE_APPLICATION_CREDENTIALS_SECRETS_NAME="gcp-credentials.json"
 GOOGLE_APPLICATION_SECRETS_SECRETS_NAME="application_secrets.json"
+ES_ALLOWED_SOURCE_IP_SECRETS_NAME="es_source_ip"
 
 # `{account_id}`, if present, will be replaced with the account ID associated with the AWS credentials used for
 # deployment. It can be safely omitted.

infra/build_deploy_config.py

@@ -59,13 +59,27 @@
     "DSS_GS_CHECKOUT_BUCKET_TEST",
     "DSS_GS_CHECKOUT_BUCKET_STAGING",
     "DSS_GS_CHECKOUT_BUCKET_PROD",
+    "DSS_ES_DOMAIN",
+    "DSS_ES_VOLUME_SIZE",
+    "DSS_ES_INSTANCE_TYPE",
+    "DSS_ES_INSTANCE_COUNT",
 ]
 
 terraform_variable_info = {'variable': dict()}
 for key in env_vars_to_infra:
     terraform_variable_info['variable'][key] = {
         'default': os.environ[key]
     }
+ip_list = boto3.client("secretsmanager").get_secret_value(
+    SecretId="{}/{}/{}".format(
+        os.environ['DSS_SECRETS_STORE'],
+        os.environ['DSS_DEPLOYMENT_STAGE'],
+        os.environ['ES_ALLOWED_SOURCE_IP_SECRETS_NAME'],
+    )
+)['SecretString'].strip()
+terraform_variable_info['variable']['es_source_ip'] = {
+    'default': json.loads(ip_list)
+}
 
 for comp in glob.glob(os.path.join(infra_root, "*/")):
     if 0 == len(glob.glob(os.path.join(infra_root, comp, "*.tf"))):

infra/elasticsearch/elasticsearch.tf

@@ -0,0 +1,102 @@
+data aws_caller_identity current {}
+data aws_region current {}
+
+
+resource "aws_cloudwatch_log_group" "dss-index-log" {
+  name = "${var.DSS_ES_DOMAIN}-index-logs"
+}
+
+
+resource "aws_cloudwatch_log_group" "dss-search-log" {
+  name = "${var.DSS_ES_DOMAIN}-search-logs"
+}
+
+
+data "aws_iam_policy_document" "dss_es_cloudwatch_policy_document" {
+  statement {
+    principals {
+      type = "Service"
+      identifiers = ["es.amazonaws.com"]
+    }
+    actions = [
+      "logs:PutLogEvents",
+      "logs:CreateLogStream"
+    ]
+    resources = [
+      "${aws_cloudwatch_log_group.dss-index-log.arn}",
+      "${aws_cloudwatch_log_group.dss-search-log.arn}"
+    ]
+  }
+}
+
+
+resource "aws_cloudwatch_log_resource_policy" "dss-es-cloudwatch-policy" {
+  policy_name = "dss-es-cloudwatch-log-policy"
+  policy_document = "${data.aws_iam_policy_document.dss_es_cloudwatch_policy_document.json}"
+}
+
+
+data "aws_iam_policy_document" "dss_es_access_policy_documennt" {
+  statement {
+    principals {
+      type = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+    actions = ["es:*"]
+    resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${var.DSS_ES_DOMAIN}/*"]
+  }
+
+  statement {
+    principals {
+      type = "AWS"
+      identifiers = ["*"]
+    }
+    actions = ["es:*"]
+    resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${var.DSS_ES_DOMAIN}/*"]
+    condition {
+      test = "IpAddress"
+      variable = "aws:SourceIp"
+      values = "${var.es_source_ip}"
+    }
+  }
+}
+
+
+resource aws_elasticsearch_domain elasticsearch {
+  count = "${length(var.DSS_ES_DOMAIN) > 0 ? 1 : 0}"
+  domain_name = "${var.DSS_ES_DOMAIN}"
+  elasticsearch_version = "5.5"
+
+  cluster_config = {
+    instance_type = "${var.DSS_ES_INSTANCE_TYPE}"
+	instance_count = "${var.DSS_ES_INSTANCE_COUNT}"
+  }
+
+  advanced_options = {
+    rest.action.multi.allow_explicit_index = "true"
+  }
+
+  ebs_options = {
+    ebs_enabled = "true"
+    volume_type = "gp2"
+    volume_size = "${var.DSS_ES_VOLUME_SIZE}"
+  }
+
+  log_publishing_options = {
+    cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.dss-index-log.arn}"
+    log_type = "INDEX_SLOW_LOGS"
+    enabled = "true"
+  }
+
+  log_publishing_options = {
+    cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.dss-search-log.arn}"
+    log_type = "SEARCH_SLOW_LOGS"
+    enabled = "true"
+  }
+
+  snapshot_options = {
+    automated_snapshot_start_hour = 23
+  }
+
+  access_policies = "${data.aws_iam_policy_document.dss_es_access_policy_documennt.json}"
+}