diff --git a/environment b/environment index 5ae6813d65..5a2a7b2328 100644 --- a/environment +++ b/environment @@ -71,12 +71,16 @@ DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_ARRAY=( DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS=${DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_ARRAY[*]} DSS_SUBSCRIPTION_AUTHORIZED_DOMAINS_TEST="human-cell-atlas-travis-test.iam.gserviceaccount.com" DSS_ES_DOMAIN="dss-index-$DSS_DEPLOYMENT_STAGE" +DSS_ES_INSTANCE_TYPE="t2.small.elasticsearch" +DSS_ES_VOLUME_SIZE="10" +DSS_ES_INSTANCE_COUNT="1" PYTHONWARNINGS=ignore:ResourceWarning,ignore::UserWarning:zipfile: DSS_SECRETS_STORE="dcp/dss" EVENT_RELAY_AWS_USERNAME="dss-event-relay-${DSS_DEPLOYMENT_STAGE}" EVENT_RELAY_AWS_ACCESS_KEY_SECRETS_NAME="event-relay-user-aws-access-key" GOOGLE_APPLICATION_CREDENTIALS_SECRETS_NAME="gcp-credentials.json" GOOGLE_APPLICATION_SECRETS_SECRETS_NAME="application_secrets.json" +ES_ALLOWED_SOURCE_IP_SECRETS_NAME="es_source_ip" # `{account_id}`, if present, will be replaced with the account ID associated with the AWS credentials used for # deployment. It can be safely omitted. diff --git a/infra/build_deploy_config.py b/infra/build_deploy_config.py index b80a8bb430..db0e9ad18b 100755 --- a/infra/build_deploy_config.py +++ b/infra/build_deploy_config.py @@ -59,6 +59,10 @@ "DSS_GS_CHECKOUT_BUCKET_TEST", "DSS_GS_CHECKOUT_BUCKET_STAGING", "DSS_GS_CHECKOUT_BUCKET_PROD", + "DSS_ES_DOMAIN", + "DSS_ES_VOLUME_SIZE", + "DSS_ES_INSTANCE_TYPE", + "DSS_ES_INSTANCE_COUNT", ] terraform_variable_info = {'variable': dict()} @@ -66,6 +70,16 @@ terraform_variable_info['variable'][key] = { 'default': os.environ[key] } +ip_list = boto3.client("secretsmanager").get_secret_value( + SecretId="{}/{}/{}".format( + os.environ['DSS_SECRETS_STORE'], + os.environ['DSS_DEPLOYMENT_STAGE'], + os.environ['ES_ALLOWED_SOURCE_IP_SECRETS_NAME'], + ) +)['SecretString'].strip() +terraform_variable_info['variable']['es_source_ip'] = { + 'default': json.loads(ip_list) +} for comp in glob.glob(os.path.join(infra_root, "*/")): if 0 == len(glob.glob(os.path.join(infra_root, comp, "*.tf"))): diff --git a/infra/elasticsearch/elasticsearch.tf b/infra/elasticsearch/elasticsearch.tf new file mode 100644 index 0000000000..d7c8648eca --- /dev/null +++ b/infra/elasticsearch/elasticsearch.tf @@ -0,0 +1,102 @@ +data aws_caller_identity current {} +data aws_region current {} + + +resource "aws_cloudwatch_log_group" "dss-index-log" { + name = "${var.DSS_ES_DOMAIN}-index-logs" +} + + +resource "aws_cloudwatch_log_group" "dss-search-log" { + name = "${var.DSS_ES_DOMAIN}-search-logs" +} + + +data "aws_iam_policy_document" "dss_es_cloudwatch_policy_document" { + statement { + principals { + type = "Service" + identifiers = ["es.amazonaws.com"] + } + actions = [ + "logs:PutLogEvents", + "logs:CreateLogStream" + ] + resources = [ + "${aws_cloudwatch_log_group.dss-index-log.arn}", + "${aws_cloudwatch_log_group.dss-search-log.arn}" + ] + } +} + + +resource "aws_cloudwatch_log_resource_policy" "dss-es-cloudwatch-policy" { + policy_name = "dss-es-cloudwatch-log-policy" + policy_document = "${data.aws_iam_policy_document.dss_es_cloudwatch_policy_document.json}" +} + + +data "aws_iam_policy_document" "dss_es_access_policy_documennt" { + statement { + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + actions = ["es:*"] + resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${var.DSS_ES_DOMAIN}/*"] + } + + statement { + principals { + type = "AWS" + identifiers = ["*"] + } + actions = ["es:*"] + resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${var.DSS_ES_DOMAIN}/*"] + condition { + test = "IpAddress" + variable = "aws:SourceIp" + values = "${var.es_source_ip}" + } + } +} + + +resource aws_elasticsearch_domain elasticsearch { + count = "${length(var.DSS_ES_DOMAIN) > 0 ? 1 : 0}" + domain_name = "${var.DSS_ES_DOMAIN}" + elasticsearch_version = "5.5" + + cluster_config = { + instance_type = "${var.DSS_ES_INSTANCE_TYPE}" + instance_count = "${var.DSS_ES_INSTANCE_COUNT}" + } + + advanced_options = { + rest.action.multi.allow_explicit_index = "true" + } + + ebs_options = { + ebs_enabled = "true" + volume_type = "gp2" + volume_size = "${var.DSS_ES_VOLUME_SIZE}" + } + + log_publishing_options = { + cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.dss-index-log.arn}" + log_type = "INDEX_SLOW_LOGS" + enabled = "true" + } + + log_publishing_options = { + cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.dss-search-log.arn}" + log_type = "SEARCH_SLOW_LOGS" + enabled = "true" + } + + snapshot_options = { + automated_snapshot_start_hour = 23 + } + + access_policies = "${data.aws_iam_policy_document.dss_es_access_policy_documennt.json}" +}