From 502012987c08676c770075278a8f590b8a4998f4 Mon Sep 17 00:00:00 2001 From: Michael Cayanan <42812746+mcayanan@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:37:14 -0800 Subject: [PATCH 1/3] HC-447: Added capability to rollover old job_status indices in Mozart ES (#94) * HC-447: Add mechansim to roll of ES indices on Mozart * install templates and bootstrap * add new files * cannot resolve role * fix base path * update * modify to modern template payload * update * add comma * dump json * fix typo * updates * typo * update * make index pattern more specific * remove alias * update * revert * update * update * use the phases properly * updated * update * update * update * add actions * update * update * update policy * update names to be consistent throughout * revert * revert back aliases * update * typo * remove ILM from event, task, and worker templates * remove alias * revert back to default 8 shards * No need to push index templates out as they are initialized elsewhere * add aliases * update * update description * bump version * make a minor update --------- Co-authored-by: Mike Cayanan --- sdscli/__init__.py | 2 +- sdscli/adapters/hysds/fabfile.py | 49 +- .../hysds/files/es_ilm_policy_mozart.json | 43 ++ .../hysds/files/event_status.template | 97 +++ .../adapters/hysds/files/job_status.template | 711 ++++++++++++++++++ .../adapters/hysds/files/task_status.template | 147 ++++ .../hysds/files/worker_status.template | 147 ++++ sdscli/adapters/hysds/update.py | 10 + 8 files changed, 1189 insertions(+), 17 deletions(-) create mode 100644 sdscli/adapters/hysds/files/es_ilm_policy_mozart.json create mode 100644 sdscli/adapters/hysds/files/event_status.template create mode 100644 sdscli/adapters/hysds/files/job_status.template create mode 100644 sdscli/adapters/hysds/files/task_status.template create mode 100644 sdscli/adapters/hysds/files/worker_status.template diff --git a/sdscli/__init__.py b/sdscli/__init__.py index 2df6519..4b4c139 100644 --- a/sdscli/__init__.py +++ b/sdscli/__init__.py @@ -3,6 +3,6 @@ from __future__ import division from __future__ import absolute_import -__version__ = "1.1.7" +__version__ = "1.2.0" __url__ = "https://github.com/sdskit/sdscli" __description__ = "Command line interface for SDSKit" diff --git a/sdscli/adapters/hysds/fabfile.py b/sdscli/adapters/hysds/fabfile.py index 883e717..6581d86 100644 --- a/sdscli/adapters/hysds/fabfile.py +++ b/sdscli/adapters/hysds/fabfile.py @@ -479,6 +479,39 @@ def install_base_es_template(): run("curl -XPUT 'localhost:9200/_template/index_defaults?pretty' -H 'Content-Type: application/json' -d@/tmp/es_template-base.json") +def install_es_policy(): + policy_file_name = "es_ilm_policy_mozart.json" + target_file = f"{ops_dir}/mozart/etc/{policy_file_name}" + send_template( + policy_file_name, + target_file + ) + run(f"curl -XPUT 'localhost:9200/_ilm/policy/ilm_policy_mozart?pretty' -H 'Content-Type: application/json' -d@{target_file}") + +def install_mozart_es_templates(): + # install index templates + # Only job_status.template has ILM policy attached + # HC-451 will focus on adding ILM to worker, task, and event status indices + templates = [ + "job_status.template", + "worker_status.template", + "task_status.template", + "event_status.template" + ] + + for template in templates: + # Copy templates to etc/ directory + target_path = f"{ops_dir}/mozart/etc/{template}" + send_template( + template, + target_path + ) + template_doc_name = template.split(".template")[0] + print(f"Creating ES index template for {template}") + run(f"curl -XPUT 'localhost:9200/_index_template/{template_doc_name}?pretty' " + f"-H 'Content-Type: application/json' -d@{target_path}") + + ########################## # grq functions ########################## @@ -814,14 +847,6 @@ def send_shipper_conf(node_type, log_dir, cluster_jobs, redis_ip_job_status, if node_type == 'mozart': upload_template('indexer.conf.mozart', '~/mozart/etc/indexer.conf', use_jinja=True, context=ctx, template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('job_status.template', '~/mozart/etc/job_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('worker_status.template', '~/mozart/etc/worker_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('task_status.template', '~/mozart/etc/task_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('event_status.template', '~/mozart/etc/event_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) upload_template('sdswatch_client.conf', '~/mozart/etc/sdswatch_client.conf', use_jinja=True, context=ctx, template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) send_template("run_sdswatch_client.sh", "~/mozart/bin/run_sdswatch_client.sh") @@ -833,14 +858,6 @@ def send_shipper_conf(node_type, log_dir, cluster_jobs, redis_ip_job_status, elif node_type == 'metrics': upload_template('indexer.conf.metrics', '~/metrics/etc/indexer.conf', use_jinja=True, context=ctx, template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('job_status.template', '~/metrics/etc/job_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('worker_status.template', '~/metrics/etc/worker_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('task_status.template', '~/metrics/etc/task_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) - upload_template('event_status.template', '~/metrics/etc/event_status.template', use_jinja=True, - template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) upload_template('sdswatch_client.conf', '~/metrics/etc/sdswatch_client.conf', use_jinja=True, context=ctx, template_dir=os.path.join(ops_dir, 'mozart/ops/hysds/configs/logstash')) send_template("run_sdswatch_client.sh", "~/metrics/bin/run_sdswatch_client.sh") diff --git a/sdscli/adapters/hysds/files/es_ilm_policy_mozart.json b/sdscli/adapters/hysds/files/es_ilm_policy_mozart.json new file mode 100644 index 0000000..972f2d6 --- /dev/null +++ b/sdscli/adapters/hysds/files/es_ilm_policy_mozart.json @@ -0,0 +1,43 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "set_priority" : { + "priority": 100 + } + } + }, + "warm": { + "min_age": "90d", + "actions": { + "migrate": { + "enabled": false + }, + "set_priority" : { + "priority": 50 + } + } + }, + "cold": { + "min_age": "97d", + "actions": { + "set_priority" : { + "priority": 0 + }, + "migrate": { + "enabled": false + }, + "freeze": {} + } + }, + "delete": { + "min_age": "104d", + "actions": { + "delete": {} + } + } + } + } +} diff --git a/sdscli/adapters/hysds/files/event_status.template b/sdscli/adapters/hysds/files/event_status.template new file mode 100644 index 0000000..2ee29aa --- /dev/null +++ b/sdscli/adapters/hysds/files/event_status.template @@ -0,0 +1,97 @@ +{ + "index_patterns": [ + "event_status*" + ], + "template": { + "mappings": { + "properties": { + "resource": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "status": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "uuid": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "copy_to": [ + "text_fields" + ] + }, + "event": { + "type": "object", + "enabled": false + }, + "text_fields": { + "type": "text" + } + } + }, + "settings": { + "number_of_shards": 8, + "index": { + "refresh_interval": "5s" + }, + "analysis": { + "analyzer": { + "default": { + "filter": [ + "lowercase", + "word_delimiter" + ], + "tokenizer": "keyword" + } + } + } + }, + "aliases": { + "job_status": {} + } + } +} diff --git a/sdscli/adapters/hysds/files/job_status.template b/sdscli/adapters/hysds/files/job_status.template new file mode 100644 index 0000000..4d63eb3 --- /dev/null +++ b/sdscli/adapters/hysds/files/job_status.template @@ -0,0 +1,711 @@ +{ + "index_patterns": [ + "job_status*" + ], + "template": { + "mappings": { + "properties": { + "resource": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "status": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "timestamp": { + "type": "date" + }, + "context": { + "type": "object", + "enabled": false + }, + "msg": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "msg_details": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job": { + "properties": { + "command": { + "type": "object", + "enabled": false + }, + "container_image_name": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "container_image_url": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "context": { + "type": "object", + "enabled": false + }, + "priority": { + "type": "integer" + }, + "delivery_info": { + "properties": { + "priority": { + "type": "integer" + }, + "routing_key": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "exchange": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "redelivered": { + "type": "boolean" + } + } + }, + "job_info": { + "properties": { + "context": { + "type": "object", + "enabled": false + }, + "payload": { + "type": "object", + "enabled": false + }, + "payload_hash": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dedup": { + "type": "boolean" + }, + "dedup_job": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_queue": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "status": { + "type": "integer" + }, + "time_start": { + "type": "date" + }, + "error_queue": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "completed_queue": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "execute_node": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "time_queued": { + "type": "date" + }, + "index": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_status_exchange": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_dir": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "time_end": { + "type": "date" + }, + "cmd_start": { + "type": "date" + }, + "cmd_end": { + "type": "date" + }, + "cmd_duration": { + "type": "double" + }, + "public_ip": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "facts": { + "properties": { + "swapsize": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "ec2_instance_type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "is_virtual": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "architecture": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "ec2_placement_availability_zone": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "memorytotal": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "virtual": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "physicalprocessorcount": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "ec2_ami_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "processorcount": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + } + } + }, + "host": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_url": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "metrics": { + "properties": { + "inputs_localized": { + "properties": { + "disk_usage": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "time_start": { + "type": "date" + }, + "url": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "time_end": { + "type": "date" + }, + "duration": { + "type": "double" + }, + "transfer_rate": { + "type": "double" + }, + "path": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + } + } + }, + "products_staged": { + "properties": { + "path": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "disk_usage": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "time_start": { + "type": "date" + }, + "time_end": { + "type": "date" + }, + "duration": { + "type": "double" + }, + "transfer_rate": { + "type": "double" + }, + "id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "urls": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "browse_urls": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dataset": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "ipath": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "system_version": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dataset_level": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dataset_type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + } + } + }, + "job_dir_size": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "product_provenance": { + "properties": { + "product_type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "acquisition_start_time": { + "type": "date" + }, + "source_production_time": { + "type": "date" + }, + "availability_time": { + "type": "date" + }, + "processing_start_time": { + "type": "date" + }, + "ground_system_latency": { + "type": "double" + }, + "access_latency": { + "type": "double" + }, + "processing_latency": { + "type": "double" + }, + "total_latency": { + "type": "double" + }, + "location": { + "type": "geo_shape", + "strategy": "recursive" + } + } + } + } + }, + "duration": { + "type": "double" + }, + "id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_payload": { + "properties": { + "job_type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "payload_task_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + } + } + } + } + }, + "localize_urls": { + "type": "object", + "enabled": false + }, + "params": { + "type": "object", + "enabled": false + }, + "type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "name": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "task_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "tag": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "username": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "job_hash": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + } + } + }, + "uuid": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "job_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "payload_id": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "payload_hash": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dedup": { + "type": "boolean" + }, + "dedup_job": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "dedup_msg": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "error": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "copy_to": [ + "text_fields" + ] + }, + "short_error": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "copy_to": [ + "text_fields" + ] + }, + "traceback": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "signum": { + "type": "integer" + }, + "celery_hostname": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "celery_timestamp": { + "type": "date" + }, + "celery_pid": { + "type": "integer" + }, + "celery_runtime": { + "type": "double" + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "copy_to": [ + "text_fields" + ] + }, + "user_tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "copy_to": [ + "text_fields" + ] + }, + "text_fields": { + "type": "text" + } + } + }, + "settings": { + "number_of_shards": 8, + "index": { + "refresh_interval": "5s", + "lifecycle": { + "name": "ilm_policy_mozart" + } + }, + "analysis": { + "analyzer": { + "default": { + "filter": [ + "lowercase", + "word_delimiter" + ], + "tokenizer": "keyword" + } + } + } + }, + "aliases": { + "job_status": {}, + "job_status-current": {} + } + } +} diff --git a/sdscli/adapters/hysds/files/task_status.template b/sdscli/adapters/hysds/files/task_status.template new file mode 100644 index 0000000..a6d58b3 --- /dev/null +++ b/sdscli/adapters/hysds/files/task_status.template @@ -0,0 +1,147 @@ +{ + "index_patterns": [ + "task_status*" + ], + "template": { + "mappings": { + "properties": { + "resource": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "status": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "celery_hostname": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "uuid": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "sw_sys": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "local_received": { + "type": "date" + }, + "clock": { + "type": "integer" + }, + "timestamp": { + "type": "date" + }, + "hostname": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "pid": { + "type": "integer" + }, + "sw_ver": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "utcoffset": { + "type": "integer" + }, + "loadavg": { + "type": "double" + }, + "processed": { + "type": "integer" + }, + "active": { + "type": "integer" + }, + "freq": { + "type": "double" + }, + "type": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "sw_ident": { + "type": "text", + "copy_to": [ + "text_fields" + ] + } + } + }, + "text_fields": { + "type": "text" + } + } + }, + "settings": { + "number_of_shards": 8, + "index": { + "refresh_interval": "5s" + }, + "analysis": { + "analyzer": { + "default": { + "filter": [ + "lowercase", + "word_delimiter" + ], + "tokenizer": "keyword" + } + } + } + }, + "aliases": { + "task_status": {}, + "job_status": {} + } + } +} diff --git a/sdscli/adapters/hysds/files/worker_status.template b/sdscli/adapters/hysds/files/worker_status.template new file mode 100644 index 0000000..2bd203a --- /dev/null +++ b/sdscli/adapters/hysds/files/worker_status.template @@ -0,0 +1,147 @@ +{ + "index_patterns": [ + "worker_status*" + ], + "template": { + "mappings": { + "properties": { + "resource": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "type": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "status": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "text_fields" + ] + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "celery_hostname": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "uuid": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "sw_sys": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "local_received": { + "type": "date" + }, + "clock": { + "type": "integer" + }, + "timestamp": { + "type": "date" + }, + "hostname": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "pid": { + "type": "integer" + }, + "sw_ver": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "utcoffset": { + "type": "integer" + }, + "loadavg": { + "type": "double" + }, + "processed": { + "type": "integer" + }, + "active": { + "type": "integer" + }, + "freq": { + "type": "double" + }, + "type": { + "type": "text", + "copy_to": [ + "text_fields" + ] + }, + "sw_ident": { + "type": "text", + "copy_to": [ + "text_fields" + ] + } + } + }, + "text_fields": { + "type": "text" + } + } + }, + "settings": { + "number_of_shards": 8, + "index": { + "refresh_interval": "5s" + }, + "analysis": { + "analyzer": { + "default": { + "filter": [ + "lowercase", + "word_delimiter" + ], + "tokenizer": "keyword" + } + } + } + }, + "aliases": { + "worker_status": {}, + "job_status": {} + } + } +} diff --git a/sdscli/adapters/hysds/update.py b/sdscli/adapters/hysds/update.py index 82e9a09..efd6dfb 100644 --- a/sdscli/adapters/hysds/update.py +++ b/sdscli/adapters/hysds/update.py @@ -71,6 +71,16 @@ def update_mozart(conf, ndeps=False, config_only=False, comp='mozart'): execute(fab.install_base_es_template, roles=[comp]) bar.update() + # set the ES ILM policy + set_bar_desc(bar, 'Setting ES Index Lifecycle Manager policy') + execute(fab.install_es_policy, roles=[comp]) + bar.update() + + # install the templates in order to attach the ILM policy onto them + set_bar_desc(bar, 'Installing ES templates') + execute(fab.install_mozart_es_templates, roles=[comp]) + bar.update() + # update logstash jvm.options to increase heap size set_bar_desc(bar, 'Updating logstash jvm.options') execute(fab.send_logstash_jvm_options, 'mozart', roles=[comp]) From b116bc7eb4fdc3c80c4e4108bc65966883b95e48 Mon Sep 17 00:00:00 2001 From: Dustin Lo Date: Wed, 1 Mar 2023 11:14:20 -0800 Subject: [PATCH 2/3] move to fab-classic as it has active support and development (#96) (#98) Co-authored-by: Gerald Manipon --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3b2a8f1..1f6aec2 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ 'elasticsearch-dsl>=7.0.0,<=7.4.0', "awscli>=1.17.1", "boto3>=1.11.1", - "fabric3>=1.14.post1", + "fab-classic>=1.19.2", "Jinja2>=3.0.0,<4.0.0", ], entry_points={"console_scripts": ["sds=sdscli.command_line:main"]}, From 84ba543f58126678ee946debd32c4fdb06bb657c Mon Sep 17 00:00:00 2001 From: Dustin Lo Date: Mon, 20 Mar 2023 13:21:20 -0700 Subject: [PATCH 3/3] NSDS-2584 - Implement rollover of old task, worker, event ES docs on Mozart ES (#99) * added lifecycle and aliases to templates * using clean_indices_from_alias.py to remove mozart indices * deleting templates is _index_template, not _template * added note to location of template files * commenting out this line b/c we shouldnt be deleting the templates --------- Co-authored-by: dustinlo --- sdscli/adapters/hysds/fabfile.py | 13 ++++++++----- sdscli/adapters/hysds/files/event_status.template | 9 +++++++-- sdscli/adapters/hysds/files/task_status.template | 6 +++++- sdscli/adapters/hysds/files/worker_status.template | 6 +++++- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/sdscli/adapters/hysds/fabfile.py b/sdscli/adapters/hysds/fabfile.py index 6581d86..1d40043 100644 --- a/sdscli/adapters/hysds/fabfile.py +++ b/sdscli/adapters/hysds/fabfile.py @@ -488,10 +488,13 @@ def install_es_policy(): ) run(f"curl -XPUT 'localhost:9200/_ilm/policy/ilm_policy_mozart?pretty' -H 'Content-Type: application/json' -d@{target_file}") + def install_mozart_es_templates(): # install index templates # Only job_status.template has ILM policy attached # HC-451 will focus on adding ILM to worker, task, and event status indices + + # template files located in ~/.sds/files templates = [ "job_status.template", "worker_status.template", @@ -619,14 +622,14 @@ def rabbitmq_queues_flush(): def mozart_es_flush(): ctx = get_context() - run('curl -XDELETE http://{MOZART_ES_PVT_IP}:9200/_template/*_status'.format(**ctx)) - run('~/mozart/ops/hysds/scripts/clean_job_status_indexes.sh http://{MOZART_ES_PVT_IP}:9200'.format( + #run('curl -XDELETE http://{MOZART_ES_PVT_IP}:9200/_index_template/*_status'.format(**ctx)) + run('~/mozart/ops/hysds/scripts/clean_indices_from_alias.py http://{MOZART_ES_PVT_IP}:9200 job_status-current'.format( **ctx)) - run('~/mozart/ops/hysds/scripts/clean_task_status_indexes.sh http://{MOZART_ES_PVT_IP}:9200'.format( + run('~/mozart/ops/hysds/scripts/clean_indices_from_alias.py http://{MOZART_ES_PVT_IP}:9200 task_status-current'.format( **ctx)) - run('~/mozart/ops/hysds/scripts/clean_worker_status_indexes.sh http://{MOZART_ES_PVT_IP}:9200'.format( + run('~/mozart/ops/hysds/scripts/clean_indices_from_alias.py http://{MOZART_ES_PVT_IP}:9200 event_status-current'.format( **ctx)) - run('~/mozart/ops/hysds/scripts/clean_event_status_indexes.sh http://{MOZART_ES_PVT_IP}:9200'.format( + run('~/mozart/ops/hysds/scripts/clean_indices_from_alias.py http://{MOZART_ES_PVT_IP}:9200 worker_status-current'.format( **ctx)) #run('~/mozart/ops/hysds/scripts/clean_job_spec_container_indexes.sh http://{MOZART_ES_PVT_IP}:9200'.format(**ctx)) diff --git a/sdscli/adapters/hysds/files/event_status.template b/sdscli/adapters/hysds/files/event_status.template index 2ee29aa..aba4534 100644 --- a/sdscli/adapters/hysds/files/event_status.template +++ b/sdscli/adapters/hysds/files/event_status.template @@ -76,7 +76,10 @@ "settings": { "number_of_shards": 8, "index": { - "refresh_interval": "5s" + "refresh_interval": "5s", + "lifecycle": { + "name": "ilm_policy_mozart" + } }, "analysis": { "analyzer": { @@ -91,7 +94,9 @@ } }, "aliases": { - "job_status": {} + "job_status": {}, + "event_status": {}, + "event_status-current": {} } } } diff --git a/sdscli/adapters/hysds/files/task_status.template b/sdscli/adapters/hysds/files/task_status.template index a6d58b3..dc1a9b5 100644 --- a/sdscli/adapters/hysds/files/task_status.template +++ b/sdscli/adapters/hysds/files/task_status.template @@ -125,7 +125,10 @@ "settings": { "number_of_shards": 8, "index": { - "refresh_interval": "5s" + "refresh_interval": "5s", + "lifecycle": { + "name": "ilm_policy_mozart" + } }, "analysis": { "analyzer": { @@ -141,6 +144,7 @@ }, "aliases": { "task_status": {}, + "task_status-current": {}, "job_status": {} } } diff --git a/sdscli/adapters/hysds/files/worker_status.template b/sdscli/adapters/hysds/files/worker_status.template index 2bd203a..4291226 100644 --- a/sdscli/adapters/hysds/files/worker_status.template +++ b/sdscli/adapters/hysds/files/worker_status.template @@ -125,7 +125,10 @@ "settings": { "number_of_shards": 8, "index": { - "refresh_interval": "5s" + "refresh_interval": "5s", + "lifecycle": { + "name": "ilm_policy_mozart" + } }, "analysis": { "analyzer": { @@ -141,6 +144,7 @@ }, "aliases": { "worker_status": {}, + "worker_status-current": {}, "job_status": {} } }