From 8fc07ec8cae206f7213c8531b3d6e7f6448b7f71 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 14:47:27 -0600 Subject: [PATCH 01/12] Add changes for workload_generator and resources/templates Signed-off-by: Ian Hoang --- osbenchmark/resources/base-workload.json.j2 | 36 ++-------- .../resources/custom-operations.json.j2 | 18 +++++ .../resources/custom-test-procedures.json.j2 | 56 ++++++++++++++++ .../resources/default-operations.json.j2 | 16 +++++ .../resources/default-test-procedures.json.j2 | 51 +++++++++++++++ .../workload_generator/workload_generator.py | 65 +++++++++++++++---- 6 files changed, 197 insertions(+), 45 deletions(-) create mode 100644 osbenchmark/resources/custom-operations.json.j2 create mode 100644 osbenchmark/resources/custom-test-procedures.json.j2 create mode 100644 osbenchmark/resources/default-operations.json.j2 create mode 100644 osbenchmark/resources/default-test-procedures.json.j2 diff --git a/osbenchmark/resources/base-workload.json.j2 b/osbenchmark/resources/base-workload.json.j2 index cfb6ced98..9f432349e 100644 --- a/osbenchmark/resources/base-workload.json.j2 +++ b/osbenchmark/resources/base-workload.json.j2 @@ -22,35 +22,9 @@ ] }{% endfor %} ], - "schedule": [ - { - "operation": "delete-index" - },{% raw %} - { - "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } - },{% endraw %} - { - "operation": { - "operation-type": "cluster-health", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": { - "operation-type": "bulk", - "bulk-size": {{bulk_size | default(5000)}}, - "ingest-percentage": {{ingest_percentage | default(100)}} - }, - "clients": {{bulk_indexing_clients | default(8)}} - },{% endraw -%} - {% block queries %}{% endblock %} - ] + "operations": [ + {% raw %}{{ benchmark.collect(parts="operations/*.json") }}{% endraw %} + ], + "test_procedures": [ + {% raw %}{{ benchmark.collect(parts="test_procedures/*.json") }}{% endraw %} } diff --git a/osbenchmark/resources/custom-operations.json.j2 b/osbenchmark/resources/custom-operations.json.j2 new file mode 100644 index 000000000..47b58ef48 --- /dev/null +++ b/osbenchmark/resources/custom-operations.json.j2 @@ -0,0 +1,18 @@ +{ + "name": "index-append", + "operation-type": "bulk", + "bulk-size": {{bulk_size | default(5000)}}, + "ingest-percentage": {{ingest_percentage | default(100)}} +}, +{%- block queries -%} + {% for query in custom_queries %} + { + "operation": { + "name": "{{query.name}}", + "operation-type": "{{query['operation-type']}}", + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, + "body": {{query.body | replace("'", '"') }} + } + }{% if not loop.last %},{% endif -%} + {% endfor %} +{%- endblock %} \ No newline at end of file diff --git a/osbenchmark/resources/custom-test-procedures.json.j2 b/osbenchmark/resources/custom-test-procedures.json.j2 new file mode 100644 index 000000000..4c04adcb8 --- /dev/null +++ b/osbenchmark/resources/custom-test-procedures.json.j2 @@ -0,0 +1,56 @@ +{ + "name": "custom-test-procedures", + "description": "Customized test procedure with custom operations generated by create-workload feature in OpenSearch Benchmark." + "default": true, + "schedule": [ + { + "operation": "delete-index" + },{% raw %} + { + "operation": { + "operation-type": "create-index", + "settings": {{index_settings | tojson}} + } + },{% endraw %} + { + "operation": { + "operation-type": "cluster-health", + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} + "request-params": { + "wait_for_status": "{{cluster_health}}", + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } + }, + { + "operation": "index-append", + {% raw -%}"clients": {{bulk_indexing_clients}}, + "ignore-response-error-level": "{{error_level}}"{% endraw -%} + }, + { + "name": "refresh-after-index", + "operation": "refresh" + }, + {% endraw -%} + {%- block queries -%} + {% for query in custom_queries %} + { + "operation":"{{query.name}}", + "warmup-iterations": {{query.warmup_iterations}}}}, + "iterations": {{query.iterations}}, + {%- if query.target_throughput %} + "target-throughput": {{ query.target_throughput}} + {%- else %} + "target-throughput": {{ query.target_throughput | tojson }} + {%- endif %} + {%-if search_clients is defined and search_clients %} + ,"clients": {{ search_clients | tojson}} + {%- endif %} + },{% if not loop.last %},{% endif -%} + {% endfor %} + + {%- endblock %} + } + ] +} diff --git a/osbenchmark/resources/default-operations.json.j2 b/osbenchmark/resources/default-operations.json.j2 new file mode 100644 index 000000000..e8696704e --- /dev/null +++ b/osbenchmark/resources/default-operations.json.j2 @@ -0,0 +1,16 @@ +{ + "name": "index-append", + "operation-type": "bulk", + "bulk-size": {{bulk_size | default(5000)}}, + "ingest-percentage": {{ingest_percentage | default(100)}} +}, +{ + "name": "default", + "operation-type": "search", + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, + "body": { + "query": { + "match_all": {} + } + } +} \ No newline at end of file diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 new file mode 100644 index 000000000..69391d2fe --- /dev/null +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -0,0 +1,51 @@ +{ + "name": "default-test-procedure", + "description": "Customized test procedure with custom operations generated by create-workload feature in OpenSearch Benchmark. Workload deletes existing indexes, creates indexes, ingests documents, and runs default match-all query.", + "default": true, + "schedule": [ + { + "operation": "delete-index" + },{% raw %} + { + "operation": { + "operation-type": "create-index", + "settings": {{index_settings | default({}) | tojson}} + } + },{% endraw %} + { + "operation": { + "operation-type": "cluster-health", + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } + },{% endraw -%} + { + "operation": "index-append", + {% raw -%} "clients": {{bulk_indexing_clients | default(8)}},{% endraw -%} + {% raw -%}"ignore-response-error-level": "{{error_level | default('non-fatal')}}"{% endraw -%} + }, + { + "name": "refresh-after-index", + "operation": "refresh" + }, + {% raw %} + { + "operation": "default", + "warmup-iterations": {{warmup_iterations | default(50)}}, + "iterations": {{iterations | default(100)}} + {%- if not target_throughput %} + ,"target-throughput": 3 + {%- elif target_throughput is string and target_throughput.lower() == 'none' %} + {%- else %} + ,"target-throughput": {{ target_throughput | tojson }} + {%- endif %} + {%-if search_clients is defined and search_clients %} + ,"clients": {{ search_clients | tojson}} + {%- endif %} + }{% endraw -%} + ] +} \ No newline at end of file diff --git a/osbenchmark/workload_generator/workload_generator.py b/osbenchmark/workload_generator/workload_generator.py index 3bcaacc42..2f1716b57 100644 --- a/osbenchmark/workload_generator/workload_generator.py +++ b/osbenchmark/workload_generator/workload_generator.py @@ -24,6 +24,7 @@ import logging import os +import shutil import json from opensearchpy import OpenSearchException @@ -34,14 +35,6 @@ from osbenchmark.workload_generator import corpus, index from osbenchmark.utils import io, opts, console - -def process_template(templates_path, template_filename, template_vars, output_path): - env = Environment(loader=FileSystemLoader(templates_path), autoescape=select_autoescape(['html', 'xml'])) - template = env.get_template(template_filename) - - with open(output_path, "w") as f: - f.write(template.render(template_vars)) - def validate_indices_docs_map(indices, indices_docs_map, docs_were_requested): if not docs_were_requested: return @@ -108,9 +101,33 @@ def process_custom_queries(custom_queries): return data +def write_template(template_vars, output_path, template_file): + template = get_template(template_file) + with open(output_path, "w") as f: + f.write(template.render(template_vars)) + +def get_template(template_file): + template_file_name = template_file + ".json.j2" + templates_path = os.path.join(cfg.opts("node", "benchmark.root"), "resources") + + env = Environment(loader=FileSystemLoader(templates_path), autoescape=select_autoescape(['html', 'xml'])) + + return env.get_template(template_file_name) + +def render_templates(workload_path, operations_path, test_procedures_path, template_vars, custom_queries): + write_template(workload_path, "base-workload") + + if custom_queries: + write_template(template_vars, operations_path, "custom-operations") + write_template(template_vars, test_procedures_path, "custom-test-procedures") + else: + write_template(template_vars, operations_path, "default-operations") + write_template(template_vars, test_procedures_path, "default-test-procedures") + def create_workload(cfg): logger = logging.getLogger(__name__) + # All inputs provided by user workload_name = cfg.opts("workload", "workload.name") indices = cfg.opts("generator", "indices") root_path = cfg.opts("generator", "output.path") @@ -119,24 +136,43 @@ def create_workload(cfg): number_of_docs = cfg.opts("generator", "number_of_docs") unprocessed_custom_queries = cfg.opts("workload", "custom_queries") + # Process custom queries custom_queries = process_custom_queries(unprocessed_custom_queries) logger.info("Creating workload [%s] matching indices [%s]", workload_name, indices) logger.info("Number of Docs: %s", number_of_docs) + + # Initialize client factory client = OsClientFactory(hosts=target_hosts.all_hosts[opts.TargetHosts.DEFAULT], client_options=client_options.all_client_options[opts.TargetHosts.DEFAULT]).create() - info = client.info() console.info(f"Connected to OpenSearch cluster [{info['name']}] version [{info['version']['number']}].\n", logger=logger) + # Establish output paths directory output_path = os.path.abspath(os.path.join(io.normalize_path(root_path), workload_name)) + + operations_path = os.path.join(output_path, "operations") + test_procedures_path = os.path.join(output_path, "test_procedures") + + try: + logger.info(f"Removing existing workload [{workload_name}] in path [{output_path}]") + shutil.rmtree(output_path) + except OSError: + pass + io.ensure_dir(output_path) + io.ensure_dir(operations_path) + io.ensure_dir(test_procedures_path) + # Extract Indices and Corpora + logger.info("Extracting indices and corpora") indices, corpora = extract_mappings_and_corpora(client, output_path, indices, number_of_docs) + logger.info("Finished extracting indices and corpora") if len(indices) == 0: raise RuntimeError("Failed to extract any indices for workload!") + # Collect all itmes into dictionary template_vars = { "workload_name": workload_name, "indices": indices, @@ -147,12 +183,13 @@ def create_workload(cfg): logger.info("Template Vars: %s", template_vars) workload_path = os.path.join(output_path, "workload.json") - templates_path = os.path.join(cfg.opts("node", "benchmark.root"), "resources") + operations_path = os.path.join(operations_path, "default.json") + test_procedures_path = os.path.join(test_procedures_path, "default.json") - if custom_queries: - process_template(templates_path, "custom-query-workload.json.j2", template_vars, workload_path) - else: - process_template(templates_path, "default-query-workload.json.j2", template_vars, workload_path) + # Render all templates + logger.info("Rendering templates") + render_templates(workload_path, operations_path, test_procedures_path, + template_vars, custom_queries) console.println("") console.info(f"Workload {workload_name} has been created. Run it with: {PROGRAM_NAME} --workload-path={output_path}") From e914c484e275cebe436b730d4860523393b3c345 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 16:51:45 -0600 Subject: [PATCH 02/12] Fix bugs to allow workload to be created Signed-off-by: Ian Hoang --- osbenchmark/resources/base-workload.json.j2 | 1 + .../resources/default-test-procedures.json.j2 | 3 +- .../workload_generator/workload_generator.py | 32 ++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/osbenchmark/resources/base-workload.json.j2 b/osbenchmark/resources/base-workload.json.j2 index 9f432349e..ca0c5668a 100644 --- a/osbenchmark/resources/base-workload.json.j2 +++ b/osbenchmark/resources/base-workload.json.j2 @@ -27,4 +27,5 @@ ], "test_procedures": [ {% raw %}{{ benchmark.collect(parts="test_procedures/*.json") }}{% endraw %} + ] } diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 index 69391d2fe..81fac37b6 100644 --- a/osbenchmark/resources/default-test-procedures.json.j2 +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -31,8 +31,7 @@ { "name": "refresh-after-index", "operation": "refresh" - }, - {% raw %} + },{% raw %} { "operation": "default", "warmup-iterations": {{warmup_iterations | default(50)}}, diff --git a/osbenchmark/workload_generator/workload_generator.py b/osbenchmark/workload_generator/workload_generator.py index 2f1716b57..a7c8e486d 100644 --- a/osbenchmark/workload_generator/workload_generator.py +++ b/osbenchmark/workload_generator/workload_generator.py @@ -101,28 +101,29 @@ def process_custom_queries(custom_queries): return data -def write_template(template_vars, output_path, template_file): - template = get_template(template_file) +def write_template(template_vars, templates_path, output_path, template_file): + template = get_template(template_file, templates_path) with open(output_path, "w") as f: f.write(template.render(template_vars)) -def get_template(template_file): +def get_template(template_file, templates_path): template_file_name = template_file + ".json.j2" - templates_path = os.path.join(cfg.opts("node", "benchmark.root"), "resources") env = Environment(loader=FileSystemLoader(templates_path), autoescape=select_autoescape(['html', 'xml'])) return env.get_template(template_file_name) -def render_templates(workload_path, operations_path, test_procedures_path, template_vars, custom_queries): - write_template(workload_path, "base-workload") +def render_templates( + workload_path, operations_path, test_procedures_path, templates_path, template_vars, custom_queries + ): + write_template(template_vars, templates_path, workload_path, "base-workload") if custom_queries: - write_template(template_vars, operations_path, "custom-operations") - write_template(template_vars, test_procedures_path, "custom-test-procedures") + write_template(template_vars, templates_path, operations_path, "custom-operations") + write_template(template_vars, templates_path, test_procedures_path, "custom-test-procedures") else: - write_template(template_vars, operations_path, "default-operations") - write_template(template_vars, test_procedures_path, "default-test-procedures") + write_template(template_vars, templates_path, operations_path, "default-operations") + write_template(template_vars, templates_path, test_procedures_path, "default-test-procedures") def create_workload(cfg): logger = logging.getLogger(__name__) @@ -135,6 +136,7 @@ def create_workload(cfg): client_options = cfg.opts("client", "options") number_of_docs = cfg.opts("generator", "number_of_docs") unprocessed_custom_queries = cfg.opts("workload", "custom_queries") + templates_path = os.path.join(cfg.opts("node", "benchmark.root"), "resources") # Process custom queries custom_queries = process_custom_queries(unprocessed_custom_queries) @@ -188,8 +190,14 @@ def create_workload(cfg): # Render all templates logger.info("Rendering templates") - render_templates(workload_path, operations_path, test_procedures_path, - template_vars, custom_queries) + render_templates( + workload_path, + operations_path, + test_procedures_path, + templates_path, + template_vars, + custom_queries + ) console.println("") console.info(f"Workload {workload_name} has been created. Run it with: {PROGRAM_NAME} --workload-path={output_path}") From 4bda72c8e9dfad3a0ef33fda84f473ae883501f2 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 16:58:22 -0600 Subject: [PATCH 03/12] Add working changes to custom operations and test procedures Signed-off-by: Ian Hoang --- .../resources/custom-operations.json.j2 | 18 +++++----- .../resources/custom-test-procedures.json.j2 | 34 +++++-------------- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/osbenchmark/resources/custom-operations.json.j2 b/osbenchmark/resources/custom-operations.json.j2 index 47b58ef48..5108d0604 100644 --- a/osbenchmark/resources/custom-operations.json.j2 +++ b/osbenchmark/resources/custom-operations.json.j2 @@ -5,14 +5,12 @@ "ingest-percentage": {{ingest_percentage | default(100)}} }, {%- block queries -%} - {% for query in custom_queries %} - { - "operation": { - "name": "{{query.name}}", - "operation-type": "{{query['operation-type']}}", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, - "body": {{query.body | replace("'", '"') }} - } - }{% if not loop.last %},{% endif -%} - {% endfor %} +{% for query in custom_queries %} +{ + "name": "{{query.name}}", + "operation-type": "{{query['operation-type']}}", + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, + "body": {{query.body | replace("'", '"') }} +}{% if not loop.last %},{% endif -%} +{% endfor %} {%- endblock %} \ No newline at end of file diff --git a/osbenchmark/resources/custom-test-procedures.json.j2 b/osbenchmark/resources/custom-test-procedures.json.j2 index 4c04adcb8..3846ba60a 100644 --- a/osbenchmark/resources/custom-test-procedures.json.j2 +++ b/osbenchmark/resources/custom-test-procedures.json.j2 @@ -1,6 +1,6 @@ { "name": "custom-test-procedures", - "description": "Customized test procedure with custom operations generated by create-workload feature in OpenSearch Benchmark." + "description": "Customized test procedure with custom operations generated by create-workload feature in OpenSearch Benchmark.", "default": true, "schedule": [ { @@ -9,15 +9,16 @@ { "operation": { "operation-type": "create-index", - "settings": {{index_settings | tojson}} + "settings": {{index_settings | default({}) | tojson}} } },{% endraw %} { "operation": { "operation-type": "cluster-health", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw -%} "request-params": { - "wait_for_status": "{{cluster_health}}", + "wait_for_status": "{{cluster_health | default('green')}}", + {% endraw -%} "wait_for_no_relocating_shards": "true" }, "retry-until-success": true @@ -25,32 +26,13 @@ }, { "operation": "index-append", - {% raw -%}"clients": {{bulk_indexing_clients}}, - "ignore-response-error-level": "{{error_level}}"{% endraw -%} + {% raw -%}"clients": {{bulk_indexing_clients | default(8)}}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}" }, + {% endraw -%} { "name": "refresh-after-index", "operation": "refresh" - }, - {% endraw -%} - {%- block queries -%} - {% for query in custom_queries %} - { - "operation":"{{query.name}}", - "warmup-iterations": {{query.warmup_iterations}}}}, - "iterations": {{query.iterations}}, - {%- if query.target_throughput %} - "target-throughput": {{ query.target_throughput}} - {%- else %} - "target-throughput": {{ query.target_throughput | tojson }} - {%- endif %} - {%-if search_clients is defined and search_clients %} - ,"clients": {{ search_clients | tojson}} - {%- endif %} - },{% if not loop.last %},{% endif -%} - {% endfor %} - - {%- endblock %} } ] } From 8eac9bbd49d6448c5f59e4e77ff4980fb7b5ea65 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 17:07:36 -0600 Subject: [PATCH 04/12] Add fixes for custom test procedures part 2 Signed-off-by: Ian Hoang --- .../resources/custom-test-procedures.json.j2 | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/osbenchmark/resources/custom-test-procedures.json.j2 b/osbenchmark/resources/custom-test-procedures.json.j2 index 3846ba60a..7301c22ea 100644 --- a/osbenchmark/resources/custom-test-procedures.json.j2 +++ b/osbenchmark/resources/custom-test-procedures.json.j2 @@ -33,6 +33,26 @@ { "name": "refresh-after-index", "operation": "refresh" - } + }, +{%- block queries -%} + {% for query in custom_queries %} + { + "operation":"{{query.name}}", + {% raw -%} + "warmup-iterations": {{warmup_iterations | default(50)}}, + "iterations": {{iterations | default(100)}} + {%- if not target_throughput %} + ,"target-throughput": 3 + {%- elif target_throughput is string and target_throughput.lower() == 'none' %} + {%- else %} + ,"target-throughput": {{ target_throughput | tojson }} + {%- endif %} + {%-if search_clients is defined and search_clients %} + ,"clients": {{ search_clients | tojson}} + {%- endif %} + {% endraw -%} + }{% if not loop.last %},{% endif -%} + {% endfor %} +{%- endblock %} ] } From 235f0a4301fdf79f52d7f5cb49affe51492d0c1e Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 17:30:19 -0600 Subject: [PATCH 05/12] Make default test procedures format the same as custom test procedures format Signed-off-by: Ian Hoang --- osbenchmark/resources/default-test-procedures.json.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 index 81fac37b6..2ee15b64f 100644 --- a/osbenchmark/resources/default-test-procedures.json.j2 +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -25,9 +25,10 @@ },{% endraw -%} { "operation": "index-append", - {% raw -%} "clients": {{bulk_indexing_clients | default(8)}},{% endraw -%} - {% raw -%}"ignore-response-error-level": "{{error_level | default('non-fatal')}}"{% endraw -%} + {% raw -%}"clients": {{bulk_indexing_clients | default(8)}}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}" }, + {% endraw -%} { "name": "refresh-after-index", "operation": "refresh" From bfa5afeb8de63df40614e7552f8e366bf02a9b1c Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 17:39:32 -0600 Subject: [PATCH 06/12] Fix pylint error Signed-off-by: Ian Hoang --- osbenchmark/workload_generator/workload_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osbenchmark/workload_generator/workload_generator.py b/osbenchmark/workload_generator/workload_generator.py index a7c8e486d..02b82b8ce 100644 --- a/osbenchmark/workload_generator/workload_generator.py +++ b/osbenchmark/workload_generator/workload_generator.py @@ -157,7 +157,7 @@ def create_workload(cfg): test_procedures_path = os.path.join(output_path, "test_procedures") try: - logger.info(f"Removing existing workload [{workload_name}] in path [{output_path}]") + logger.info("Removing existing workload [%s] in path [%s]", workload_name, output_path) shutil.rmtree(output_path) except OSError: pass From 11aa710bb109e15887b0f6ad0871f621afff3480 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Tue, 23 Jan 2024 17:45:51 -0600 Subject: [PATCH 07/12] Remove old unused workload.json templates Signed-off-by: Ian Hoang --- .../resources/custom-query-workload.json.j2 | 14 -------------- .../resources/default-query-workload.json.j2 | 16 ---------------- 2 files changed, 30 deletions(-) delete mode 100644 osbenchmark/resources/custom-query-workload.json.j2 delete mode 100644 osbenchmark/resources/default-query-workload.json.j2 diff --git a/osbenchmark/resources/custom-query-workload.json.j2 b/osbenchmark/resources/custom-query-workload.json.j2 deleted file mode 100644 index 76028db77..000000000 --- a/osbenchmark/resources/custom-query-workload.json.j2 +++ /dev/null @@ -1,14 +0,0 @@ -{% extends "base-workload.json.j2" %} - -{%- block queries -%} - {% for query in custom_queries %} - { - "operation": { - "name": "{{query.name}}", - "operation-type": "{{query['operation-type']}}", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, - "body": {{query.body | replace("'", '"') }} - } - }{% if not loop.last %},{% endif -%} - {% endfor %} -{%- endblock %} diff --git a/osbenchmark/resources/default-query-workload.json.j2 b/osbenchmark/resources/default-query-workload.json.j2 deleted file mode 100644 index 38c296738..000000000 --- a/osbenchmark/resources/default-query-workload.json.j2 +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "base-workload.json.j2" %} - -{% block queries %} - { - "operation": { - "operation-type": "search", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, - "body": { - "query": { - "match_all": {} - } - } - },{% raw %} - "clients": {{search_clients | default(8)}}{% endraw %} - } -{%- endblock %} \ No newline at end of file From f75c3a5b8ab310461442f007ca5f7240a8242998 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Thu, 25 Jan 2024 11:12:07 -0600 Subject: [PATCH 08/12] Add fixes for default operations and default test procedures Signed-off-by: Ian Hoang --- .../resources/default-operations.json.j2 | 14 +++++++++- .../resources/default-test-procedures.json.j2 | 26 +++++++++++++++---- .../workload_generator/workload_generator.py | 21 +++++++++------ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/osbenchmark/resources/default-operations.json.j2 b/osbenchmark/resources/default-operations.json.j2 index e8696704e..969a3b913 100644 --- a/osbenchmark/resources/default-operations.json.j2 +++ b/osbenchmark/resources/default-operations.json.j2 @@ -5,10 +5,22 @@ "ingest-percentage": {{ingest_percentage | default(100)}} }, { - "name": "default", + "name": "wait-until-merges-finish", + "operation-type": "index-stats", + "index": "_all", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false +}, +{ + "name": "match-all", "operation-type": "search", "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, "body": { + "size": {{match_all_size | default(10)}}, "query": { "match_all": {} } diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 index 2ee15b64f..69ead4000 100644 --- a/osbenchmark/resources/default-test-procedures.json.j2 +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -22,19 +22,35 @@ }, "retry-until-success": true } - },{% endraw -%} + }, + {% endraw -%} { "operation": "index-append", {% raw -%}"clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - {% endraw -%} + {% endraw -%} + {% raw -%}"ignore-response-error-level": "{{error_level | default('non-fatal')}}" + {% endraw -%}}, { "name": "refresh-after-index", "operation": "refresh" + }, + { + "operation": { + "operation-type": "force-merge", + "request-timeout": 7200{%- if force_merge_max_num_segments is defined %}, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + {%- endif %} + } + }, + { + "name": "refresh-after-force-merge", + "operation": "refresh" + }, + { + "operation": "wait-until-merges-finish" },{% raw %} { - "operation": "default", + "operation": "match-all", "warmup-iterations": {{warmup_iterations | default(50)}}, "iterations": {{iterations | default(100)}} {%- if not target_throughput %} diff --git a/osbenchmark/workload_generator/workload_generator.py b/osbenchmark/workload_generator/workload_generator.py index 02b82b8ce..acb736cb4 100644 --- a/osbenchmark/workload_generator/workload_generator.py +++ b/osbenchmark/workload_generator/workload_generator.py @@ -113,9 +113,12 @@ def get_template(template_file, templates_path): return env.get_template(template_file_name) -def render_templates( - workload_path, operations_path, test_procedures_path, templates_path, template_vars, custom_queries - ): +def render_templates(workload_path, + operations_path, + test_procedures_path, + templates_path, + template_vars, + custom_queries): write_template(template_vars, templates_path, workload_path, "base-workload") if custom_queries: @@ -156,11 +159,13 @@ def create_workload(cfg): operations_path = os.path.join(output_path, "operations") test_procedures_path = os.path.join(output_path, "test_procedures") - try: - logger.info("Removing existing workload [%s] in path [%s]", workload_name, output_path) - shutil.rmtree(output_path) - except OSError: - pass + if os.path.exists(output_path): + try: + logger.info("Workload already exists. Removing existing workload [%s] in path [%s]", workload_name, output_path) + shutil.rmtree(output_path) + except OSError: + logger.error("Had issues removing existing workload [%s] in path [%s]", workload_name, output_path) + pass io.ensure_dir(output_path) io.ensure_dir(operations_path) From 74b86421978066a161315ef27a14e19d0c0bc6fb Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Thu, 25 Jan 2024 11:37:45 -0600 Subject: [PATCH 09/12] Fix spacing for jinja parameters and simplify operation parameters Signed-off-by: Ian Hoang --- .../resources/default-test-procedures.json.j2 | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 index 69ead4000..841ab5250 100644 --- a/osbenchmark/resources/default-test-procedures.json.j2 +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -9,7 +9,7 @@ { "operation": { "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} + "settings": {{ index_settings | default({}) | tojson }} } },{% endraw %} { @@ -17,7 +17,7 @@ "operation-type": "cluster-health", "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", + "wait_for_status": "{{ cluster_health | default('green') }}", "wait_for_no_relocating_shards": "true" }, "retry-until-success": true @@ -26,9 +26,9 @@ {% endraw -%} { "operation": "index-append", - {% raw -%}"clients": {{bulk_indexing_clients | default(8)}}, + {% raw -%}"clients": {{ bulk_indexing_clients | default(8) }}, {% endraw -%} - {% raw -%}"ignore-response-error-level": "{{error_level | default('non-fatal')}}" + {% raw -%}"ignore-response-error-level": "{{ error_level | default('non-fatal') }}" {% endraw -%}}, { "name": "refresh-after-index", @@ -51,17 +51,10 @@ },{% raw %} { "operation": "match-all", - "warmup-iterations": {{warmup_iterations | default(50)}}, - "iterations": {{iterations | default(100)}} - {%- if not target_throughput %} - ,"target-throughput": 3 - {%- elif target_throughput is string and target_throughput.lower() == 'none' %} - {%- else %} - ,"target-throughput": {{ target_throughput | tojson }} - {%- endif %} - {%-if search_clients is defined and search_clients %} - ,"clients": {{ search_clients | tojson}} - {%- endif %} + "warmup-iterations": {{ warmup_iterations | default(50) }}, + "iterations": {{ iterations | default(100) }}, + "target-throughput": {{ target_throughput | default(3) }}, + "clients": {{ search_clients | default(1) }} }{% endraw -%} ] } \ No newline at end of file From 641d9fcb52b2ae7d264e7790de690a9e370f7b79 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Thu, 25 Jan 2024 11:43:46 -0600 Subject: [PATCH 10/12] Fix raw / endraw pair to be more concise Signed-off-by: Ian Hoang --- .../resources/default-test-procedures.json.j2 | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/osbenchmark/resources/default-test-procedures.json.j2 b/osbenchmark/resources/default-test-procedures.json.j2 index 841ab5250..b8f4c73da 100644 --- a/osbenchmark/resources/default-test-procedures.json.j2 +++ b/osbenchmark/resources/default-test-procedures.json.j2 @@ -5,25 +5,24 @@ "schedule": [ { "operation": "delete-index" - },{% raw %} + }, { "operation": { "operation-type": "create-index", - "settings": {{ index_settings | default({}) | tojson }} - } - },{% endraw %} + {% raw %}"settings": {{ index_settings | default({}) | tojson }} + {% endraw %}} + }, { "operation": { "operation-type": "cluster-health", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw %} + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, "request-params": { - "wait_for_status": "{{ cluster_health | default('green') }}", - "wait_for_no_relocating_shards": "true" + {% raw %}"wait_for_status": "{{ cluster_health | default('green') }}", + {% endraw -%}"wait_for_no_relocating_shards": "true" }, "retry-until-success": true } }, - {% endraw -%} { "operation": "index-append", {% raw -%}"clients": {{ bulk_indexing_clients | default(8) }}, From 59adc88eac3ad79fed5dd1566312e425e9a37a72 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Thu, 25 Jan 2024 12:15:55 -0600 Subject: [PATCH 11/12] Add suggested fixes to custom-operations and custom-test-procedures Signed-off-by: Ian Hoang --- .../resources/custom-operations.json.j2 | 11 ++++ .../resources/custom-test-procedures.json.j2 | 56 ++++++++++--------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/osbenchmark/resources/custom-operations.json.j2 b/osbenchmark/resources/custom-operations.json.j2 index 5108d0604..16fb38606 100644 --- a/osbenchmark/resources/custom-operations.json.j2 +++ b/osbenchmark/resources/custom-operations.json.j2 @@ -4,6 +4,17 @@ "bulk-size": {{bulk_size | default(5000)}}, "ingest-percentage": {{ingest_percentage | default(100)}} }, +{ + "name": "wait-until-merges-finish", + "operation-type": "index-stats", + "index": "_all", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false +}, {%- block queries -%} {% for query in custom_queries %} { diff --git a/osbenchmark/resources/custom-test-procedures.json.j2 b/osbenchmark/resources/custom-test-procedures.json.j2 index 7301c22ea..5fc247db1 100644 --- a/osbenchmark/resources/custom-test-procedures.json.j2 +++ b/osbenchmark/resources/custom-test-procedures.json.j2 @@ -5,53 +5,59 @@ "schedule": [ { "operation": "delete-index" - },{% raw %} + }, { "operation": { "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } - },{% endraw %} + {% raw %}"settings": {{ index_settings | default({}) | tojson }} + {% endraw %}} + }, { "operation": { "operation-type": "cluster-health", - "index": {{ indices | map(attribute='name') | list | join(',') | tojson }},{% raw -%} + "index": {{ indices | map(attribute='name') | list | join(',') | tojson }}, "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - {% endraw -%} - "wait_for_no_relocating_shards": "true" + {% raw %}"wait_for_status": "{{ cluster_health | default('green') }}", + {% endraw -%}"wait_for_no_relocating_shards": "true" }, "retry-until-success": true } }, { "operation": "index-append", - {% raw -%}"clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - {% endraw -%} + {% raw -%}"clients": {{ bulk_indexing_clients | default(8) }}, + {% endraw -%} + {% raw -%}"ignore-response-error-level": "{{ error_level | default('non-fatal') }}" + {% endraw -%}}, { "name": "refresh-after-index", "operation": "refresh" }, + { + "operation": { + "operation-type": "force-merge", + "request-timeout": 7200{%- if force_merge_max_num_segments is defined %}, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + {%- endif %} + } + }, + { + "name": "refresh-after-force-merge", + "operation": "refresh" + }, + { + "operation": "wait-until-merges-finish" + }, {%- block queries -%} {% for query in custom_queries %} { "operation":"{{query.name}}", {% raw -%} - "warmup-iterations": {{warmup_iterations | default(50)}}, - "iterations": {{iterations | default(100)}} - {%- if not target_throughput %} - ,"target-throughput": 3 - {%- elif target_throughput is string and target_throughput.lower() == 'none' %} - {%- else %} - ,"target-throughput": {{ target_throughput | tojson }} - {%- endif %} - {%-if search_clients is defined and search_clients %} - ,"clients": {{ search_clients | tojson}} - {%- endif %} - {% endraw -%} - }{% if not loop.last %},{% endif -%} + "warmup-iterations": {{ warmup_iterations | default(50) }}, + "iterations": {{ iterations | default(100) }}, + "target-throughput": {{ target_throughput | default(3) }}, + "clients": {{ search_clients | default(1) }} + {% endraw -%}}{% if not loop.last %},{% endif -%} {% endfor %} {%- endblock %} ] From c9e7135aed3b5f85d1e93c210bcb065841733921 Mon Sep 17 00:00:00 2001 From: Ian Hoang Date: Thu, 25 Jan 2024 12:38:10 -0600 Subject: [PATCH 12/12] Remove pass statement for pylint Signed-off-by: Ian Hoang --- osbenchmark/workload_generator/workload_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/osbenchmark/workload_generator/workload_generator.py b/osbenchmark/workload_generator/workload_generator.py index acb736cb4..60163701e 100644 --- a/osbenchmark/workload_generator/workload_generator.py +++ b/osbenchmark/workload_generator/workload_generator.py @@ -165,7 +165,6 @@ def create_workload(cfg): shutil.rmtree(output_path) except OSError: logger.error("Had issues removing existing workload [%s] in path [%s]", workload_name, output_path) - pass io.ensure_dir(output_path) io.ensure_dir(operations_path)