logs large dataset challenge

elastic/logs/challenges/logging-large-dataset-search.json

@@ -0,0 +1,155 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "logging-large-dataset-search",
+  "description": "Applies a query workload after replicating data to increase the datset size. Ensures data streams exist so queries can be run, but does not remove existing data.",
+  "parameters": {
+    "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }}
+  },
+  "schedule": [
+    {
+      "operation": "discovery-search-request-size-100",
+      "clients": 1,
+      "warmup-iterations": 0,
+      "iterations": 50
+    },
+    {
+      "operation": "discovery-search-request-size-500",
+      "clients": 1,
+      "warmup-iterations": 0,
+      "iterations": 50
+    },
+
+    {
+      "operation": "discovery-search-request-size-1000",
+      "clients": 1,
+      "warmup-iterations": 0,
+      "iterations": 50
+    },
+    {
+      "name": "logging-queries",
+      "parallel": {
+        "time-period": {{ p_query_time_period }},
+        "warmup-time-period": {{ p_query_warmup_time_period }},
+        "tasks": [
+          {% for workflow in p_query_workflows %}
+            {
+              "name": {{workflow | tojson }},
+              "operation": {
+                "operation-type": "composite",
+                "param-source": "workflow-selector",
+                "workflow": {{workflow | tojson }},
+                "task-offset": {{ loop.index }},
+                "request-params": {{ p_query_request_params | tojson(indent=2) }}
+              },
+              "think-time-interval": {{ p_user_think_time }},
+              "workflow-interval": {{ p_user_workflow_time }},
+              "clients": {{ p_search_clients }},
+              "schedule": "workflow-scheduler"
+            }{{ ", " if not loop.last else "" }}
+          {% endfor %}
+        ]
+      }
+    }
+    {%- if p_include_esql_queries %}
+    ,{
+      "operation": "esql_basic_count_group_0_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_basic_count_group_1_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_basic_count_group_2_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_basic_count_group_3_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_basic_count_group_4_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_dissect_duration_and_stats_limit_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "limit0"]
+    },
+    {
+      "operation": "esql_basic_count_group_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "count"]
+    },
+    {
+      "operation": "esql_basic_count_group_1",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "count"]
+    },
+    {
+      "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter",
+      "clients": 1,
+      "warmup-iterations": 5,
+      "iterations": 20,
+      "tags": ["esql", "date_histogram"]
+    },
+    {
+      "operation": "esql_dissect_duration_and_stats",
+      "clients": 1,
+      "warmup-iterations": 5,
+      "iterations": 20,
+      "tags": ["esql", "dissect"]
+    },
+    {
+      "operation": "disable_query_cache",
+      "tags": ["esql", "settings"]
+    },
+    {
+      "operation": "search_basic_count_group_0",
+      "clients": 1,
+      "warmup-iterations": 10,
+      "iterations": 50,
+      "tags": ["esql", "count", "search"]
+    },
+    {
+      "operation": "enable_query_cache",
+      "tags": ["esql", "settings"]
+    }
+    {%- endif %}
+  ]
+}

elastic/logs/challenges/logging-large-index.json

@@ -0,0 +1,71 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "logging-large-dataset-index",
+  "description": "Applies a query workload after replicating data to increase the datset size. Ensures data streams exist so queries can be run, but does not remove existing data.",
+  "parameters": {
+    "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }}
+  },
+  "schedule": [
+    {% include "tasks/index-setup.json" %},
+    {
+      "name": "bulk-index",
+      "operation": {
+        "operation-type": "raw-bulk",
+        "param-source": "processed-source",
+        "time-format": "milliseconds",
+        "profile": "fixed_interval",
+        "init-load": true,
+        "bulk-size": {{ p_bulk_size }},
+        "detailed-results": true
+      },
+      "clients": {{ p_bulk_indexing_clients }},
+      "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
+    }
+    {%- if force_merge_max_num_segments is defined %},
+    {
+      "name": "refresh-after-index",
+      "index": "logs-*",
+      "operation": "refresh"
+    },
+    {
+      "name": "wait-until-index-merges-fininshes",
+      "operation": {
+        "operation-type": "index-stats",
+        "index": "logs-*",
+        "condition": {
+          "path": "_all.total.merges.current",
+          "expected-value": 0
+        },
+        "retry-until-success": true,
+        "include-in-reporting": false
+      }
+    },
+    {
+      "operation": {
+        "operation-type": "force-merge",
+        "index": "logs-*",
+        "request-timeout": 36000,
+        "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+      }
+    },
+    {
+      "name": "wait-until-merges-finish",
+      "operation": {
+        "operation-type": "index-stats",
+        "index": "logs-*",
+        "condition": {
+          "path": "_all.total.merges.current",
+          "expected-value": 0
+        },
+        "retry-until-success": true,
+        "include-in-reporting": false
+      }
+    },
+    {
+      "name": "refresh-after-force-merge",
+      "index": "logs-*",
+      "operation": "refresh"
+    }
+    {%- endif %}
+  ]
+}

elastic/logs/tasks/index-setup.json

@@ -23,6 +23,7 @@
   "tags": ["setup"],
   "operation": "delete-data-stream"
 },
+{% if p_skip_deleting_templates == false %}
 {
   "name": "delete-all-composable-templates",
   "tags": ["setup"],
@@ -33,6 +34,7 @@
   "tags": ["setup"],
   "operation": "delete-component-template"
 },
+{% endif %}
 {
   "name": "create-all-component-templates",
   "tags": ["setup"],

elastic/logs/track.json

@@ -49,6 +49,9 @@
 
 {% set es_version = "7.13.2" %}
 
+{% set p_snapshot_restore_counts = (snapshot_restore_counts | default(1) ) %}
+{% set p_skip_deleting_templates = (skip_deleting_templates | default(false) ) %}
+
 {% set p_integration_ratios = (integration_ratios | default({
     "kafka": {
       "corpora": {