fix(test-case): update 5000 tables test case configuration

List of changes:
- Disable per-table metrics due to significant perf impact.
- Enable cluster health checks which work with this case just fine.
- Decrease the nemesis interval from 60 minutes to just 3 keeping
  in mind that health checks will take some time too.
- Reduce stress time for each of the 5000 commands.
  Having 20 minutes per cmd we will get about 1.5 days long test runs
  instead of the 2.5 days.
- Reduce number of loaders from 5 to 3 to use resources more
  efficiently. In current case the bottleneck is the RAM.

Note that this scenario hits following bug:
- scylladb/scylla-enterprise#5093

If 'destroy_data_then_repair' nemesis gets triggered aganst the setup
of this scenario.

(cherry picked from commit 0c7fa60)

jenkins-pipelines/oss/tier2/scale-5000-tables.jenkinsfile

@@ -8,6 +8,4 @@ longevityPipeline(
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_user_batch_custom_time',
     test_config: 'test-cases/scale/longevity-5000-tables.yaml',
-
-    timeout: [time: 4440, unit: 'MINUTES'],
 )

sdcm/provision/scylla_yaml/scylla_yaml.py

@@ -292,6 +292,7 @@ def set_authorizer(cls, authorizer: str):
     large_memory_allocation_warning_threshold: int = None  # 2 ** 20
     enable_deprecated_partitioners: bool = None  # False
     enable_keyspace_column_family_metrics: bool = None  # False
+    enable_node_aggregated_table_metrics: bool = None  # True
     enable_sstable_data_integrity_check: bool = None  # False
     enable_sstable_key_validation: bool = None  # None
     cpu_scheduler: bool = None  # True

test-cases/scale/longevity-5000-tables.yaml

@@ -1,6 +1,6 @@
-test_duration: 2160
+test_duration: 2520
 
-cs_duration: '55m'
+cs_duration: '20m'
 cs_user_profiles:
     - scylla-qa-internal/cust_d/templated_tables_mv.yaml
 
@@ -9,24 +9,25 @@ pre_create_schema: true
 user_profile_table_count: 5000
 batch_size: 100
 
-n_loaders: 5
+n_loaders: 3
 n_monitor_nodes: 1
 n_db_nodes: 1
 add_node_cnt: 5
 
-jmx_heap_memory: 1024 # this is a fix/workaround for https://github.com/scylladb/scylla/issues/7609
-
 instance_type_db: 'i4i.8xlarge'
-instance_type_loader: 'c6i.4xlarge'
-user_prefix: 'longevity-5000-tables'
+instance_type_loader: 'c6i.8xlarge'
+instance_type_monitor: 'm6i.xlarge'
+root_disk_size_monitor: 120
 root_disk_size_runner: 120
 
-
-cluster_health_check: false
+user_prefix: 'long-5000-tables'
+cluster_health_check: true
 
 nemesis_class_name: 'SisyphusMonkey'
 nemesis_seed: '404'
-nemesis_interval: 60
+# NOTE: health checks will take some time, so, don't add significant waiting in this part
+nemesis_interval: 3
 
-# TODO: remove when https://github.com/scylladb/scylla-tools-java/issues/175 resolved
-stop_test_on_stress_failure: false
+append_scylla_yaml:
+  # NOTE: https://github.com/scylladb/scylla-monitoring/issues/2429
+  enable_node_aggregated_table_metrics: false

unit_tests/test_scylla_yaml.py

@@ -240,6 +240,7 @@ def test_scylla_yaml(self):
                 'enable_in_memory_data_store': None,
                 'enable_ipv6_dns_lookup': None,
                 'enable_keyspace_column_family_metrics': None,
+                'enable_node_aggregated_table_metrics': None,
                 'enable_repair_based_node_ops': None,
                 'allowed_repair_based_node_ops': None,
                 'enable_shard_aware_drivers': None,