diff --git a/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-3dcs.jenkinsfile b/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-3dcs.jenkinsfile new file mode 100644 index 0000000000..51ab420da2 --- /dev/null +++ b/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-3dcs.jenkinsfile @@ -0,0 +1,12 @@ +#!groovy + +// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43 +def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm) + +// NOTE: multiple real regions must be in alphabetical order for the 'customd_d2/workflow1' case +longevityPipeline( + backend: 'aws', + region: '''["eu-north-1", "eu-west-1", "us-east-1"]''', + test_name: 'longevity_test.LongevityTest.test_custom_time', + test_config: 'test-cases/longevity/longevity-aws-custom-d2-workload1-3dcs.yaml', +) diff --git a/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-5dcs.jenkinsfile b/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-5dcs.jenkinsfile new file mode 100644 index 0000000000..663e42b204 --- /dev/null +++ b/jenkins-pipelines/oss/longevity/longevity-aws-custom-d2-workload1-5dcs.jenkinsfile @@ -0,0 +1,12 @@ +#!groovy + +// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43 +def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm) + +// NOTE: multiple real regions must be in alphabetical order for the 'customd_d2/workflow1' case +longevityPipeline( + backend: 'aws', + region: '''["eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "us-east-1"]''', + test_name: 'longevity_test.LongevityTest.test_custom_time', + test_config: 'test-cases/longevity/longevity-aws-custom-d2-workload1-5dcs.yaml', +) diff --git a/sdcm/provision/scylla_yaml/scylla_yaml.py b/sdcm/provision/scylla_yaml/scylla_yaml.py index 208f0d5fca..ea588adf2e 100644 --- a/sdcm/provision/scylla_yaml/scylla_yaml.py +++ b/sdcm/provision/scylla_yaml/scylla_yaml.py @@ -269,6 +269,7 @@ def set_authorizer(cls, authorizer: str): # NOTE: example for disabling RBNO for 'bootstrap' and 'decommission' operations: # allowed_repair_based_node_ops: "replace,removenode,rebuild" allowed_repair_based_node_ops: str = None + enable_small_table_optimization_for_rbno: bool = None # False ring_delay_ms: int = None # 30 * 1000 shadow_round_ms: int = None # 300 * 1000 fd_max_interval_ms: int = None # 2 * 1000 diff --git a/test-cases/longevity/longevity-aws-custom-d2-workload1-3dcs.yaml b/test-cases/longevity/longevity-aws-custom-d2-workload1-3dcs.yaml new file mode 100644 index 0000000000..d0fc60a03e --- /dev/null +++ b/test-cases/longevity/longevity-aws-custom-d2-workload1-3dcs.yaml @@ -0,0 +1,179 @@ +test_duration: 255 +n_monitor_nodes: 1 +n_db_nodes: '3 3 3' +n_loaders: '3 1 1' + +instance_type_db: 'i4i.2xlarge' +instance_type_loader: 'c6i.xlarge' + +user_prefix: 'long-custom-d2-wrkld1-3dc' +enterprise_disable_kms: true +use_preinstalled_scylla: true + +endpoint_snitch: GossipingPropertyFileSnitch +internode_compression: 'all' +client_encrypt: true +append_scylla_yaml: + consistent_cluster_management: false + compaction_static_shares: 100 + compaction_enforce_min_threshold: true + compaction_throughput_mb_per_sec: 0 + # NOTE: workaround for the https://github.com/scylladb/scylladb/issues/19131 + # Uncomment below line running Scylla version equal to or older than 2024.2 and 6.2 + # allowed_repair_based_node_ops: "replace,removenode,rebuild,decommission" + enable_repair_based_node_ops: true + # NOTE: the 'enable_small_table_optimization_for_rbno' config option + # is supported only on Scylla versions newer than '2024.2' and '6.2'. + # See: https://github.com/scylladb/scylladb/pull/21207 + enable_small_table_optimization_for_rbno: true + enable_tablets: false + +nemesis_class_name: 'SisyphusMonkey' +nemesis_seed: '029' +nemesis_interval: 5 +nemesis_during_prepare: true + +round_robin: true +prepare_write_cmd: + # NOTE: --duration in these commands is number of rows that will be written. + # Time gets specified with 's', 'm' or 'h' letters. + + # NOTE: first group of loaders from a single/first region write to all per-region main tables + - >- + latte run --tag latte-prepare-01 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-02 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-03 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: other loaders write multi-region tables, first half + - >- + latte run --tag latte-prepare-04 --duration 250050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 30 --rate 1500 -P offset=0 + --function custom -P row_count=250050 -P codes="\"T1F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-05 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=550050 -P codes="\"T2F1,T3F1,T4F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: first group of loaders from a single/first region write to all per-region secondary tables + - >- + latte run --tag latte-prepare-06 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T6F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-07 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T7F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-08 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T8F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: other loaders write multi-region tables, second half + - >- + latte run --tag latte-prepare-09 --duration 250050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 30 --rate 1500 -P offset=250050 + --function custom -P row_count=250050 -P codes="\"T1F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-10 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=550050 + --function custom -P row_count=550050 -P codes="\"T2F1,T3F1,T4F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + +stress_cmd: + # NOTE: write to main per-dc tables from loaders of dc-1 + - >- + latte run --tag latte-main-01 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 50000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F1\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-02 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 50000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-03 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 50000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: read dc-2 main table from dc-2 loader + - >- + latte run --tag latte-main-04 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 6000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: read dc-3 main table from dc-3 loader + - >- + latte run --tag latte-main-05 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 6000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: read dc-1 main table from dc-1 loader + - >- + latte run --tag latte-main-06 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 6000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + # NOTE: read from special table with replication everywhere + - >- + latte run --tag latte-main-07 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 240 -P offset=0 + --function custom -P row_count=500100 -P codes="\"T1F3\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + # NOTE: read dc-1 secondary table from dc-1 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-08 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 240 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T6F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: read dc-2 secondary table from dc-2 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-09 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 240 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T7F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: read dc-3 secondary table from dc-3 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-10 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 240 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T8F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: run per-dc main table specific scenario functions with insertions and deletions + - >- + latte run --tag latte-main-11 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 550 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T12F7\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-12 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 550 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T13F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-13 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 550 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T14F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn diff --git a/test-cases/longevity/longevity-aws-custom-d2-workload1-5dcs.yaml b/test-cases/longevity/longevity-aws-custom-d2-workload1-5dcs.yaml new file mode 100644 index 0000000000..6f68d047ad --- /dev/null +++ b/test-cases/longevity/longevity-aws-custom-d2-workload1-5dcs.yaml @@ -0,0 +1,274 @@ +test_duration: 1440 +n_monitor_nodes: 1 +n_db_nodes: '12 12 12 12 12' +n_loaders: '5 1 1 1 1' + +instance_type_db: 'i3en.large' +instance_type_loader: 'c6i.2xlarge' + +user_prefix: 'long-custom-d2-wrkld1-5dc' +enterprise_disable_kms: true +use_preinstalled_scylla: true + +endpoint_snitch: GossipingPropertyFileSnitch +internode_compression: 'all' +client_encrypt: true +append_scylla_yaml: + consistent_cluster_management: false + compaction_static_shares: 100 + compaction_enforce_min_threshold: true + compaction_throughput_mb_per_sec: 0 + # NOTE: workaround for the https://github.com/scylladb/scylladb/issues/19131 + # Uncomment below line running Scylla version equal to or older than 2024.2 and 6.2 + # allowed_repair_based_node_ops: "replace,removenode,rebuild,decommission" + enable_repair_based_node_ops: true + # NOTE: the 'enable_small_table_optimization_for_rbno' config option + # is supported only on Scylla versions newer than '2024.2' and '6.2'. + # See: https://github.com/scylladb/scylladb/pull/21207 + enable_small_table_optimization_for_rbno: true + enable_tablets: false + +nemesis_class_name: 'SisyphusMonkey' +nemesis_seed: '007' +nemesis_interval: 5 +nemesis_during_prepare: true +# NOTE: enable health checks back when following bug gets fixed: +# https://github.com/scylladb/scylla-cluster-tests/issues/9547 +cluster_health_check: false + +round_robin: true +prepare_write_cmd: + # NOTE: --duration in these commands is number of rows that will be written. + # Time gets specified with 's', 'm' or 'h' letters. + + # NOTE: [region-1, loaders 1-5] first group of loaders from a single/first region write to all per-region main tables + - >- + latte run --tag latte-prepare-01 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-02 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-03 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-04 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T15F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-05 --duration 50100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 25000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T16F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-2-5, loaders 1] other loaders write multi-region tables, first half + - >- + latte run --tag latte-prepare-06 --duration 250050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 30 --rate 1500 -P offset=0 + --function custom -P row_count=250050 -P codes="\"T1F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-07 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=550050 -P codes="\"T2F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-08 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=550050 -P codes="\"T3F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-09 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=550050 -P codes="\"T4F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 1-5] first group of loaders from a single/first region write to all per-region secondary tables + - >- + latte run --tag latte-prepare-10 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T6F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-11 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T7F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-12 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T8F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-13 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T9F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-14 --duration 1100100 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T10F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-2-5, loaders 1] other loaders write multi-region tables, second half + - >- + latte run --tag latte-prepare-15 --duration 250050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 30 --rate 1500 -P offset=250050 + --function custom -P row_count=250050 -P codes="\"T1F1\"" --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-16 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=550050 + --function custom -P row_count=550050 -P codes="\"T2F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-17 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=550050 + --function custom -P row_count=550050 -P codes="\"T3F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-prepare-18 --duration 550050 --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 28 --connections 2 --concurrency 56 --rate 560 -P offset=550050 + --function custom -P row_count=550050 -P codes="\"T4F1\"" + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + +stress_cmd: + # NOTE: [region-1, loaders 1-5] write to main per-dc tables from loaders of dc-1 + - >- + latte run --tag latte-main-01 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 12000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F1\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-02 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 12000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-03 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 12000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-04 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 12000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T15F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-05 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 30 --connections 3 --concurrency 180 --rate 12000 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T16F1\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-2, loaders 1] read dc-2 main table from dc-2 loader + - >- + latte run --tag latte-main-06 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 4800 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T13F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-3, loaders 1] read dc-3 main table from dc-3 loader + - >- + latte run --tag latte-main-07 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 4800 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T14F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-4, loaders 1] read dc-4 main table from dc-4 loader + - >- + latte run --tag latte-main-08 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 4800 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T15F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-5, loaders 1] read dc-5 main table from dc-5 loader + - >- + latte run --tag latte-main-09 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 4800 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T16F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 1] read dc-1 main table from dc-1 loader + - >- + latte run --tag latte-main-10 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 4800 -P offset=0 + --function custom -P row_count=50100100 -P codes="\"T12F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 2] read from special table with replication everywhere + - >- + latte run --tag latte-main-11 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=500100 -P codes="\"T1F3\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 3-4] run per-dc main table specific scenario functions with insertions and deletions for regions 1-2 + - >- + latte run --tag latte-main-12 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 440 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T12F7\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-13 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 440 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T13F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 5] read dc-1 secondary table from dc-1 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-14 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T6F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-2, loaders 1] read dc-2 secondary table from dc-2 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-15 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T7F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-3, loaders 1] read dc-3 secondary table from dc-3 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-16 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T8F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-4, loaders 1] read dc-4 secondary table from dc-4 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-17 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T9F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-5, loaders 1] read dc-5 secondary table from dc-5 loader and read 3 other multi-region tables + - >- + latte run --tag latte-main-18 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 24 --rate 192 -P offset=0 + --function custom -P row_count=1100100 -P codes="\"T10F3,T2F2,T3F3,T4F3\"" -P print_applied_func_names=2 + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + + # NOTE: [region-1, loaders 1-3] run per-dc main table specific scenario functions with insertions and deletions for regions 3-5 + - >- + latte run --tag latte-main-19 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 440 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T14F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-20 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 440 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T15F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn + - >- + latte run --tag latte-main-21 --duration 180m --request-timeout 60 --retry-interval '2s,10s' + --sampling 5s --threads 24 --connections 3 --concurrency 96 --rate 440 -P offset=50100100 + --function custom -P row_count=1100100 -P codes="\"T16F7\"" -P print_applied_func_names=2 --consistency=QUORUM + scylla-qa-internal/custom_d2/workload1/latte/custom_d2_workload1.rn diff --git a/unit_tests/test_scylla_yaml.py b/unit_tests/test_scylla_yaml.py index d678c5aac1..31821a25f8 100644 --- a/unit_tests/test_scylla_yaml.py +++ b/unit_tests/test_scylla_yaml.py @@ -243,6 +243,7 @@ def test_scylla_yaml(self): 'enable_repair_based_node_ops': None, 'allowed_repair_based_node_ops': None, 'enable_shard_aware_drivers': None, + 'enable_small_table_optimization_for_rbno': None, 'enable_sstable_data_integrity_check': None, 'enable_sstable_key_validation': None, 'enable_sstables_mc_format': None, diff --git a/utils/lint_test_cases.sh b/utils/lint_test_cases.sh index b58a6b2bce..89953e8de9 100755 --- a/utils/lint_test_cases.sh +++ b/utils/lint_test_cases.sh @@ -1,7 +1,7 @@ #! /bin/bash OUT=0 -SCT_AMI_ID_DB_SCYLLA=ami-1234 ./sct.py lint-yamls -i '.yaml' -e 'azure,multi-dc,multiDC,multidc,multiple-dc,rolling,docker,artifacts,private-repo,ics/long,scylla-operator,gce,jepsen,repair-based-operations,add-new-dc,baremetal' +SCT_AMI_ID_DB_SCYLLA=ami-1234 ./sct.py lint-yamls -i '.yaml' -e 'azure,multi-dc,multiDC,multidc,multiple-dc,3dcs,5dcs,rolling,docker,artifacts,private-repo,ics/long,scylla-operator,gce,jepsen,repair-based-operations,add-new-dc,baremetal' OUT=$(($OUT + $?)) SCT_AZURE_IMAGE_DB=image SCT_AZURE_REGION_NAME="eastus" ./sct.py lint-yamls --backend azure -i azure @@ -15,7 +15,11 @@ SCT_GCE_IMAGE_DB=image SCT_GCE_DATACENTER="us-east1 us-west1" SCT_SCYLLA_REPO='h OUT=$(($OUT + $?)) echo "multi dc yamls with 3 regions" -SCT_GCE_IMAGE_DB=image SCT_GCE_DATACENTER="us-east1 us-west1 eu-north1" SCT_SCYLLA_REPO='http://downloads.scylladb.com.s3.amazonaws.com/rpm/centos/scylla-2021.1.repo' ./sct.py lint-yamls -b gce -i '24h-multidc,large-cluster,counters-multidc,cdc-8h-multi-dc' -e 'docker,azure,shutdown' +SCT_GCE_IMAGE_DB=image SCT_GCE_DATACENTER="us-east1 us-west1 eu-north1" SCT_SCYLLA_REPO='http://downloads.scylladb.com.s3.amazonaws.com/rpm/centos/scylla-2021.1.repo' ./sct.py lint-yamls -b gce -i '3dcs,24h-multidc,large-cluster,counters-multidc,cdc-8h-multi-dc' -e 'docker,azure,shutdown' +OUT=$(($OUT + $?)) + +echo "multi dc yamls with 5 regions" +SCT_AMI_ID_DB_SCYLLA="ami-1 ami-2 ami-3 ami-4 ami-5" SCT_REGION_NAME="eu-central-1 eu-north-1 eu-west-1 eu-west-2 us-east-1" SCT_SCYLLA_REPO='http://downloads.scylladb.com.s3.amazonaws.com/rpm/centos/scylla-2021.1.repo' ./sct.py lint-yamls -b aws -i '5dcs' -e 'docker,azure,shutdown' OUT=$(($OUT + $?))