|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# This script split the integration tests into 16 groups to support parallel group tests execution. |
| 4 | +# all the integration tests are located in br/tests directory. only the directories |
| 5 | +# containing run.sh will be considered as integration tests. the script will print the total # # # number |
| 6 | + |
| 7 | +set -eo pipefail |
| 8 | + |
| 9 | +# Step 1 |
| 10 | +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) |
| 11 | +group=$1 |
| 12 | +export COV_DIR="/tmp/group_cover" |
| 13 | +rm -rf COV_DIR |
| 14 | +mkdir $COV_DIR |
| 15 | + |
| 16 | +# Define groups |
| 17 | +# Note: If new group is added, the group name must also be added to CI |
| 18 | +# * https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy |
| 19 | +# Each group of tests consumes as much time as possible, thus reducing CI waiting time. |
| 20 | +# Putting multiple light tests together and heavy tests in a separate group. |
| 21 | +declare -A groups |
| 22 | +groups=( |
| 23 | + ["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable" |
| 24 | + ["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full" |
| 25 | + ["G02"]="br_full_cluster_restore br_full_ddl br_full_index br_gcs br_history" |
| 26 | + ["G03"]='br_incompatible_tidb_config br_incremental br_incremental_ddl br_incremental_index' |
| 27 | + ["G04"]='br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other br_partition_add_index' |
| 28 | + ["G05"]='br_range br_rawkv br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table' |
| 29 | + ["G06"]='br_skip_checksum br_small_batch_size br_split_region_fail br_systables br_table_filter br_txn' |
| 30 | + ["G07"]='br_clustered_index br_crypter br_table_partition br_tidb_placement_policy br_tiflash br_tikv_outage' |
| 31 | + ["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint lightning_add_index lightning_alter_random lightning_auto_columns' |
| 32 | + ["G09"]='lightning_auto_random_default lightning_bom_file lightning_character_sets lightning_check_partial_imported lightning_checkpoint lightning_checkpoint_chunks lightning_checkpoint_columns lightning_checkpoint_dirty_tableid' |
| 33 | + ["G10"]='lightning_checkpoint_engines lightning_checkpoint_engines_order lightning_checkpoint_error_destroy lightning_checkpoint_parquet lightning_checkpoint_timestamp lightning_checksum_mismatch lightning_cmdline_override lightning_column_permutation lightning_common_handle' |
| 34 | + ["G11"]='lightning_compress lightning_concurrent-restore lightning_config_max_error lightning_config_skip_csv_header lightning_csv lightning_default-columns lightning_disable_scheduler_by_key_range lightning_disk_quota lightning_distributed_import' |
| 35 | + ["G12"]='lightning_drop_other_tables_halfway lightning_duplicate_detection lightning_duplicate_detection_new lightning_duplicate_resolution lightning_duplicate_resolution_incremental lightning_error_summary lightning_examples lightning_exotic_filenames lightning_extend_routes lightning_fail_fast' |
| 36 | + ["G13"]='lightning_fail_fast_on_nonretry_err lightning_file_routing lightning_foreign_key lightning_gcs lightning_generated_columns lightning_ignore_columns lightning_import_compress lightning_incremental lightning_issue_282' |
| 37 | + ["G14"]='lightning_issue_40657 lightning_issue_410 lightning_issue_519 lightning_local_backend lightning_max_incr lightning_max_random lightning_multi_valued_index lightning_new_collation lightning_no_schema' |
| 38 | + ["G15"]='lightning_parquet lightning_partition_incremental lightning_partitioned-table lightning_record_network lightning_reload_cert lightning_restore lightning_routes lightning_routes_panic lightning_row-format-v2 lightning_s3' |
| 39 | + ["G16"]='lightning_shard_rowid lightning_source_linkfile lightning_sqlmode lightning_tidb_duplicate_data lightning_tidb_rowid lightning_tiflash lightning_tikv_multi_rocksdb lightning_too_many_columns lightning_tool_135' |
| 40 | + ["G17"]='lightning_tool_1420 lightning_tool_1472 lightning_tool_241 lightning_ttl lightning_unused_config_keys lightning_various_types lightning_view lightning_write_batch lightning_write_limit' |
| 41 | +) |
| 42 | + |
| 43 | +# Get other cases not in groups, to avoid missing any case |
| 44 | +others=() |
| 45 | +for script in "$CUR"/*/run.sh; do |
| 46 | + test_name="$(basename "$(dirname "$script")")" |
| 47 | + # shellcheck disable=SC2076 |
| 48 | + if [[ ! " ${groups[*]} " =~ " ${test_name} " ]]; then |
| 49 | + others=("${others[@]} ${test_name}") |
| 50 | + fi |
| 51 | +done |
| 52 | + |
| 53 | +if [[ "$group" == "others" ]]; then |
| 54 | + if [[ -z $others ]]; then |
| 55 | + echo "All br&lightning integration test cases have been added to groups" |
| 56 | + exit 0 |
| 57 | + fi |
| 58 | + echo "Error: "$others" is not added to any group in br/tests/run_group.sh" |
| 59 | + exit 1 |
| 60 | +elif [[ " ${!groups[*]} " =~ " ${group} " ]]; then |
| 61 | + test_names="${groups[${group}]}" |
| 62 | + # Run test cases |
| 63 | + if [[ -n $test_names ]]; then |
| 64 | + echo "" |
| 65 | + echo "Run cases: ${test_names}" |
| 66 | + for case_name in $test_names; do |
| 67 | + echo "Run cases: ${case_name}" |
| 68 | + rm -rf /tmp/backup_restore_test |
| 69 | + mkdir -p /tmp/backup_restore_test |
| 70 | + TEST_NAME=${case_name} ${CUR}/run.sh |
| 71 | + done |
| 72 | + fi |
| 73 | +else |
| 74 | + echo "Error: invalid group name: ${group}" |
| 75 | + exit 1 |
| 76 | +fi |
0 commit comments