From 6f797d985cb3ac47040d22065af471240a31e15c Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Wed, 24 Oct 2018 13:55:12 +0200 Subject: [PATCH 01/29] Make ct a behaviour for test suites --- lib/common_test/src/ct.erl | 71 +++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index bfa7b25862eb..629067670e3f 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -86,8 +86,77 @@ {hosts,[key_or_name()]}. -type conn_log_type() :: raw | pretty | html | silent. -type conn_log_mod() :: ct_netconfc | ct_telnet. -%%---------------------------------------------------------------------- +%%------------------------------------------------------------------ +%% Test Suite Behaviour +%% ------------------------------------------------------------------ +-export_type([ct_testname/0, + ct_groupname/0, + ct_config/0, + ct_status/0, + ct_group_props/0, + ct_groupdef/0, + ct_subgroups/0, + ct_groupref/0 + ]). + +-type ct_testname() :: atom(). +-type ct_groupname() :: atom(). +-type ct_config() :: [{atom(), term()}]. +-type ct_status() :: ok | skipped | failed. +-type ct_group_props() :: [term()]. +-type ct_groupdef() :: + {ct_groupname(), ct_group_props(), + [ct_testname() | ct_groupdef() | {group,ct_groupname()}]}. +-type ct_subgroups() :: + {ct_groupname(), ct_group_props()} + | {ct_groupname(), ct_group_props(), ct_subgroups()}. +-type ct_groupref() :: + {group, ct_groupname()} + | {group, ct_groupname(), ct_group_props()} + | {group, ct_groupname(), ct_group_props(), ct_subgroups()}. + +-callback all() -> [ct_testname() | ct_groupref()] + | {skip, Reason::term()}. + +-callback groups() -> [ct_groupdef()]. + +-callback suite() -> [tuple()]. + +-callback init_per_suite(ct_config()) -> + NewConfig::ct_config() | {skip,Reason::term()} + | {skip_and_save,Reason::term(),ct_config()}. + +-callback end_per_suite(ct_config()) -> + term() | {save_config,ct_config()}. + +-callback group(ct_groupname()) -> [tuple()]. + +-callback init_per_group(ct_groupname(), ct_config()) -> + NewConfig::ct_config() | {skip,Reason::term()}. + +-callback end_per_group(ct_groupname(), ct_config()) -> + term() | {return_group_result, ct_status()}. + +-callback init_per_testcase(ct_testname(), ct_config()) -> + NewConfig::ct_config() | {fail,Reason::term()} | {skip,Reason::term()}. + +-callback end_per_testcase(ct_testname(), ct_config()) -> + term() | {fail,Reason::term()} | {save_config,ct_config()}. + +%% only all/0 is mandatory +-optional_callbacks([groups/0, + suite/0, + init_per_suite/1, + end_per_suite/1, + group/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 + ]). + +%%---------------------------------------------------------------------- install(Opts) -> ct_run:install(Opts). From 2132d78940900d91a9707dfad21256efb57851ed Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 22:04:21 +0100 Subject: [PATCH 02/29] Prepare to move behaviour-related code to new module ct_suite --- lib/common_test/src/ct.erl | 71 +------------------------------------- 1 file changed, 1 insertion(+), 70 deletions(-) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 629067670e3f..bfa7b25862eb 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -86,78 +86,9 @@ {hosts,[key_or_name()]}. -type conn_log_type() :: raw | pretty | html | silent. -type conn_log_mod() :: ct_netconfc | ct_telnet. - -%%------------------------------------------------------------------ -%% Test Suite Behaviour -%% ------------------------------------------------------------------ --export_type([ct_testname/0, - ct_groupname/0, - ct_config/0, - ct_status/0, - ct_group_props/0, - ct_groupdef/0, - ct_subgroups/0, - ct_groupref/0 - ]). - --type ct_testname() :: atom(). --type ct_groupname() :: atom(). --type ct_config() :: [{atom(), term()}]. --type ct_status() :: ok | skipped | failed. --type ct_group_props() :: [term()]. --type ct_groupdef() :: - {ct_groupname(), ct_group_props(), - [ct_testname() | ct_groupdef() | {group,ct_groupname()}]}. --type ct_subgroups() :: - {ct_groupname(), ct_group_props()} - | {ct_groupname(), ct_group_props(), ct_subgroups()}. --type ct_groupref() :: - {group, ct_groupname()} - | {group, ct_groupname(), ct_group_props()} - | {group, ct_groupname(), ct_group_props(), ct_subgroups()}. - --callback all() -> [ct_testname() | ct_groupref()] - | {skip, Reason::term()}. - --callback groups() -> [ct_groupdef()]. - --callback suite() -> [tuple()]. - --callback init_per_suite(ct_config()) -> - NewConfig::ct_config() | {skip,Reason::term()} - | {skip_and_save,Reason::term(),ct_config()}. - --callback end_per_suite(ct_config()) -> - term() | {save_config,ct_config()}. - --callback group(ct_groupname()) -> [tuple()]. - --callback init_per_group(ct_groupname(), ct_config()) -> - NewConfig::ct_config() | {skip,Reason::term()}. - --callback end_per_group(ct_groupname(), ct_config()) -> - term() | {return_group_result, ct_status()}. - --callback init_per_testcase(ct_testname(), ct_config()) -> - NewConfig::ct_config() | {fail,Reason::term()} | {skip,Reason::term()}. - --callback end_per_testcase(ct_testname(), ct_config()) -> - term() | {fail,Reason::term()} | {save_config,ct_config()}. - -%% only all/0 is mandatory --optional_callbacks([groups/0, - suite/0, - init_per_suite/1, - end_per_suite/1, - group/1, - init_per_group/2, - end_per_group/2, - init_per_testcase/2, - end_per_testcase/2 - ]). - %%---------------------------------------------------------------------- + install(Opts) -> ct_run:install(Opts). From 07c77edd9e0f9ff39cd18c7df3c9fae57a5a414b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 22:05:57 +0100 Subject: [PATCH 03/29] Copy richcarl's behaviour, as-is, to new behaviour-defining module ct_suite --- lib/common_test/src/ct_suite.erl | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 lib/common_test/src/ct_suite.erl diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl new file mode 100644 index 000000000000..4a0483e69cc5 --- /dev/null +++ b/lib/common_test/src/ct_suite.erl @@ -0,0 +1,70 @@ +-module(ct_suite). + +%%------------------------------------------------------------------ +%% Test Suite Behaviour +%% ------------------------------------------------------------------ +-export_type([ct_testname/0, + ct_groupname/0, + ct_config/0, + ct_status/0, + ct_group_props/0, + ct_groupdef/0, + ct_subgroups/0, + ct_groupref/0 + ]). + +-type ct_testname() :: atom(). +-type ct_groupname() :: atom(). +-type ct_config() :: [{atom(), term()}]. +-type ct_status() :: ok | skipped | failed. +-type ct_group_props() :: [term()]. +-type ct_groupdef() :: + {ct_groupname(), ct_group_props(), + [ct_testname() | ct_groupdef() | {group,ct_groupname()}]}. +-type ct_subgroups() :: + {ct_groupname(), ct_group_props()} + | {ct_groupname(), ct_group_props(), ct_subgroups()}. +-type ct_groupref() :: + {group, ct_groupname()} + | {group, ct_groupname(), ct_group_props()} + | {group, ct_groupname(), ct_group_props(), ct_subgroups()}. + +-callback all() -> [ct_testname() | ct_groupref()] + | {skip, Reason::term()}. + +-callback groups() -> [ct_groupdef()]. + +-callback suite() -> [tuple()]. + +-callback init_per_suite(ct_config()) -> + NewConfig::ct_config() | {skip,Reason::term()} + | {skip_and_save,Reason::term(),ct_config()}. + +-callback end_per_suite(ct_config()) -> + term() | {save_config,ct_config()}. + +-callback group(ct_groupname()) -> [tuple()]. + +-callback init_per_group(ct_groupname(), ct_config()) -> + NewConfig::ct_config() | {skip,Reason::term()}. + +-callback end_per_group(ct_groupname(), ct_config()) -> + term() | {return_group_result, ct_status()}. + +-callback init_per_testcase(ct_testname(), ct_config()) -> + NewConfig::ct_config() | {fail,Reason::term()} | {skip,Reason::term()}. + +-callback end_per_testcase(ct_testname(), ct_config()) -> + term() | {fail,Reason::term()} | {save_config,ct_config()}. + +%% only all/0 is mandatory +-optional_callbacks([groups/0, + suite/0, + init_per_suite/1, + end_per_suite/1, + group/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 + ]). From 27c2d4f8a0ffe0971604958a52d04f39cc74a020 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:04:56 +0100 Subject: [PATCH 04/29] Improve -spec(_).s by using doc. as reference (also add missing elements) --- lib/common_test/src/ct_suite.erl | 56 +++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 4a0483e69cc5..6e3ad7f605d7 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -17,10 +17,21 @@ -type ct_groupname() :: atom(). -type ct_config() :: [{atom(), term()}]. -type ct_status() :: ok | skipped | failed. --type ct_group_props() :: [term()]. +-type ct_group_props() :: [parallel | sequence | ct_shuffle() | {ct_group_repeat_type(), ct_test_repeat()}] | default. +-type ct_shuffle() :: shuffle | {shuffle, ct_shuffle_seed()}. +-type ct_shuffle_seed() :: {integer(), integer(), integer()}. +-type ct_group_repeat_type() :: + repeat + | repeat_until_all_ok + | repeat_until_all_fail + | repeat_until_any_ok + | repeat_until_any_fail. +-type ct_test_repeat() :: + integer() + | forever. -type ct_groupdef() :: {ct_groupname(), ct_group_props(), - [ct_testname() | ct_groupdef() | {group,ct_groupname()}]}. + [ct_testname() | ct_groupdef() | {group, ct_groupname()} | ct_testcase_ref()]}. -type ct_subgroups() :: {ct_groupname(), ct_group_props()} | {ct_groupname(), ct_group_props(), ct_subgroups()}. @@ -28,13 +39,48 @@ {group, ct_groupname()} | {group, ct_groupname(), ct_group_props()} | {group, ct_groupname(), ct_group_props(), ct_subgroups()}. +-type ct_testcase_ref() :: + {testcase, ct_testname(), ct_testcase_repeat_props()}. +-type ct_testcase_repeat_props() :: + {repeat, ct_test_repeat()} + | {repeat_until_ok, ct_test_repeat()} + | {repeat_until_fail, ct_test_repeat()}. +-type ct_group_info() :: + {timetrap, ct_group_info_timetrap()} + | {require, ct_group_info_required()} + | {require, Name :: atom(), ct_group_info_required()} + | {userdata, UserData :: term()} + | {silent_connections, Conns :: [atom()]} + | {stylesheet, CSSFile :: string()} + | {ct_hooks, CTHs :: ct_hooks()}. +-type ct_group_info_timetrap() :: + MilliSec :: integer() + | {seconds, integer()} + | {minutes, integer()} + | {hours, integer()} + | {Mod :: module(), Func :: atom(), Args :: list()} + | ct_group_info_timetrap_fun(). +-type ct_group_info_timetrap_fun() :: fun(). +-type ct_group_info_required() :: + Key :: atom() + | {Key :: atom(), SubKeys :: ct_group_info_required_subkeys()} + | {Key :: atom(), Subkey :: atom()} + | {Key :: atom(), Subkey :: atom(), SubKeys :: ct_group_info_required_subkeys()}. +-type ct_group_info_required_subkeys() :: + SubKey :: atom() + | [SubKey :: atom()]. +-type ct_hooks() :: [ + CTHModule :: atom() + | {CTHModule :: atom(), CTHInitArgs :: term()} + | {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} +]. --callback all() -> [ct_testname() | ct_groupref()] +-callback all() -> [ct_testname() | ct_groupref() | ct_testcase_ref()] | {skip, Reason::term()}. -callback groups() -> [ct_groupdef()]. --callback suite() -> [tuple()]. +-callback suite() -> [ct_group_info()]. -callback init_per_suite(ct_config()) -> NewConfig::ct_config() | {skip,Reason::term()} @@ -43,7 +89,7 @@ -callback end_per_suite(ct_config()) -> term() | {save_config,ct_config()}. --callback group(ct_groupname()) -> [tuple()]. +-callback group(ct_groupname()) -> [ct_group_info()]. -callback init_per_group(ct_groupname(), ct_config()) -> NewConfig::ct_config() | {skip,Reason::term()}. From 5a21b49f5e0a02baed2c586383b5e14ba56fc510 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:16:21 +0100 Subject: [PATCH 05/29] Format as per other elements in common_test (everything except -callback) --- lib/common_test/src/ct_suite.erl | 112 ++++++++++++++++--------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 6e3ad7f605d7..b1eb05f3bc09 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -16,64 +16,66 @@ -type ct_testname() :: atom(). -type ct_groupname() :: atom(). -type ct_config() :: [{atom(), term()}]. --type ct_status() :: ok | skipped | failed. --type ct_group_props() :: [parallel | sequence | ct_shuffle() | {ct_group_repeat_type(), ct_test_repeat()}] | default. --type ct_shuffle() :: shuffle | {shuffle, ct_shuffle_seed()}. +-type ct_status() :: ok | + skipped | + failed. +-type ct_group_props() :: [ + parallel | + sequence | + ct_shuffle() | + {ct_group_repeat_type(), ct_test_repeat()} + ] | + default. +-type ct_shuffle() :: shuffle | + {shuffle, ct_shuffle_seed()}. -type ct_shuffle_seed() :: {integer(), integer(), integer()}. --type ct_group_repeat_type() :: - repeat - | repeat_until_all_ok - | repeat_until_all_fail - | repeat_until_any_ok - | repeat_until_any_fail. --type ct_test_repeat() :: - integer() - | forever. --type ct_groupdef() :: - {ct_groupname(), ct_group_props(), - [ct_testname() | ct_groupdef() | {group, ct_groupname()} | ct_testcase_ref()]}. --type ct_subgroups() :: - {ct_groupname(), ct_group_props()} - | {ct_groupname(), ct_group_props(), ct_subgroups()}. --type ct_groupref() :: - {group, ct_groupname()} - | {group, ct_groupname(), ct_group_props()} - | {group, ct_groupname(), ct_group_props(), ct_subgroups()}. --type ct_testcase_ref() :: - {testcase, ct_testname(), ct_testcase_repeat_props()}. --type ct_testcase_repeat_props() :: - {repeat, ct_test_repeat()} - | {repeat_until_ok, ct_test_repeat()} - | {repeat_until_fail, ct_test_repeat()}. --type ct_group_info() :: - {timetrap, ct_group_info_timetrap()} - | {require, ct_group_info_required()} - | {require, Name :: atom(), ct_group_info_required()} - | {userdata, UserData :: term()} - | {silent_connections, Conns :: [atom()]} - | {stylesheet, CSSFile :: string()} - | {ct_hooks, CTHs :: ct_hooks()}. --type ct_group_info_timetrap() :: - MilliSec :: integer() - | {seconds, integer()} - | {minutes, integer()} - | {hours, integer()} - | {Mod :: module(), Func :: atom(), Args :: list()} - | ct_group_info_timetrap_fun(). +-type ct_group_repeat_type() :: repeat | + repeat_until_all_ok | + repeat_until_all_fail | + repeat_until_any_ok | + repeat_until_any_fail. +-type ct_test_repeat() :: integer() | + forever. +-type ct_groupdef() :: {ct_groupname(), ct_group_props(), [ + ct_testname() | + ct_groupdef() | + {group, ct_groupname()} | + ct_testcase_ref() + ]}. +-type ct_subgroups() :: {ct_groupname(), ct_group_props()} | + {ct_groupname(), ct_group_props(), ct_subgroups()}. +-type ct_groupref() :: {group, ct_groupname()} | + {group, ct_groupname(), ct_group_props()} | + {group, ct_groupname(), ct_group_props(), ct_subgroups()}. +-type ct_testcase_ref() :: {testcase, ct_testname(), ct_testcase_repeat_props()}. +-type ct_testcase_repeat_props() :: {repeat, ct_test_repeat()} | + {repeat_until_ok, ct_test_repeat()} | + {repeat_until_fail, ct_test_repeat()}. +-type ct_group_info() :: {timetrap, ct_group_info_timetrap()} | + {require, ct_group_info_required()} | + {require, Name :: atom(), ct_group_info_required()} | + {userdata, UserData :: term()} | + {silent_connections, Conns :: [atom()]} | + {stylesheet, CSSFile :: string()} | + {ct_hooks, CTHs :: ct_hooks()}. +-type ct_group_info_timetrap() :: MilliSec :: integer() | + {seconds, integer()} | + {minutes, integer()} | + {hours, integer()} | + {Mod :: module(), Func :: atom(), Args :: list()} | + ct_group_info_timetrap_fun(). -type ct_group_info_timetrap_fun() :: fun(). --type ct_group_info_required() :: - Key :: atom() - | {Key :: atom(), SubKeys :: ct_group_info_required_subkeys()} - | {Key :: atom(), Subkey :: atom()} - | {Key :: atom(), Subkey :: atom(), SubKeys :: ct_group_info_required_subkeys()}. --type ct_group_info_required_subkeys() :: - SubKey :: atom() - | [SubKey :: atom()]. +-type ct_group_info_required() :: Key :: atom() | + {Key :: atom(), SubKeys :: ct_group_info_required_subkeys()} | + {Key :: atom(), Subkey :: atom()} | + {Key :: atom(), Subkey :: atom(), SubKeys :: ct_group_info_required_subkeys()}. +-type ct_group_info_required_subkeys() :: SubKey :: atom() | + [SubKey :: atom()]. -type ct_hooks() :: [ - CTHModule :: atom() - | {CTHModule :: atom(), CTHInitArgs :: term()} - | {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} -]. + CTHModule :: atom() | + {CTHModule :: atom(), CTHInitArgs :: term()} | + {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} + ]. -callback all() -> [ct_testname() | ct_groupref() | ct_testcase_ref()] | {skip, Reason::term()}. From 5a7b1ee2026b9b6f0013280d2e749c43f1660ed4 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:18:56 +0100 Subject: [PATCH 06/29] Format -callback elements as per other behaviour-defining modules in the project --- lib/common_test/src/ct_suite.erl | 36 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index b1eb05f3bc09..103b8f473868 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -77,33 +77,45 @@ {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} ]. --callback all() -> [ct_testname() | ct_groupref() | ct_testcase_ref()] - | {skip, Reason::term()}. +-callback all() -> + [ct_testname() | ct_groupref() | ct_testcase_ref()] | + {skip, Reason::term()}. --callback groups() -> [ct_groupdef()]. +-callback groups() -> + [ct_groupdef()]. --callback suite() -> [ct_group_info()]. +-callback suite() -> + [ct_group_info()]. -callback init_per_suite(ct_config()) -> - NewConfig::ct_config() | {skip,Reason::term()} - | {skip_and_save,Reason::term(),ct_config()}. + NewConfig::ct_config() | + {skip,Reason::term()} | + {skip_and_save,Reason::term(),ct_config()}. -callback end_per_suite(ct_config()) -> - term() | {save_config,ct_config()}. + term() | + {save_config,ct_config()}. --callback group(ct_groupname()) -> [ct_group_info()]. +-callback group(ct_groupname()) -> + [ct_group_info()]. -callback init_per_group(ct_groupname(), ct_config()) -> - NewConfig::ct_config() | {skip,Reason::term()}. + NewConfig::ct_config() | + {skip,Reason::term()}. -callback end_per_group(ct_groupname(), ct_config()) -> - term() | {return_group_result, ct_status()}. + term() | + {return_group_result, ct_status()}. -callback init_per_testcase(ct_testname(), ct_config()) -> - NewConfig::ct_config() | {fail,Reason::term()} | {skip,Reason::term()}. + NewConfig::ct_config() | + {fail,Reason::term()} | + {skip,Reason::term()}. -callback end_per_testcase(ct_testname(), ct_config()) -> - term() | {fail,Reason::term()} | {save_config,ct_config()}. + term() | + {fail,Reason::term()} | + {save_config,ct_config()}. %% only all/0 is mandatory -optional_callbacks([groups/0, From b5825310a2f8caea674beb222ef0e1203ef8445e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:20:14 +0100 Subject: [PATCH 07/29] Further tweak the format --- lib/common_test/src/ct_suite.erl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 103b8f473868..cc37f9bd405e 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -79,7 +79,7 @@ -callback all() -> [ct_testname() | ct_groupref() | ct_testcase_ref()] | - {skip, Reason::term()}. + {skip, Reason :: term()}. -callback groups() -> [ct_groupdef()]. @@ -88,34 +88,34 @@ [ct_group_info()]. -callback init_per_suite(ct_config()) -> - NewConfig::ct_config() | - {skip,Reason::term()} | - {skip_and_save,Reason::term(),ct_config()}. + NewConfig :: ct_config() | + {skip, Reason :: term()} | + {skip_and_save, Reason :: term(), ct_config()}. -callback end_per_suite(ct_config()) -> term() | - {save_config,ct_config()}. + {save_config, ct_config()}. -callback group(ct_groupname()) -> [ct_group_info()]. -callback init_per_group(ct_groupname(), ct_config()) -> - NewConfig::ct_config() | - {skip,Reason::term()}. + NewConfig :: ct_config() | + {skip, Reason :: term()}. -callback end_per_group(ct_groupname(), ct_config()) -> term() | {return_group_result, ct_status()}. -callback init_per_testcase(ct_testname(), ct_config()) -> - NewConfig::ct_config() | - {fail,Reason::term()} | - {skip,Reason::term()}. + NewConfig :: ct_config() | + {fail, Reason :: term()} | + {skip, Reason :: term()}. -callback end_per_testcase(ct_testname(), ct_config()) -> term() | - {fail,Reason::term()} | - {save_config,ct_config()}. + {fail, Reason :: term()} | + {save_config, ct_config()}. %% only all/0 is mandatory -optional_callbacks([groups/0, From d9241db76d35dac409052083ba5227475dc6574e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:35:09 +0100 Subject: [PATCH 08/29] Re-read the online doc. and name input accordingly (eases reference) --- lib/common_test/src/ct_suite.erl | 75 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index cc37f9bd405e..bfbb5a1ea2be 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -8,27 +8,25 @@ ct_config/0, ct_status/0, ct_group_props/0, - ct_groupdef/0, + ct_groups_def/0, ct_subgroups/0, - ct_groupref/0 + ct_group_ref/0, ]). -type ct_testname() :: atom(). -type ct_groupname() :: atom(). --type ct_config() :: [{atom(), term()}]. +-type ct_config() :: [{Key :: atom(), Value :: term()}]. -type ct_status() :: ok | skipped | failed. -type ct_group_props() :: [ parallel | sequence | - ct_shuffle() | + shuffle | + {shuffle, Seed :: {integer(), integer(), integer()}} | {ct_group_repeat_type(), ct_test_repeat()} ] | default. --type ct_shuffle() :: shuffle | - {shuffle, ct_shuffle_seed()}. --type ct_shuffle_seed() :: {integer(), integer(), integer()}. -type ct_group_repeat_type() :: repeat | repeat_until_all_ok | repeat_until_all_fail | @@ -36,86 +34,87 @@ repeat_until_any_fail. -type ct_test_repeat() :: integer() | forever. --type ct_groupdef() :: {ct_groupname(), ct_group_props(), [ +-type ct_groups_def() :: {ct_groupname(), ct_group_props(), [ ct_testname() | - ct_groupdef() | + ct_groups_def() | {group, ct_groupname()} | ct_testcase_ref() ]}. --type ct_subgroups() :: {ct_groupname(), ct_group_props()} | - {ct_groupname(), ct_group_props(), ct_subgroups()}. --type ct_groupref() :: {group, ct_groupname()} | +-type ct_subgroups_def() :: {ct_groupname(), ct_group_props()} | + {ct_groupname(), ct_group_props(), ct_subgroups_def()}. +-type ct_group_ref() :: {group, ct_groupname()} | {group, ct_groupname(), ct_group_props()} | - {group, ct_groupname(), ct_group_props(), ct_subgroups()}. + {group, ct_groupname(), ct_group_props(), ct_subgroups_def()}. -type ct_testcase_ref() :: {testcase, ct_testname(), ct_testcase_repeat_props()}. -type ct_testcase_repeat_props() :: {repeat, ct_test_repeat()} | {repeat_until_ok, ct_test_repeat()} | {repeat_until_fail, ct_test_repeat()}. --type ct_group_info() :: {timetrap, ct_group_info_timetrap()} | - {require, ct_group_info_required()} | - {require, Name :: atom(), ct_group_info_required()} | +-type ct_info() :: {timetrap, ct_info_timetrap()} | + {require, ct_info_required()} | + {require, Name :: atom(), ct_info_required()} | {userdata, UserData :: term()} | {silent_connections, Conns :: [atom()]} | {stylesheet, CSSFile :: string()} | {ct_hooks, CTHs :: ct_hooks()}. --type ct_group_info_timetrap() :: MilliSec :: integer() | +-type ct_info_timetrap() :: MilliSec :: integer() | {seconds, integer()} | {minutes, integer()} | {hours, integer()} | {Mod :: module(), Func :: atom(), Args :: list()} | - ct_group_info_timetrap_fun(). --type ct_group_info_timetrap_fun() :: fun(). --type ct_group_info_required() :: Key :: atom() | - {Key :: atom(), SubKeys :: ct_group_info_required_subkeys()} | + ct_info_timetrap_fun(). +-type ct_info_timetrap_fun() :: fun(). +-type ct_info_required() :: Key :: atom() | + {Key :: atom(), SubKeys :: ct_info_required_subkeys()} | {Key :: atom(), Subkey :: atom()} | - {Key :: atom(), Subkey :: atom(), SubKeys :: ct_group_info_required_subkeys()}. --type ct_group_info_required_subkeys() :: SubKey :: atom() | + {Key :: atom(), Subkey :: atom(), SubKeys :: ct_info_required_subkeys()}. +-type ct_info_required_subkeys() :: SubKey :: atom() | [SubKey :: atom()]. -type ct_hooks() :: [ CTHModule :: atom() | {CTHModule :: atom(), CTHInitArgs :: term()} | {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} ]. +-type ct_tests_def() :: ct_testname() | ct_group_ref() | ct_testcase_ref(). -callback all() -> - [ct_testname() | ct_groupref() | ct_testcase_ref()] | + Tests :: [ct_tests_def()] | {skip, Reason :: term()}. -callback groups() -> - [ct_groupdef()]. + GroupDefs :: [ct_groups_def()]. -callback suite() -> - [ct_group_info()]. + Info :: [ct_info()]. --callback init_per_suite(ct_config()) -> +-callback init_per_suite(Config :: ct_config()) -> NewConfig :: ct_config() | {skip, Reason :: term()} | - {skip_and_save, Reason :: term(), ct_config()}. + {skip_and_save, Reason :: term(), SaveConfig :: ct_config()}. --callback end_per_suite(ct_config()) -> +-callback end_per_suite(Config :: ct_config()) -> term() | - {save_config, ct_config()}. + {save_config, SaveConfig :: ct_config()}. --callback group(ct_groupname()) -> - [ct_group_info()]. +-callback group(GroupName :: ct_groupname()) -> + [Info :: ct_info()]. --callback init_per_group(ct_groupname(), ct_config()) -> +-callback init_per_group(GroupName :: ct_groupname(), Config :: ct_config()) -> NewConfig :: ct_config() | {skip, Reason :: term()}. --callback end_per_group(ct_groupname(), ct_config()) -> +-callback end_per_group(GroupName :: ct_groupname(), Config :: ct_config()) -> term() | - {return_group_result, ct_status()}. + {return_group_result, Status :: ct_status()}. --callback init_per_testcase(ct_testname(), ct_config()) -> +-callback init_per_testcase(TestCase :: ct_testname(), Config :: ct_config()) -> NewConfig :: ct_config() | {fail, Reason :: term()} | {skip, Reason :: term()}. --callback end_per_testcase(ct_testname(), ct_config()) -> +-callback end_per_testcase(TestCase :: ct_testname(), Config :: ct_config()) -> term() | {fail, Reason :: term()} | - {save_config, ct_config()}. + {save_config, SaveConfig :: ct_config()}. %% only all/0 is mandatory -optional_callbacks([groups/0, From 713e0a0cc3bb2a439d3f5b28292119c7b8b01b41 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:37:09 +0100 Subject: [PATCH 09/29] Export types only if referenced by -callback (we might decide to still export some more - go down a level) --- lib/common_test/src/ct_suite.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index bfbb5a1ea2be..bfd3bfc3d4d2 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -7,10 +7,10 @@ ct_groupname/0, ct_config/0, ct_status/0, - ct_group_props/0, ct_groups_def/0, - ct_subgroups/0, ct_group_ref/0, + ct_tests_def/0, + ct_info/0 ]). -type ct_testname() :: atom(). From e0c43c9f51b785045fb4a3e35b82d80734a2bd8b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 9 Oct 2020 23:38:50 +0100 Subject: [PATCH 10/29] Export some more types as per previous commit's "promise" --- lib/common_test/src/ct_suite.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index bfd3bfc3d4d2..93a82a65b64c 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -8,7 +8,10 @@ ct_config/0, ct_status/0, ct_groups_def/0, + ct_group_props/0, + ct_testcase_ref/0, ct_group_ref/0, + ct_subgroups_def/0, ct_tests_def/0, ct_info/0 ]). From 4bb5aebe191e2b413a82b545fada896caaa3615e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 13 Oct 2020 11:39:12 +0100 Subject: [PATCH 11/29] Segregate `default` from Module:groups() --- lib/common_test/src/ct_suite.erl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 93a82a65b64c..20fab3361e41 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -28,7 +28,9 @@ shuffle | {shuffle, Seed :: {integer(), integer(), integer()}} | {ct_group_repeat_type(), ct_test_repeat()} - ] | + ]. +-type ct_group_props_ref() :: + ct_group_props() | default. -type ct_group_repeat_type() :: repeat | repeat_until_all_ok | @@ -43,11 +45,11 @@ {group, ct_groupname()} | ct_testcase_ref() ]}. --type ct_subgroups_def() :: {ct_groupname(), ct_group_props()} | - {ct_groupname(), ct_group_props(), ct_subgroups_def()}. +-type ct_subgroups_def() :: {ct_groupname(), ct_group_props_ref()} | + {ct_groupname(), ct_group_props_ref(), ct_subgroups_def()}. -type ct_group_ref() :: {group, ct_groupname()} | - {group, ct_groupname(), ct_group_props()} | - {group, ct_groupname(), ct_group_props(), ct_subgroups_def()}. + {group, ct_groupname(), ct_group_props_ref()} | + {group, ct_groupname(), ct_group_props_ref(), ct_subgroups_def()}. -type ct_testcase_ref() :: {testcase, ct_testname(), ct_testcase_repeat_props()}. -type ct_testcase_repeat_props() :: {repeat, ct_test_repeat()} | {repeat_until_ok, ct_test_repeat()} | From a720ef3ba94c02b08667e695df923791519ca1b0 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 13 Oct 2020 13:53:30 +0100 Subject: [PATCH 12/29] Fix CTHPriority's type --- lib/common_test/src/ct_suite.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 20fab3361e41..caa5de3ffa77 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -77,7 +77,7 @@ -type ct_hooks() :: [ CTHModule :: atom() | {CTHModule :: atom(), CTHInitArgs :: term()} | - {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: term()} + {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: integer()} ]. -type ct_tests_def() :: ct_testname() | ct_group_ref() | ct_testcase_ref(). From fa7e27ace4f1f3f58db23d6b9c9c0d2455e30fdc Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 12:38:40 +0100 Subject: [PATCH 13/29] Add Makefile info (as per ssh/src - example) --- lib/common_test/src/Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index a20de14e0e3e..7913512b1beb 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -40,6 +40,9 @@ RELSYSDIR = $(RELEASE_PATH)/lib/common_test-$(VSN) # Target Specs # ---------------------------------------------------- +BEHAVIOUR_MODULES= \ + ct_suite + MODULES= \ ct \ ct_logs \ @@ -90,7 +93,10 @@ MODULES= \ TARGET_MODULES= $(MODULES:%=$(EBIN)/%) BEAM_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) -ERL_FILES= $(MODULES:=.erl) +ERL_FILES= \ + $(MODULES:=.erl) \ + $(BEHAVIOUR_MODULES:%=%.erl) + HRL_FILES = \ ct_util.hrl \ ct_netconfc.hrl @@ -118,6 +124,8 @@ TARGET_FILES = \ $(BEAM_FILES) \ $(APP_TARGET) $(APPUP_TARGET) +BEHAVIOUR_TARGET_FILES= $(BEHAVIOUR_MODULES:%=$(EBIN)/%.$(EMULATOR)) + APP_FILE= common_test.app APP_SRC= $(APP_FILE).src APP_TARGET= $(EBIN)/$(APP_FILE) @@ -152,7 +160,7 @@ release_spec: opt $(INSTALL_DIR) "$(RELSYSDIR)/src" $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) "$(RELSYSDIR)/src" $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(TARGET_FILES) "$(RELSYSDIR)/ebin" + $(INSTALL_DATA) $(BEHAVIOUR_TARGET_FILES) $(TARGET_FILES) "$(RELSYSDIR)/ebin" $(INSTALL_DIR) "$(RELSYSDIR)/include" $(INSTALL_DATA) $(EXTERNAL_HRL_FILES) "$(RELSYSDIR)/include" From f368e575456a763deeb1e9b4859df801e1fd0cf7 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 14:08:55 +0100 Subject: [PATCH 14/29] Approach code and documentation --- lib/common_test/src/ct_suite.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index caa5de3ffa77..5e111b22dda8 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -65,13 +65,13 @@ {seconds, integer()} | {minutes, integer()} | {hours, integer()} | - {Mod :: module(), Func :: atom(), Args :: list()} | + {Mod :: atom(), Func :: atom(), Args :: list()} | ct_info_timetrap_fun(). -type ct_info_timetrap_fun() :: fun(). -type ct_info_required() :: Key :: atom() | {Key :: atom(), SubKeys :: ct_info_required_subkeys()} | - {Key :: atom(), Subkey :: atom()} | - {Key :: atom(), Subkey :: atom(), SubKeys :: ct_info_required_subkeys()}. + {Key :: atom(), SubKey :: atom()} | + {Key :: atom(), SubKey :: atom(), SubKeys :: ct_info_required_subkeys()}. -type ct_info_required_subkeys() :: SubKey :: atom() | [SubKey :: atom()]. -type ct_hooks() :: [ @@ -82,14 +82,14 @@ -type ct_tests_def() :: ct_testname() | ct_group_ref() | ct_testcase_ref(). -callback all() -> - Tests :: [ct_tests_def()] | + [TestDef :: ct_tests_def()] | {skip, Reason :: term()}. -callback groups() -> - GroupDefs :: [ct_groups_def()]. + [GroupDef :: ct_groups_def()]. -callback suite() -> - Info :: [ct_info()]. + [Info :: ct_info()]. -callback init_per_suite(Config :: ct_config()) -> NewConfig :: ct_config() | From bbe9eacaf3f84036257fe3bedeb9c5849f6aff3e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 14:26:03 +0100 Subject: [PATCH 15/29] Conform to type-export policy (only input and output) --- lib/common_test/src/ct_suite.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index 5e111b22dda8..eca9996cac67 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -8,10 +8,6 @@ ct_config/0, ct_status/0, ct_groups_def/0, - ct_group_props/0, - ct_testcase_ref/0, - ct_group_ref/0, - ct_subgroups_def/0, ct_tests_def/0, ct_info/0 ]). From d7f8a18d3a417a1b83b35cffe7e81a37cd422332 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 14:54:23 +0100 Subject: [PATCH 16/29] Tweak common_test_app.xml (and fix some issue while at it) --- lib/common_test/doc/src/common_test_app.xml | 123 ++++++++++---------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 3fcbda538aa7..090097fde66d 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -69,19 +69,19 @@ for a test suite.

- Module:all() -> Tests | {skip,Reason} + Module:all() -> [TestDef] | {skip, Reason} Returns the list of all test case groups and test cases in the module. - Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] + TestDef = TestCase | {group, GroupName} | {group, GroupName, Properties} | {group, GroupName, Properties, SubGroups} TestCase = atom() TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] GroupName = atom() - Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default - SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}] - Shuffle = shuffle | {shuffle,Seed} - Seed = {integer(),integer(),integer()} - GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] | default + SubGroups = [{GroupName, Properties} | {GroupName, Properties, SubGroups}] + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever Reason = term() @@ -106,7 +106,7 @@ (With value default, the group definition properties are used).

-

If {skip,Reason} is returned, all test cases +

If {skip, Reason} is returned, all test cases in the module are skipped and Reason is printed on the HTML result page.

@@ -118,19 +118,17 @@ - Module:groups() -> GroupDefs + Module:groups() -> [GroupDef] Returns a list of test case group definitions. - GroupDefs = [Group] - Group = {GroupName,Properties,GroupsAndTestCases} + GroupDef = {GroupName, Properties, GroupsAndTestCases} GroupName = atom() - Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] - GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}] + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] + GroupsAndTestCases = [Group | {group, GroupName} | TestCase] TestCase = atom() - TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] - Shuffle = shuffle | {shuffle,Seed} - Seed = {integer(),integer(),integer()} - GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever @@ -144,20 +142,20 @@ - Module:suite() -> [Info] + Module:suite() -> [Info] Test suite info function (providing default data for the suite). - Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} | {stylesheet,CSSFile} | {ct_hooks, CTHs} + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()} - TimeFunc = {Mod,Func,Args} | Fun + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun MilliSec = integer() Mod = atom() Func = atom() Args = list() Fun = fun() - Required = Key | {Key,SubKeys} | {Key,SubKey} | {Key,SubKey,SubKeys} + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} Key = atom() SubKeys = SubKey | [SubKey] SubKey = atom() @@ -170,6 +168,7 @@         {CTHModule, CTHInitArgs, CTHPriority}] CTHModule = atom() CTHInitArgs = term() + CTHPriority = integer() @@ -217,11 +216,11 @@ - Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | - {skip_and_save,Reason,SaveConfig} + Module:init_per_suite(Config) -> NewConfig | {skip, Reason} | + {skip_and_save, Reason, SaveConfig} Test suite initializations. - Config = NewConfig = SaveConfig = [{Key,Value}] + Config = NewConfig = SaveConfig = [{Key, Value}] Key = atom() Value = term() Reason = term() @@ -240,7 +239,7 @@ function is specified as Config to all configuration functions and test cases in the suite.

-

If {skip,Reason} +

If {skip, Reason} is returned, all test cases in the suite are skipped and Reason is printed in the overview log for the suite.

@@ -252,11 +251,11 @@ - Module:end_per_suite(Config) -> term() | - {save_config,SaveConfig} + Module:end_per_suite(Config) -> term() | + {save_config, SaveConfig} Test suite finalization. - Config = SaveConfig = [{Key,Value}] + Config = SaveConfig = [{Key, Value}] Key = atom() Value = term() @@ -276,21 +275,22 @@ - Module:group(GroupName) -> [Info] + Module:group(GroupName) -> [Info] Test case group information function (providing default data for a test case group, that is, its test cases and subgroups). - Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} | {stylesheet,CSSFile} | {ct_hooks, CTHs} + GroupName = atom() + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()} - TimeFunc = {Mod,Func,Args} | Fun + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun MilliSec = integer() Mod = atom() Func = atom() Args = list() Fun = fun() - Required = Key | {Key,SubKeys} | {Key,Subkey} | {Key,Subkey,SubKeys} + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} Key = atom() SubKeys = SubKey | [SubKey] SubKey = atom() @@ -303,6 +303,7 @@         {CTHModule, CTHInitArgs, CTHPriority}] CTHModule = atom() CTHInitArgs = term() + CTHPriority = integer() @@ -357,11 +358,11 @@ Module:init_per_group(GroupName, Config) -> NewConfig | - {skip,Reason} + {skip, Reason} Test case group initializations. GroupName = atom() - Config = NewConfig = [{Key,Value}] + Config = NewConfig = [{Key, Value}] Key = atom() Value = term() Reason = term() @@ -383,7 +384,7 @@ The return value of this function is given as Config to all test cases and subgroups in the group.

-

If {skip,Reason} +

If {skip, Reason} is returned, all test cases in the group are skipped and Reason is printed in the overview log for the group.

@@ -395,11 +396,11 @@ Module:end_per_group(GroupName, Config) -> term() | - {return_group_result,Status} + {return_group_result, Status} Test case group finalization. GroupName = atom() - Config = [{Key,Value}] + Config = [{Key, Value}] Key = atom() Value = term() Status = ok | skipped | failed @@ -414,7 +415,7 @@ is finished. It is meant to be used for cleaning up after init_per_group/2. A status value for a nested subgroup can be returned with - {return_group_result,Status}. The status can be retrieved in + {return_group_result, Status}. The status can be retrieved in end_per_group/2 for the group on the level above. The status is also used by Common Test for deciding if execution of a group is to @@ -428,14 +429,14 @@ - Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail,Reason} | {skip,Reason} + Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} Test case initializations. - TestCase = atom() - Config = NewConfig = [{Key,Value}] - Key = atom() - Value = term() - Reason = term() + TestCase = atom() + Config = NewConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() @@ -449,20 +450,20 @@ Config (list of key-value tuples) is the configuration data that can be modified. The NewConfig list returned from this function is given as Config to the test case. - If {fail,Reason} is returned, the test case is + If {fail, Reason} is returned, the test case is marked as failed without being executed.

-

If {skip,Reason} is returned, the test case is skipped +

If {skip, Reason} is returned, the test case is skipped and Reason is printed in the overview log for the suite.

 - Module:end_per_testcase(TestCase, Config) -> term() | {fail,Reason} | {save_config,SaveConfig} + Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} Test case finalization. TestCase = atom() - Config = SaveConfig = [{Key,Value}] + Config = SaveConfig = [{Key, Value}] Key = atom() Value = term() Reason = term() @@ -477,9 +478,9 @@

This function is called after each test case, and can be used to clean up after init_per_testcase/2 - and the test case. Any return value (besides {fail,Reason} - and {save_config,SaveConfig}) is ignored. By returning - {fail,Reason}, TestCase is marked as faulty (even + and the test case. Any return value (besides {fail, Reason} + and {save_config, SaveConfig}) is ignored. By returning + {fail, Reason}, TestCase is marked as faulty (even though it was successful in the sense that it returned a value instead of terminating).

@@ -493,16 +494,16 @@ Module:Testcase() -> [Info] Test case information function. - Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()} - TimeFunc = {Mod,Func,Args} | Fun + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun MilliSec = integer() Mod = atom() Func = atom() Args = list() Fun = fun() - Required = Key | {Key,SubKeys} | {Key,Subkey} | {Key,Subkey,SubKeys} + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} Key = atom() SubKeys = SubKey | [SubKey] SubKey = atom() @@ -564,10 +565,10 @@ - Module:Testcase(Config) -> term() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() + Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() A test case. - Config = SaveConfig = [{Key,Value}] + Config = SaveConfig = [{Key, Value}] Key = atom() Value = term() Reason = term() @@ -588,11 +589,11 @@ (or the macro ?config defined in ct.hrl).

If you decide not to run the test case after all, return - {skip,Reason}. Reason is then + {skip, Reason}. Reason is then printed in field Comment on the HTML result page.

To print some information in field Comment on the HTML - result page, return {comment,Comment}.

+ result page, return {comment, Comment}.

If the function returns anything else, the test case is considered successful. The return value always gets printed From 6f400fa566aa07fd30ac24b4cb98da35bc8d4282 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 22:37:07 +0100 Subject: [PATCH 17/29] Add ct_suite with data types and callback definitions Most of the information was moved from common_test_app.xml as-was --- lib/common_test/doc/src/Makefile | 3 +- lib/common_test/doc/src/common_test_app.xml | 560 +----------------- lib/common_test/doc/src/ct_suite.xml | 624 ++++++++++++++++++++ lib/common_test/doc/src/ref_man.xml | 1 + lib/common_test/doc/src/specs.xml | 1 + 5 files changed, 629 insertions(+), 560 deletions(-) create mode 100644 lib/common_test/doc/src/ct_suite.xml diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index cd5f2e2de01e..62035d9f9117 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -50,7 +50,8 @@ XML_REF3_FILES = ct.xml \ ct_property_test.xml \ ct_netconfc.xml \ ct_hooks.xml \ - ct_testspec.xml + ct_testspec.xml \ + ct_suite.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 090097fde66d..d05fa7640f19 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -4,7 +4,7 @@

- 20032020 + 200320172020 Ericsson AB. All Rights Reserved. @@ -52,565 +52,7 @@ Step-by-step execution of test cases -

The following section describes the mandatory and optional test suite - functions that Common Test calls during test execution. - For more details, see section - Writing Test Suites - in the User's Guide.

- - - - - - Test Case Callback Functions -

The following functions define the callback interface - for a test suite.

- - - Module:all() -> [TestDef] | {skip, Reason} - Returns the list of all test case groups and test cases - in the module. - - TestDef = TestCase | {group, GroupName} | {group, GroupName, Properties} | {group, GroupName, Properties, SubGroups} - TestCase = atom() - TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] - GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType, N}] | default - SubGroups = [{GroupName, Properties} | {GroupName, Properties, SubGroups}] - Shuffle = shuffle | {shuffle, Seed} - Seed = {integer(), integer(), integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail - N = integer() | forever - Reason = term() - - - -

MANDATORY

- -

Returns the list of all test cases and test case groups in the - test suite module to be executed. This list also specifies the - order the cases and groups are executed by Common Test. - A test case is represented by an atom, - the name of the test case function, or a testcase tuple - indicating that the test case shall be repeated. A test case group is - represented by a group tuple, where GroupName, - an atom, is the name of the group (defined in - groups/0). - Execution properties for groups can also be specified, both - for a top-level group and for any of its subgroups. - Group execution properties specified here override - properties in the group definition (see - groups/0). - (With value default, the group definition properties - are used).

- -

If {skip, Reason} is returned, all test cases - in the module are skipped and Reason - is printed on the HTML result page.

- -

For details on groups, see section - Test Case - Groups in the User's Guide.

- - - - - - Module:groups() -> [GroupDef] - Returns a list of test case group definitions. - - GroupDef = {GroupName, Properties, GroupsAndTestCases} - GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType, N}] - GroupsAndTestCases = [Group | {group, GroupName} | TestCase] - TestCase = atom() - Shuffle = shuffle | {shuffle, Seed} - Seed = {integer(), integer(), integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail - N = integer() | forever - - - -

OPTIONAL

- -

Defines test case groups. For details, see section - Test Case - Groups in the User's Guide.

- - - - - Module:suite() -> [Info] - Test suite info function (providing default data - for the suite). - - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] - CSSFile = string() - CTHs = [CTHModule | -         {CTHModule, CTHInitArgs} | -         {CTHModule, CTHInitArgs, CTHPriority}] - CTHModule = atom() - CTHInitArgs = term() - CTHPriority = integer() - - - -

OPTIONAL

- -

The test suite information function. Returns a list of tagged - tuples specifying various properties related to the execution of - this test suite (common for all test cases in the suite).

- -

Tag timetrap sets the maximum time that each - test case is allowed to execute (including - init_per_testcase/2 - and - end_per_testcase/2). - If the timetrap time is exceeded, the test case fails with reason - timetrap_timeout. A TimeFunc function can be used to - set a new timetrap by returning a TimeVal. It can also be - used to trigger a timetrap time-out by, at some point, returning a - value other than a TimeVal. For details, see section - Timetrap Time-Outs - in the User's Guide.

- -

Tag require specifies configuration variables - required by test cases (or configuration functions) - in the suite. If the required configuration variables are not found - in any of the configuration files, all test cases are skipped. - For details about the require functionality, see funtion - ct:require/1,2.

- -

With userdata, the user can - specify any test suite-related information, which can be - read by calling - ct:userdata/2.

- -

Tag ct_hooks specifies the - Common Test Hooks - to be run with this suite.

- -

Other tuples than the ones defined are ignored.

- -

For details about the test suite information function, see section - Test - Suite Information Function in the User's Guide.

- - - - - Module:init_per_suite(Config) -> NewConfig | {skip, Reason} | - {skip_and_save, Reason, SaveConfig} - Test suite initializations. - - Config = NewConfig = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL; if this function is defined, then end_per_suite/1 - must also be defined.

- -

This configuration function is called as the first function in the - suite. It typically contains initializations that are common for - all test cases in the suite, and that must only be done - once. Parameter Config is the configuration data - that can be modified. Whatever is returned from this - function is specified as Config to all configuration functions - and test cases in the suite.

- -

If {skip, Reason} - is returned, all test cases in the suite are skipped - and Reason is printed in the overview log for the suite.

- -

For information on save_config and skip_and_save, - see section - Saving - Configuration Data in the User's Guide.

- - - - - Module:end_per_suite(Config) -> term() | - {save_config, SaveConfig} - Test suite finalization. - - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() - - - -

OPTIONAL; if this function is defined, then init_per_suite/1 - must also be defined.

- -

This function is called as the last test case in the - suite. It is meant to be used for cleaning up after - init_per_suite/1.

-

For information on save_config, see section - Saving - Configuration Data in the User's Guide.

- - - - - Module:group(GroupName) -> [Info] - Test case group information function (providing default data - for a test case group, that is, its test cases and - subgroups). - - GroupName = atom() - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] - CSSFile = string() - CTHs = [CTHModule | -         {CTHModule, CTHInitArgs} | -         {CTHModule, CTHInitArgs, CTHPriority}] - CTHModule = atom() - CTHInitArgs = term() - CTHPriority = integer() - - - -

OPTIONAL

- -

The test case group information function. It is supposed to - return a list of tagged tuples that specify various properties - related to the execution of a test case group (that is, its test - cases and subgroups). Properties set by - group/1 override - properties with the same key that have been set previously by - suite/0.

- -

Tag timetrap sets the maximum time that each - test case is allowed to execute (including - init_per_testcase/2 - and - end_per_testcase/2). - If the timetrap time is - exceeded, the test case fails with reason - timetrap_timeout. A TimeFunc function can be used to - set a new timetrap by returning a TimeVal. It can also be - used to trigger a timetrap time-out by, at some point, returning a - value other than a TimeVal. For details, see section - Timetrap - Time-Outs in the User's Guide.

- -

Tag require specifies configuration variables - required by test cases (or configuration functions) - in the suite. If the required configuration variables are not found - in any of the configuration files, all test cases in this group are - skipped. For details about the require functionality, see - function - ct:require/1,2.

- -

With userdata, the user can - specify any test case group related information that can be - read by calling - ct:userdata/2.

- -

Tag ct_hooks specifies the - Common Test Hooks - to be run with this suite.

- -

Other tuples than the ones defined are ignored.

- -

For details about the test case group information function, - see section Group - Information Function in the User's Guide.

- - - - - Module:init_per_group(GroupName, Config) -> NewConfig | - {skip, Reason} - Test case group initializations. - - GroupName = atom() - Config = NewConfig = [{Key, Value}] - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL; if this function is defined, then end_per_group/2 - must also be defined.

- -

This configuration function is called before execution of a - test case group. It typically contains initializations that are - common for all test cases and subgroups in the group, and that - must only be performed once. GroupName is the name of the - group, as specified in the group definition (see - groups/0). - Parameter Config is the configuration data that can be - modified. - The return value of this function is given as Config - to all test cases and subgroups in the group.

- -

If {skip, Reason} - is returned, all test cases in the group are skipped and - Reason is printed in the overview log for the group.

- -

For information about test case groups, see section - Test Case - Groups in the User's Guide.

- - - - - Module:end_per_group(GroupName, Config) -> term() | - {return_group_result, Status} - Test case group finalization. - - GroupName = atom() - Config = [{Key, Value}] - Key = atom() - Value = term() - Status = ok | skipped | failed - - - -

OPTIONAL; if this function is defined, then init_per_group/2 - must also be defined.

- -

This function is called after the execution of a test case group - is finished. It is meant to be used for cleaning up after - init_per_group/2. - A status value for a nested subgroup can be returned with - {return_group_result, Status}. The status can be retrieved in - end_per_group/2 - for the group on the level above. The status is also used by - Common Test for deciding if execution of a group is to - proceed if property sequence or repeat_until_* - is set.

- -

For details about test case groups, see section - Test Case - Groups in the User's Guide.

- - - - - Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} - Test case initializations. - - TestCase = atom() - Config = NewConfig = [{Key, Value}] - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL; if this function is defined, - then - end_per_testcase/2 must also be - defined.

- -

This function is called before each test case. Argument - TestCase is the test case name, and - Config (list of key-value tuples) is the configuration - data that can be modified. The NewConfig list returned - from this function is given as Config to the test case. - If {fail, Reason} is returned, the test case is - marked as failed without being executed.

- -

If {skip, Reason} is returned, the test case is skipped - and Reason is printed in the overview log for the suite.

- - - - - Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} - Test case finalization. - - TestCase = atom() - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL; if this function is defined, - then - init_per_testcase/2 must also be - defined.

- -

This function is called after each test case, and can be used - to clean up after - init_per_testcase/2 - and the test case. Any return value (besides {fail, Reason} - and {save_config, SaveConfig}) is ignored. By returning - {fail, Reason}, TestCase is marked as faulty (even - though it was successful in the sense that it returned - a value instead of terminating).

- -

For information on save_config, see section - Saving - Configuration Data in the User's Guide.

- - - - - Module:Testcase() -> [Info] - Test case information function. - - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] - - - - -

OPTIONAL

- -

The test case information function. It is supposed to - return a list of tagged tuples that specify various properties - related to the execution of this particular test case. - Properties set by - Testcase/0 - override properties set previously for the test case by - group/1 or - suite/0.

- -

Tag timetrap sets the maximum time that the - test case is allowed to execute. If the timetrap time is - exceeded, the test case fails with reason timetrap_timeout. - init_per_testcase/2 - and - end_per_testcase/2 - are included in the timetrap time. - A TimeFunc function can be used to - set a new timetrap by returning a TimeVal. It can also be - used to trigger a timetrap time-out by, at some point, returning a - value other than a TimeVal. For details, see section - Timetrap - Time-Outs in the User's Guide.

- -

Tag require specifies configuration variables - that are required by the test case (or init_per_testcase/2 - or end_per_testcase/2). - If the required configuration variables are not found in any of the - configuration files, the test case is skipped. For details about - the require functionality, see function - ct:require/1,2.

- -

If timetrap or require is not set, the - default values specified by - suite/0 (or - group/1) are used.

- -

With userdata, the user can specify any test case-related - information that can be read by calling - ct:userdata/3.

- -

Other tuples than the ones defined are ignored.

- -

For details about the test case information function, see section - Test - Case Information Function in the User's Guide.

- - - - - Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() - A test case. - - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() - Reason = term() - Comment = string() - - - -

MANDATORY

- -

The implementation of a test case. Call the functions to test and - check the result. If something fails, ensure the - function causes a runtime error or call - ct:fail/1,2 - (which also causes the test case process to terminate).

- -

Elements from the Config list can, for example, be read - with proplists:get_value/2 in STDLIB - (or the macro ?config defined in ct.hrl).

- -

If you decide not to run the test case after all, return - {skip, Reason}. Reason is then - printed in field Comment on the HTML result page.

- -

To print some information in field Comment on the HTML - result page, return {comment, Comment}.

- -

If the function returns anything else, the test case is - considered successful. The return value always gets printed - in the test case log file.

- -

For details about test case implementation, see section - Test Cases - in the User's Guide.

- -

For information on save_config and skip_and_save, - see section - Saving - Configuration Data in the User's Guide.

- - - - - diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml new file mode 100644 index 000000000000..dfabcb525960 --- /dev/null +++ b/lib/common_test/doc/src/ct_suite.xml @@ -0,0 +1,624 @@ + + + + +
+ + 2020 + Ericsson AB, All Rights Reserved + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Initial Developer of the Original Code is Ericsson AB. + + ct_suite + + + + +
+ ct_suite + -behaviour(ct_suite). + + +

The following section describes the mandatory and optional test suite + functions that Common Test calls during test execution. + For more details, see section + Writing Test Suites + in the User's Guide.

+ + + + + + The name of the testcase function. + + + + The name of the test group. + + + + The configuration data that can be modified. + + + + The status value for a nested subgroup. + + + + The test group definition. + + + + The test suite definition. + + + + The test suite information. + + + +
+ Callback Functions +

+ The following functions are to be exported from a + ct_suite callback module. +

+
+ + + + Module:all() -> [TestDef] | {skip, Reason} + Returns the list of all test case groups and test cases + in the module. + + TestDef = TestCase | {group, GroupName} | {group, GroupName, Properties} | {group, GroupName, Properties, SubGroups} + TestCase = atom() + GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] | default + SubGroups = [{GroupName, Properties} | {GroupName, Properties, SubGroups}] + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + Reason = term() + + + +

MANDATORY

+ +

Returns the list of all test cases and test case groups in the + test suite module to be executed. This list also specifies the + order the cases and groups are executed by Common Test. + A test case is represented by an atom, + the name of the test case function. A test case group is + represented by a group tuple, where GroupName, + an atom, is the name of the group (defined in + groups/0). + Execution properties for groups can also be specified, both + for a top-level group and for any of its subgroups. + Group execution properties specified here override + properties in the group definition (see + groups/0). + (With value default, the group definition properties + are used).

+ +

If {skip, Reason} is returned, all test cases + in the module are skipped and Reason + is printed on the HTML result page.

+ +

For details on groups, see section + Test Case + Groups in the User's Guide.

+ + + + + + Module:groups() -> [GroupDef] + Returns a list of test case group definitions. + + GroupDef = {GroupName, Properties, GroupsAndTestCases} + GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] + GroupsAndTestCases = [Group | {group, GroupName} | TestCase] + TestCase = atom() + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + + + +

OPTIONAL

+ +

Defines test case groups. For details, see section + Test Case + Groups in the User's Guide.

+ + + + + Module:suite() -> [Info] + Test suite info function (providing default data + for the suite). + + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | +         {CTHModule, CTHInitArgs} | +         {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + CTHPriority = integer() + + + +

OPTIONAL

+ +

The test suite information function. Returns a list of tagged + tuples specifying various properties related to the execution of + this test suite (common for all test cases in the suite).

+ +

Tag timetrap sets the maximum time that each + test case is allowed to execute (including + init_per_testcase/2 + and + end_per_testcase/2). + If the timetrap time is exceeded, the test case fails with reason + timetrap_timeout. A TimeFunc function can be used to + set a new timetrap by returning a TimeVal. It can also be + used to trigger a timetrap time-out by, at some point, returning a + value other than a TimeVal. For details, see section + Timetrap Time-Outs + in the User's Guide.

+ +

Tag require specifies configuration variables + required by test cases (or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases are skipped. + For details about the require functionality, see funtion + ct:require/1,2.

+ +

With userdata, the user can + specify any test suite-related information, which can be + read by calling + ct:userdata/2.

+ +

Tag ct_hooks specifies the + Common Test Hooks + to be run with this suite.

+ +

Other tuples than the ones defined are ignored.

+ +

For details about the test suite information function, see section + Test + Suite Information Function in the User's Guide.

+ + + + + Module:init_per_suite(Config) -> NewConfig | {skip, Reason} | + {skip_and_save, Reason, SaveConfig} + Test suite initializations. + + Config = NewConfig = SaveConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL; if this function is defined, then end_per_suite/1 + must also be defined.

+ +

This configuration function is called as the first function in the + suite. It typically contains initializations that are common for + all test cases in the suite, and that must only be done + once. Parameter Config is the configuration data + that can be modified. Whatever is returned from this + function is specified as Config to all configuration functions + and test cases in the suite.

+ +

If {skip, Reason} + is returned, all test cases in the suite are skipped + and Reason is printed in the overview log for the suite.

+ +

For information on save_config and skip_and_save, + see section + Saving + Configuration Data in the User's Guide.

+ + + + + Module:end_per_suite(Config) -> term() | + {save_config, SaveConfig} + Test suite finalization. + + Config = SaveConfig = [{Key, Value}] + Key = atom() + Value = term() + + + +

OPTIONAL; if this function is defined, then init_per_suite/1 + must also be defined.

+ +

This function is called as the last test case in the + suite. It is meant to be used for cleaning up after + init_per_suite/1.

+

For information on save_config, see section + Saving + Configuration Data in the User's Guide.

+ + + + + Module:group(GroupName) -> [Info] + Test case group information function (providing default data + for a test case group, that is, its test cases and + subgroups). + + GroupName = atom() + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | +         {CTHModule, CTHInitArgs} | +         {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + CTHPriority = integer() + + + +

OPTIONAL

+ +

The test case group information function. It is supposed to + return a list of tagged tuples that specify various properties + related to the execution of a test case group (that is, its test + cases and subgroups). Properties set by + group/1 override + properties with the same key that have been set previously by + suite/0.

+ +

Tag timetrap sets the maximum time that each + test case is allowed to execute (including + init_per_testcase/2 + and + end_per_testcase/2). + If the timetrap time is + exceeded, the test case fails with reason + timetrap_timeout. A TimeFunc function can be used to + set a new timetrap by returning a TimeVal. It can also be + used to trigger a timetrap time-out by, at some point, returning a + value other than a TimeVal. For details, see section + Timetrap + Time-Outs in the User's Guide.

+ +

Tag require specifies configuration variables + required by test cases (or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases in this group are + skipped. For details about the require functionality, see + function + ct:require/1,2.

+ +

With userdata, the user can + specify any test case group related information that can be + read by calling + ct:userdata/2.

+ +

Tag ct_hooks specifies the + Common Test Hooks + to be run with this suite.

+ +

Other tuples than the ones defined are ignored.

+ +

For details about the test case group information function, + see section Group + Information Function in the User's Guide.

+ + + + + Module:init_per_group(GroupName, Config) -> NewConfig | + {skip, Reason} + Test case group initializations. + + GroupName = atom() + Config = NewConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL; if this function is defined, then end_per_group/2 + must also be defined.

+ +

This configuration function is called before execution of a + test case group. It typically contains initializations that are + common for all test cases and subgroups in the group, and that + must only be performed once. GroupName is the name of the + group, as specified in the group definition (see + groups/0). + Parameter Config is the configuration data that can be + modified. + The return value of this function is given as Config + to all test cases and subgroups in the group.

+ +

If {skip, Reason} + is returned, all test cases in the group are skipped and + Reason is printed in the overview log for the group.

+ +

For information about test case groups, see section + Test Case + Groups in the User's Guide.

+ + + + + Module:end_per_group(GroupName, Config) -> term() | + {return_group_result, Status} + Test case group finalization. + + GroupName = atom() + Config = [{Key, Value}] + Key = atom() + Value = term() + Status = ok | skipped | failed + + + +

OPTIONAL; if this function is defined, then init_per_group/2 + must also be defined.

+ +

This function is called after the execution of a test case group + is finished. It is meant to be used for cleaning up after + init_per_group/2. + A status value for a nested subgroup can be returned with + {return_group_result, Status}. The status can be retrieved in + end_per_group/2 + for the group on the level above. The status is also used by + Common Test for deciding if execution of a group is to + proceed if property sequence or repeat_until_* + is set.

+ +

For details about test case groups, see section + Test Case + Groups in the User's Guide.

+ + + + + Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} + Test case initializations. + + TestCase = atom() + Config = NewConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL; if this function is defined, + then + end_per_testcase/2 must also be + defined.

+ +

This function is called before each test case. Argument + TestCase is the test case name, and + Config (list of key-value tuples) is the configuration + data that can be modified. The NewConfig list returned + from this function is given as Config to the test case. + If {fail, Reason} is returned, the test case is + marked as failed without being executed.

+ +

If {skip, Reason} is returned, the test case is skipped + and Reason is printed in the overview log for the suite.

+ + + + + Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} + Test case finalization. + + TestCase = atom() + Config = SaveConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL; if this function is defined, + then + init_per_testcase/2 must also be + defined.

+ +

This function is called after each test case, and can be used + to clean up after + init_per_testcase/2 + and the test case. Any return value (besides {fail, Reason} + and {save_config, SaveConfig}) is ignored. By returning + {fail, Reason}, TestCase is marked as faulty (even + though it was successful in the sense that it returned + a value instead of terminating).

+ +

For information on save_config, see section + Saving + Configuration Data in the User's Guide.

+ + + + + Module:Testcase() -> [Info] + Test case information function. + + Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + + + + +

OPTIONAL

+ +

The test case information function. It is supposed to + return a list of tagged tuples that specify various properties + related to the execution of this particular test case. + Properties set by + Testcase/0 + override properties set previously for the test case by + group/1 or + suite/0.

+ +

Tag timetrap sets the maximum time that the + test case is allowed to execute. If the timetrap time is + exceeded, the test case fails with reason timetrap_timeout. + init_per_testcase/2 + and + end_per_testcase/2 + are included in the timetrap time. + A TimeFunc function can be used to + set a new timetrap by returning a TimeVal. It can also be + used to trigger a timetrap time-out by, at some point, returning a + value other than a TimeVal. For details, see section + Timetrap + Time-Outs in the User's Guide.

+ +

Tag require specifies configuration variables + that are required by the test case (or init_per_testcase/2 + or end_per_testcase/2). + If the required configuration variables are not found in any of the + configuration files, the test case is skipped. For details about + the require functionality, see function + ct:require/1,2.

+ +

If timetrap or require is not set, the + default values specified by + suite/0 (or + group/1) are used.

+ +

With userdata, the user can specify any test case-related + information that can be read by calling + ct:userdata/3.

+ +

Other tuples than the ones defined are ignored.

+ +

For details about the test case information function, see section + Test + Case Information Function in the User's Guide.

+ + + + + Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() + A test case. + + Config = SaveConfig = [{Key, Value}] + Key = atom() + Value = term() + Reason = term() + Comment = string() + + + +

MANDATORY

+ +

The implementation of a test case. Call the functions to test and + check the result. If something fails, ensure the + function causes a runtime error or call + ct:fail/1,2 + (which also causes the test case process to terminate).

+ +

Elements from the Config list can, for example, be read + with proplists:get_value/2 in STDLIB + (or the macro ?config defined in ct.hrl).

+ +

If you decide not to run the test case after all, return + {skip, Reason}. Reason is then + printed in field Comment on the HTML result page.

+ +

To print some information in field Comment on the HTML + result page, return {comment, Comment}.

+ +

If the function returns anything else, the test case is + considered successful. The return value always gets printed + in the test case log file.

+ +

For details about test case implementation, see section + Test Cases + in the User's Guide.

+ +

For information on save_config and skip_and_save, + see section + Saving + Configuration Data in the User's Guide.

+ + + + + + diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index e916fc7cec7a..b8184ed7f8ba 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -48,6 +48,7 @@ + diff --git a/lib/common_test/doc/src/specs.xml b/lib/common_test/doc/src/specs.xml index 7e40e8351d74..def8f761a544 100644 --- a/lib/common_test/doc/src/specs.xml +++ b/lib/common_test/doc/src/specs.xml @@ -2,4 +2,5 @@ + From 4f9bceff7113d2b687a35e96de428ab09847191b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 22:50:42 +0100 Subject: [PATCH 18/29] Simplify doc. by using seetype references --- lib/common_test/doc/src/ct_suite.xml | 108 +++++---------------------- 1 file changed, 17 insertions(+), 91 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index dfabcb525960..c2034239b28e 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -84,15 +84,7 @@ Returns the list of all test case groups and test cases in the module. - TestDef = TestCase | {group, GroupName} | {group, GroupName, Properties} | {group, GroupName, Properties, SubGroups} - TestCase = atom() - GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType, N}] | default - SubGroups = [{GroupName, Properties} | {GroupName, Properties, SubGroups}] - Shuffle = shuffle | {shuffle, Seed} - Seed = {integer(), integer(), integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail - N = integer() | forever + TestDef = ct_test_def() Reason = term() @@ -130,15 +122,7 @@ Module:groups() -> [GroupDef] Returns a list of test case group definitions. - GroupDef = {GroupName, Properties, GroupsAndTestCases} - GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType, N}] - GroupsAndTestCases = [Group | {group, GroupName} | TestCase] - TestCase = atom() - Shuffle = shuffle | {shuffle, Seed} - Seed = {integer(), integer(), integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail - N = integer() | forever + GroupDef = ct_group_def() @@ -155,29 +139,7 @@ Test suite info function (providing default data for the suite). - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] - CSSFile = string() - CTHs = [CTHModule | -         {CTHModule, CTHInitArgs} | -         {CTHModule, CTHInitArgs, CTHPriority}] - CTHModule = atom() - CTHInitArgs = term() - CTHPriority = integer() + Info = ct_info() @@ -229,9 +191,7 @@ {skip_and_save, Reason, SaveConfig} Test suite initializations. - Config = NewConfig = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() + Config = NewConfig = SaveConfig = ct_config() Reason = term() @@ -264,9 +224,7 @@ {save_config, SaveConfig} Test suite finalization. - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() + Config = SaveConfig = ct_config() @@ -289,30 +247,8 @@ for a test case group, that is, its test cases and subgroups). - GroupName = atom() - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] - CSSFile = string() - CTHs = [CTHModule | -         {CTHModule, CTHInitArgs} | -         {CTHModule, CTHInitArgs, CTHPriority}] - CTHModule = atom() - CTHInitArgs = term() - CTHPriority = integer() + GroupName = ct_groupname() + Info = ct_info() @@ -370,10 +306,8 @@ {skip, Reason} Test case group initializations. - GroupName = atom() - Config = NewConfig = [{Key, Value}] - Key = atom() - Value = term() + GroupName = ct_groupname() + Config = NewConfig = ct_config() Reason = term() @@ -408,11 +342,9 @@ {return_group_result, Status} Test case group finalization. - GroupName = atom() - Config = [{Key, Value}] - Key = atom() - Value = term() - Status = ok | skipped | failed + GroupName = ct_groupname() + Config = ct_config() + Status = ct_status() @@ -441,10 +373,8 @@ Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} Test case initializations. - TestCase = atom() - Config = NewConfig = [{Key, Value}] - Key = atom() - Value = term() + TestCase = ct_testname() + Config = NewConfig = ct_config() Reason = term() @@ -471,10 +401,8 @@ Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} Test case finalization. - TestCase = atom() - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() + TestCase = ct_testname() + Config = SaveConfig = ct_config() Reason = term() @@ -577,9 +505,7 @@ Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() A test case. - Config = SaveConfig = [{Key, Value}] - Key = atom() - Value = term() + Config = SaveConfig = ct_config() Reason = term() Comment = string() From e3cd578c5f04dad77b47982625119ae065cce36a Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 14 Oct 2020 22:52:10 +0100 Subject: [PATCH 19/29] Prefer plural when in lists, prefer singular when not --- lib/common_test/src/ct_suite.erl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/common_test/src/ct_suite.erl b/lib/common_test/src/ct_suite.erl index eca9996cac67..a2d23e15ef14 100644 --- a/lib/common_test/src/ct_suite.erl +++ b/lib/common_test/src/ct_suite.erl @@ -7,8 +7,8 @@ ct_groupname/0, ct_config/0, ct_status/0, - ct_groups_def/0, - ct_tests_def/0, + ct_group_def/0, + ct_test_def/0, ct_info/0 ]). @@ -35,9 +35,9 @@ repeat_until_any_fail. -type ct_test_repeat() :: integer() | forever. --type ct_groups_def() :: {ct_groupname(), ct_group_props(), [ +-type ct_group_def() :: {ct_groupname(), ct_group_props(), [ ct_testname() | - ct_groups_def() | + ct_group_def() | {group, ct_groupname()} | ct_testcase_ref() ]}. @@ -46,8 +46,8 @@ -type ct_group_ref() :: {group, ct_groupname()} | {group, ct_groupname(), ct_group_props_ref()} | {group, ct_groupname(), ct_group_props_ref(), ct_subgroups_def()}. --type ct_testcase_ref() :: {testcase, ct_testname(), ct_testcase_repeat_props()}. --type ct_testcase_repeat_props() :: {repeat, ct_test_repeat()} | +-type ct_testcase_ref() :: {testcase, ct_testname(), ct_testcase_repeat_prop()}. +-type ct_testcase_repeat_prop() :: {repeat, ct_test_repeat()} | {repeat_until_ok, ct_test_repeat()} | {repeat_until_fail, ct_test_repeat()}. -type ct_info() :: {timetrap, ct_info_timetrap()} | @@ -75,14 +75,14 @@ {CTHModule :: atom(), CTHInitArgs :: term()} | {CTHModule :: atom(), CTHInitArgs :: term(), CTHPriority :: integer()} ]. --type ct_tests_def() :: ct_testname() | ct_group_ref() | ct_testcase_ref(). +-type ct_test_def() :: ct_testname() | ct_group_ref() | ct_testcase_ref(). -callback all() -> - [TestDef :: ct_tests_def()] | + [TestDef :: ct_test_def()] | {skip, Reason :: term()}. -callback groups() -> - [GroupDef :: ct_groups_def()]. + [GroupDef :: ct_group_def()]. -callback suite() -> [Info :: ct_info()]. From 8c9fd13b8e4d381f5f058616a3156c8e1cd3c262 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 15 Oct 2020 00:23:19 +0100 Subject: [PATCH 20/29] Fix doc as per generation procedure --- lib/common_test/doc/src/ct_suite.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index c2034239b28e..f12a7408d741 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -57,11 +57,11 @@ The status value for a nested subgroup. - + The test group definition. - + The test suite definition. From 0011eb4e3ea7e5531bd09f231acd9b924be5462d Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 15 Oct 2020 00:47:13 +0100 Subject: [PATCH 21/29] Fix post-rebase elements that had been transported from common_test_app.xml --- lib/common_test/doc/src/common_test_app.xml | 2 +- lib/common_test/doc/src/ct_suite.xml | 186 ++++++++++---------- 2 files changed, 95 insertions(+), 93 deletions(-) diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index d05fa7640f19..07a2a3e2cdc2 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -4,7 +4,7 @@
- 200320172020 + 20032020 Ericsson AB. All Rights Reserved. diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index f12a7408d741..d2d6d08942bc 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -28,14 +28,14 @@
- ct_suite + ct_suite -behaviour(ct_suite).

The following section describes the mandatory and optional test suite functions that Common Test calls during test execution. For more details, see section - Writing Test Suites + Writing Test Suites in the User's Guide.

 @@ -70,17 +70,18 @@ -
- Callback Functions -

- The following functions are to be exported from a - ct_suite callback module. -

-
- + + Callback Functions +

+ The following functions are to be exported from a + ct_suite callback module in order to define + the callback interface for a test suite. +

+ + - Module:all() -> [TestDef] | {skip, Reason} + Module:all() -> [TestDef] | {skip, Reason} Returns the list of all test case groups and test cases in the module. @@ -95,15 +96,16 @@ test suite module to be executed. This list also specifies the order the cases and groups are executed by Common Test. A test case is represented by an atom, - the name of the test case function. A test case group is + the name of the test case function, or a testcase tuple + indicating that the test case shall be repeated. A test case group is represented by a group tuple, where GroupName, an atom, is the name of the group (defined in - groups/0). + groups/0). Execution properties for groups can also be specified, both for a top-level group and for any of its subgroups. Group execution properties specified here override properties in the group definition (see - groups/0). + groups/0). (With value default, the group definition properties are used).

@@ -112,14 +114,14 @@ is printed on the HTML result page.

For details on groups, see section - Test Case - Groups in the User's Guide.

+ Test Case + Groups in the User's Guide.

 - Module:groups() -> [GroupDef] + Module:groups() -> [GroupDef] Returns a list of test case group definitions. GroupDef = ct_group_def() @@ -129,13 +131,13 @@

OPTIONAL

Defines test case groups. For details, see section - Test Case - Groups in the User's Guide.

+ Test Case + Groups in the User's Guide.

 - Module:suite() -> [Info] + Module:suite() -> [Info] Test suite info function (providing default data for the suite). @@ -151,15 +153,15 @@

Tag timetrap sets the maximum time that each test case is allowed to execute (including - init_per_testcase/2 + init_per_testcase/2 and - end_per_testcase/2). + end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. A TimeFunc function can be used to set a new timetrap by returning a TimeVal. It can also be used to trigger a timetrap time-out by, at some point, returning a value other than a TimeVal. For details, see section - Timetrap Time-Outs + Timetrap Time-Outs in the User's Guide.

Tag require specifies configuration variables @@ -167,27 +169,27 @@ in the suite. If the required configuration variables are not found in any of the configuration files, all test cases are skipped. For details about the require functionality, see funtion - ct:require/1,2.

+ ct:require/1,2.

With userdata, the user can specify any test suite-related information, which can be read by calling - ct:userdata/2.

+ ct:userdata/2.

Tag ct_hooks specifies the - Common Test Hooks + Common Test Hooks to be run with this suite.

Other tuples than the ones defined are ignored.

For details about the test suite information function, see section - Test - Suite Information Function in the User's Guide.

+ Test + Suite Information Function in the User's Guide.

 - Module:init_per_suite(Config) -> NewConfig | {skip, Reason} | + Module:init_per_suite(Config) -> NewConfig | {skip, Reason} | {skip_and_save, Reason, SaveConfig} Test suite initializations. @@ -196,8 +198,8 @@ -

OPTIONAL; if this function is defined, then end_per_suite/1 +

OPTIONAL; if this function is defined, then end_per_suite/1 must also be defined.

This configuration function is called as the first function in the @@ -214,13 +216,13 @@

For information on save_config and skip_and_save, see section - Saving - Configuration Data in the User's Guide.

+ Saving + Configuration Data in the User's Guide.

 - Module:end_per_suite(Config) -> term() | + Module:end_per_suite(Config) -> term() | {save_config, SaveConfig} Test suite finalization. @@ -228,21 +230,21 @@ -

OPTIONAL; if this function is defined, then init_per_suite/1 +

OPTIONAL; if this function is defined, then init_per_suite/1 must also be defined.

This function is called as the last test case in the suite. It is meant to be used for cleaning up after - init_per_suite/1.

+ init_per_suite/1.

For information on save_config, see section - Saving - Configuration Data in the User's Guide.

+ Saving + Configuration Data in the User's Guide.

 - Module:group(GroupName) -> [Info] + Module:group(GroupName) -> [Info] Test case group information function (providing default data for a test case group, that is, its test cases and subgroups). @@ -258,23 +260,23 @@ return a list of tagged tuples that specify various properties related to the execution of a test case group (that is, its test cases and subgroups). Properties set by - group/1 override + group/1 override properties with the same key that have been set previously by - suite/0.

+ suite/0.

Tag timetrap sets the maximum time that each test case is allowed to execute (including - init_per_testcase/2 + init_per_testcase/2 and - end_per_testcase/2). + end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. A TimeFunc function can be used to set a new timetrap by returning a TimeVal. It can also be used to trigger a timetrap time-out by, at some point, returning a value other than a TimeVal. For details, see section - Timetrap - Time-Outs in the User's Guide.

+ Timetrap + Time-Outs in the User's Guide.

Tag require specifies configuration variables required by test cases (or configuration functions) @@ -282,27 +284,27 @@ in any of the configuration files, all test cases in this group are skipped. For details about the require functionality, see function - ct:require/1,2.

+ ct:require/1,2.

With userdata, the user can specify any test case group related information that can be read by calling - ct:userdata/2.

+ ct:userdata/2.

Tag ct_hooks specifies the - Common Test Hooks + Common Test Hooks to be run with this suite.

Other tuples than the ones defined are ignored.

For details about the test case group information function, - see section Group - Information Function in the User's Guide.

+ see section Group + Information Function in the User's Guide.

 - Module:init_per_group(GroupName, Config) -> NewConfig | + Module:init_per_group(GroupName, Config) -> NewConfig | {skip, Reason} Test case group initializations. @@ -312,8 +314,8 @@ -

OPTIONAL; if this function is defined, then end_per_group/2 +

OPTIONAL; if this function is defined, then end_per_group/2 must also be defined.

This configuration function is called before execution of a @@ -321,7 +323,7 @@ common for all test cases and subgroups in the group, and that must only be performed once. GroupName is the name of the group, as specified in the group definition (see - groups/0). + groups/0). Parameter Config is the configuration data that can be modified. The return value of this function is given as Config @@ -332,13 +334,13 @@ Reason is printed in the overview log for the group.

For information about test case groups, see section - Test Case - Groups in the User's Guide.

+ Test Case + Groups in the User's Guide.

 - Module:end_per_group(GroupName, Config) -> term() | + Module:end_per_group(GroupName, Config) -> term() | {return_group_result, Status} Test case group finalization. @@ -348,29 +350,29 @@ -

OPTIONAL; if this function is defined, then init_per_group/2 +

OPTIONAL; if this function is defined, then init_per_group/2 must also be defined.

This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after - init_per_group/2. + init_per_group/2. A status value for a nested subgroup can be returned with {return_group_result, Status}. The status can be retrieved in - end_per_group/2 + end_per_group/2 for the group on the level above. The status is also used by Common Test for deciding if execution of a group is to proceed if property sequence or repeat_until_* is set.

For details about test case groups, see section - Test Case - Groups in the User's Guide.

+ Test Case + Groups in the User's Guide.

 - Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} + Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail, Reason} | {skip, Reason} Test case initializations. TestCase = ct_testname() @@ -380,8 +382,8 @@

OPTIONAL; if this function is defined, - then - end_per_testcase/2 must also be + then + end_per_testcase/2 must also be defined.

This function is called before each test case. Argument @@ -398,7 +400,7 @@ - Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} + Module:end_per_testcase(TestCase, Config) -> term() | {fail, Reason} | {save_config, SaveConfig} Test case finalization. TestCase = ct_testname() @@ -408,13 +410,13 @@

OPTIONAL; if this function is defined, - then - init_per_testcase/2 must also be + then + init_per_testcase/2 must also be defined.

This function is called after each test case, and can be used to clean up after - init_per_testcase/2 + init_per_testcase/2 and the test case. Any return value (besides {fail, Reason} and {save_config, SaveConfig}) is ignored. By returning {fail, Reason}, TestCase is marked as faulty (even @@ -422,13 +424,13 @@ a value instead of terminating).

For information on save_config, see section - Saving - Configuration Data in the User's Guide.

+ Saving + Configuration Data in the User's Guide.

 - Module:Testcase() -> [Info] + Module:Testcase() -> [Info] Test case information function. Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} @@ -457,24 +459,24 @@ return a list of tagged tuples that specify various properties related to the execution of this particular test case. Properties set by - Testcase/0 + Testcase/0 override properties set previously for the test case by - group/1 or - suite/0.

+ group/1 or + suite/0.

Tag timetrap sets the maximum time that the test case is allowed to execute. If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. - init_per_testcase/2 + init_per_testcase/2 and - end_per_testcase/2 + end_per_testcase/2 are included in the timetrap time. A TimeFunc function can be used to set a new timetrap by returning a TimeVal. It can also be used to trigger a timetrap time-out by, at some point, returning a value other than a TimeVal. For details, see section - Timetrap - Time-Outs in the User's Guide.

+ Timetrap + Time-Outs in the User's Guide.

Tag require specifies configuration variables that are required by the test case (or init_per_testcase/2 @@ -482,27 +484,27 @@ If the required configuration variables are not found in any of the configuration files, the test case is skipped. For details about the require functionality, see function - ct:require/1,2.

+ ct:require/1,2.

If timetrap or require is not set, the default values specified by - suite/0 (or - group/1) are used.

+ suite/0 (or + group/1) are used.

With userdata, the user can specify any test case-related information that can be read by calling - ct:userdata/3.

+ ct:userdata/3.

Other tuples than the ones defined are ignored.

For details about the test case information function, see section - Test - Case Information Function in the User's Guide.

+ Test + Case Information Function in the User's Guide.

 - Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() + Module:Testcase(Config) -> term() | {skip, Reason} | {comment, Comment} | {save_config, SaveConfig} | {skip_and_save, Reason, SaveConfig} | exit() A test case. Config = SaveConfig = ct_config() @@ -516,7 +518,7 @@

The implementation of a test case. Call the functions to test and check the result. If something fails, ensure the function causes a runtime error or call - ct:fail/1,2 + ct:fail/1,2 (which also causes the test case process to terminate).

Elements from the Config list can, for example, be read @@ -535,13 +537,13 @@ in the test case log file.

For details about test case implementation, see section - Test Cases + Test Cases in the User's Guide.

For information on save_config and skip_and_save, see section - Saving - Configuration Data in the User's Guide.

+ Saving + Configuration Data in the User's Guide.

 From edf6e02b2d7e5f63a9d23b894ed49d06cc3e138d Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 15 Oct 2020 00:50:55 +0100 Subject: [PATCH 22/29] Simplify the doc further --- lib/common_test/doc/src/ct_suite.xml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index d2d6d08942bc..6c5a8169e7e5 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -433,22 +433,7 @@ Module:Testcase() -> [Info] Test case information function. - Info = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} - Time = TimeVal | TimeFunc - TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} - TimeFunc = {Mod, Func, Args} | Fun - MilliSec = integer() - Mod = atom() - Func = atom() - Args = list() - Fun = fun() - Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} - Key = atom() - SubKeys = SubKey | [SubKey] - SubKey = atom() - Name = atom() - UserData = term() - Conns = [atom()] + Info = ct_info() From 627e3ef33bb0caee71ad1ed59160a00120ce0d58 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 15 Oct 2020 10:09:57 +0100 Subject: [PATCH 23/29] Fix doc generation issues --- lib/common_test/doc/src/Makefile | 2 +- lib/common_test/doc/src/ct_suite.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index 62035d9f9117..a5f2f0975e6d 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -87,7 +87,7 @@ XML_FILES=$(XML_APPLICATION_FILES) $(XML_REF1_FILES) $(XML_REF3_FILES) $(XML_RE TOP_SPECS_FILE = specs.xml -NO_CHUNKS = ct_hooks.xml +NO_CHUNKS = ct_hooks.xml ct_suite.xml # ---------------------------------------------------- diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index 6c5a8169e7e5..0c7e5e59ebd4 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -28,7 +28,7 @@
- ct_suite + ct_suite -behaviour(ct_suite). From bfd6ba8d448fca4bdb6a1d63dab1471134020837 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 16 Oct 2020 01:50:17 +0100 Subject: [PATCH 24/29] Fix after looking at generated doc. --- lib/common_test/doc/src/ct_suite.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index 0c7e5e59ebd4..048c766d3213 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -350,8 +350,8 @@ -

OPTIONAL; if this function is defined, then init_per_group/2 +

OPTIONAL; if this function is defined, then init_per_group/2 must also be defined.

This function is called after the execution of a test case group From 4ddf2bc91286aac0dc77dad2af2f8f7fede15f9e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 16 Oct 2020 02:20:25 +0100 Subject: [PATCH 25/29] Apply newly acquired knowledge to some previously existing common test suites --- lib/common_test/src/common_test.app.src | 3 ++- lib/ssl/test/ssl_ECC.erl | 2 ++ lib/ssl/test/ssl_ECC_SUITE.erl | 2 ++ lib/ssl/test/ssl_alert_SUITE.erl | 2 ++ lib/ssl/test/ssl_alpn_SUITE.erl | 2 ++ lib/ssl/test/ssl_api_SUITE.erl | 2 ++ lib/ssl/test/ssl_app_env_SUITE.erl | 2 ++ lib/ssl/test/ssl_basic_SUITE.erl | 2 ++ lib/ssl/test/ssl_bench_SUITE.erl | 2 ++ lib/ssl/test/ssl_bench_test_lib.erl | 2 ++ lib/ssl/test/ssl_cert_SUITE.erl | 2 ++ lib/ssl/test/ssl_cert_tests.erl | 2 ++ lib/ssl/test/ssl_cipher_SUITE.erl | 2 ++ lib/ssl/test/ssl_cipher_suite_SUITE.erl | 2 ++ lib/ssl/test/ssl_crl_SUITE.erl | 2 ++ lib/ssl/test/ssl_dist_SUITE.erl | 2 ++ lib/ssl/test/ssl_dist_bench_SUITE.erl | 2 ++ lib/ssl/test/ssl_dist_test_lib.erl | 2 ++ lib/ssl/test/ssl_engine_SUITE.erl | 2 ++ lib/ssl/test/ssl_eqc_SUITE.erl | 1 + lib/ssl/test/ssl_handshake_SUITE.erl | 2 ++ lib/ssl/test/ssl_key_update_SUITE.erl | 2 ++ lib/ssl/test/ssl_mfl_SUITE.erl | 3 +++ lib/ssl/test/ssl_npn_SUITE.erl | 2 ++ lib/ssl/test/ssl_npn_hello_SUITE.erl | 2 ++ lib/ssl/test/ssl_packet_SUITE.erl | 2 ++ lib/ssl/test/ssl_payload_SUITE.erl | 2 ++ lib/ssl/test/ssl_pem_cache_SUITE.erl | 2 ++ lib/ssl/test/ssl_renegotiate_SUITE.erl | 2 ++ lib/ssl/test/ssl_rfc_5869_SUITE.erl | 2 ++ lib/ssl/test/ssl_session_SUITE.erl | 2 ++ lib/ssl/test/ssl_session_cache_SUITE.erl | 2 ++ lib/ssl/test/ssl_session_ticket_SUITE.erl | 2 ++ lib/ssl/test/ssl_sni_SUITE.erl | 2 ++ lib/ssl/test/ssl_socket_SUITE.erl | 2 ++ lib/ssl/test/ssl_test_lib.erl | 2 ++ lib/ssl/test/ssl_upgrade_SUITE.erl | 2 ++ 37 files changed, 74 insertions(+), 1 deletion(-) diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 07b35b71800c..8aa15efa7e67 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -63,7 +63,8 @@ test_server_gl, test_server_io, test_server_node, - test_server_sup + test_server_sup, + ct_suite ]}, {registered, [ct_logs, ct_util_server, diff --git a/lib/ssl/test/ssl_ECC.erl b/lib/ssl/test/ssl_ECC.erl index 56b67361e175..73dafcd1fc93 100644 --- a/lib/ssl/test/ssl_ECC.erl +++ b/lib/ssl/test/ssl_ECC.erl @@ -23,6 +23,8 @@ -module(ssl_ECC). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 496cb38fd98d..f834ac410573 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_ECC_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_alert_SUITE.erl b/lib/ssl/test/ssl_alert_SUITE.erl index ffdc9ea91677..3b32d3ece15e 100644 --- a/lib/ssl/test/ssl_alert_SUITE.erl +++ b/lib/ssl/test/ssl_alert_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_alert_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_alpn_SUITE.erl b/lib/ssl/test/ssl_alpn_SUITE.erl index f0be74071658..9cc3303604b7 100644 --- a/lib/ssl/test/ssl_alpn_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_alpn_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Callback functions diff --git a/lib/ssl/test/ssl_api_SUITE.erl b/lib/ssl/test/ssl_api_SUITE.erl index d52645cc52fb..95e3c0a058fc 100644 --- a/lib/ssl/test/ssl_api_SUITE.erl +++ b/lib/ssl/test/ssl_api_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_api_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("ssl/src/ssl_api.hrl"). diff --git a/lib/ssl/test/ssl_app_env_SUITE.erl b/lib/ssl/test/ssl_app_env_SUITE.erl index ee9c3ad74552..98e96023b247 100644 --- a/lib/ssl/test/ssl_app_env_SUITE.erl +++ b/lib/ssl/test/ssl_app_env_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_app_env_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("ssl/src/ssl_api.hrl"). diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 0039b0244f51..1426c7ef49b8 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_basic_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("ssl/src/ssl_api.hrl"). diff --git a/lib/ssl/test/ssl_bench_SUITE.erl b/lib/ssl/test/ssl_bench_SUITE.erl index fe30a83a7e46..f78ac9c9cc80 100644 --- a/lib/ssl/test/ssl_bench_SUITE.erl +++ b/lib/ssl/test/ssl_bench_SUITE.erl @@ -19,6 +19,8 @@ %% -module(ssl_bench_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct_event.hrl"). %% Callback functions diff --git a/lib/ssl/test/ssl_bench_test_lib.erl b/lib/ssl/test/ssl_bench_test_lib.erl index cbb9cfe47abd..74ab142993e1 100644 --- a/lib/ssl/test/ssl_bench_test_lib.erl +++ b/lib/ssl/test/ssl_bench_test_lib.erl @@ -19,6 +19,8 @@ %% -module(ssl_bench_test_lib). +-behaviour(ct_suite). + %% API -export([setup/1]). diff --git a/lib/ssl/test/ssl_cert_SUITE.erl b/lib/ssl/test/ssl_cert_SUITE.erl index bec6057d9123..655d29b6ae76 100644 --- a/lib/ssl/test/ssl_cert_SUITE.erl +++ b/lib/ssl/test/ssl_cert_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_cert_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_cert_tests.erl b/lib/ssl/test/ssl_cert_tests.erl index 5422ff7fe4fe..d2bf39d88c65 100644 --- a/lib/ssl/test/ssl_cert_tests.erl +++ b/lib/ssl/test/ssl_cert_tests.erl @@ -21,6 +21,8 @@ %% -module(ssl_cert_tests). +-behaviour(ct_suite). + -include_lib("public_key/include/public_key.hrl"). %% Test cases diff --git a/lib/ssl/test/ssl_cipher_SUITE.erl b/lib/ssl/test/ssl_cipher_SUITE.erl index eee8d8078be1..31e60269f8bb 100644 --- a/lib/ssl/test/ssl_cipher_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_cipher_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include("tls_record.hrl"). -include("ssl_cipher.hrl"). diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl index 6952472300c6..99911af3706a 100644 --- a/lib/ssl/test/ssl_cipher_suite_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_cipher_suite_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Callback functions -export([all/0, diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl index ad75d29a78f3..f9fac829629e 100644 --- a/lib/ssl/test/ssl_crl_SUITE.erl +++ b/lib/ssl/test/ssl_crl_SUITE.erl @@ -21,6 +21,8 @@ -module(ssl_crl_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 20463114f4fb..019e22eaa842 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_dist_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). -include("ssl_dist_test_lib.hrl"). diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 67944c74d27c..2216c6b04b71 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -19,6 +19,8 @@ %% -module(ssl_dist_bench_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct_event.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_dist_test_lib.erl b/lib/ssl/test/ssl_dist_test_lib.erl index fa1f4217feb1..bf010e6ad4b4 100644 --- a/lib/ssl/test/ssl_dist_test_lib.erl +++ b/lib/ssl/test/ssl_dist_test_lib.erl @@ -20,6 +20,8 @@ -module(ssl_dist_test_lib). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). -include("ssl_dist_test_lib.hrl"). diff --git a/lib/ssl/test/ssl_engine_SUITE.erl b/lib/ssl/test/ssl_engine_SUITE.erl index 4e88c6e46cfe..c0e28120bee7 100644 --- a/lib/ssl/test/ssl_engine_SUITE.erl +++ b/lib/ssl/test/ssl_engine_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_engine_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_eqc_SUITE.erl b/lib/ssl/test/ssl_eqc_SUITE.erl index 52edb39d9cca..b663597835ed 100644 --- a/lib/ssl/test/ssl_eqc_SUITE.erl +++ b/lib/ssl/test/ssl_eqc_SUITE.erl @@ -20,6 +20,7 @@ -module(ssl_eqc_SUITE). +-behaviour(ct_suite). %% Common test -export([all/0, diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index f990b9328ae4..6da6da4051f7 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_handshake_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include("ssl_alert.hrl"). -include("ssl_handshake.hrl"). diff --git a/lib/ssl/test/ssl_key_update_SUITE.erl b/lib/ssl/test/ssl_key_update_SUITE.erl index 2816f1a39e22..aba6a02ddc7a 100644 --- a/lib/ssl/test/ssl_key_update_SUITE.erl +++ b/lib/ssl/test/ssl_key_update_SUITE.erl @@ -19,6 +19,8 @@ %% -module(ssl_key_update_SUITE). +-behaviour(ct_suite). + %% Callback functions -export([all/0, groups/0, diff --git a/lib/ssl/test/ssl_mfl_SUITE.erl b/lib/ssl/test/ssl_mfl_SUITE.erl index e8fa0ddf5264..0f9aeb1c674a 100644 --- a/lib/ssl/test/ssl_mfl_SUITE.erl +++ b/lib/ssl/test/ssl_mfl_SUITE.erl @@ -18,6 +18,9 @@ %% %CopyrightEnd% %% -module(ssl_mfl_SUITE). + +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Common test diff --git a/lib/ssl/test/ssl_npn_SUITE.erl b/lib/ssl/test/ssl_npn_SUITE.erl index addf105e592b..81c75ecff04a 100644 --- a/lib/ssl/test/ssl_npn_SUITE.erl +++ b/lib/ssl/test/ssl_npn_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_npn_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Callback functions diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index ee84383c612d..dae07aae6309 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_npn_hello_SUITE). +-behaviour(ct_suite). + -include_lib("ssl/src/tls_record.hrl"). -include_lib("ssl/src/tls_handshake.hrl"). -include_lib("ssl/src/ssl_cipher.hrl"). diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index bd390d67bf06..a65173f1723e 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_packet_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Callback functions diff --git a/lib/ssl/test/ssl_payload_SUITE.erl b/lib/ssl/test/ssl_payload_SUITE.erl index df8394ab148a..e594c745b694 100644 --- a/lib/ssl/test/ssl_payload_SUITE.erl +++ b/lib/ssl/test/ssl_payload_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_payload_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Common test -export([all/0, diff --git a/lib/ssl/test/ssl_pem_cache_SUITE.erl b/lib/ssl/test/ssl_pem_cache_SUITE.erl index 2469b47040ab..50d808905485 100644 --- a/lib/ssl/test/ssl_pem_cache_SUITE.erl +++ b/lib/ssl/test/ssl_pem_cache_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_pem_cache_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). diff --git a/lib/ssl/test/ssl_renegotiate_SUITE.erl b/lib/ssl/test/ssl_renegotiate_SUITE.erl index 18ca56b6f976..4b4686341585 100644 --- a/lib/ssl/test/ssl_renegotiate_SUITE.erl +++ b/lib/ssl/test/ssl_renegotiate_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_renegotiate_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_rfc_5869_SUITE.erl b/lib/ssl/test/ssl_rfc_5869_SUITE.erl index d350dd099da8..497a05107b4c 100644 --- a/lib/ssl/test/ssl_rfc_5869_SUITE.erl +++ b/lib/ssl/test/ssl_rfc_5869_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_rfc_5869_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Common test diff --git a/lib/ssl/test/ssl_session_SUITE.erl b/lib/ssl/test/ssl_session_SUITE.erl index d123b38d5597..0a614f8b8c76 100644 --- a/lib/ssl/test/ssl_session_SUITE.erl +++ b/lib/ssl/test/ssl_session_SUITE.erl @@ -21,6 +21,8 @@ %% -module(ssl_session_SUITE). +-behaviour(ct_suite). + -include("tls_handshake.hrl"). -include("ssl_record.hrl"). diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index c53161f16868..7c182ee063de 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -22,6 +22,8 @@ -module(ssl_session_cache_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Callback functions diff --git a/lib/ssl/test/ssl_session_ticket_SUITE.erl b/lib/ssl/test/ssl_session_ticket_SUITE.erl index 218c4a156471..16791e2c3691 100644 --- a/lib/ssl/test/ssl_session_ticket_SUITE.erl +++ b/lib/ssl/test/ssl_session_ticket_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_session_ticket_SUITE). +-behaviour(ct_suite). + %% Callback functions -export([all/0, groups/0, diff --git a/lib/ssl/test/ssl_sni_SUITE.erl b/lib/ssl/test/ssl_sni_SUITE.erl index 74200c7ab00b..9e2f9fddb1e7 100644 --- a/lib/ssl/test/ssl_sni_SUITE.erl +++ b/lib/ssl/test/ssl_sni_SUITE.erl @@ -21,6 +21,8 @@ -module(ssl_sni_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("kernel/include/inet.hrl"). diff --git a/lib/ssl/test/ssl_socket_SUITE.erl b/lib/ssl/test/ssl_socket_SUITE.erl index a6bce2a414ae..e10ec5afafa1 100644 --- a/lib/ssl/test/ssl_socket_SUITE.erl +++ b/lib/ssl/test/ssl_socket_SUITE.erl @@ -20,6 +20,8 @@ -module(ssl_socket_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 03c8c710ba7c..fc5291b326fe 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -21,6 +21,8 @@ %% -module(ssl_test_lib). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). diff --git a/lib/ssl/test/ssl_upgrade_SUITE.erl b/lib/ssl/test/ssl_upgrade_SUITE.erl index b258b9b05709..cc5db6978873 100644 --- a/lib/ssl/test/ssl_upgrade_SUITE.erl +++ b/lib/ssl/test/ssl_upgrade_SUITE.erl @@ -19,6 +19,8 @@ %% -module(ssl_upgrade_SUITE). +-behaviour(ct_suite). + -include_lib("common_test/include/ct.hrl"). %% Common test From 6fbd047d439e026b29bebf7c8e95489d032d521a Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 20 Oct 2020 23:13:21 +0100 Subject: [PATCH 26/29] Tentatively fix source file generation (trying to have `make test` working) --- lib/common_test/src/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 7913512b1beb..7d7b5ed203d7 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -91,7 +91,8 @@ MODULES= \ TARGET_MODULES= $(MODULES:%=$(EBIN)/%) -BEAM_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +BEAM_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) \ + $(BEHAVIOUR_MODULES:%=$(EBIN)/%.$(EMULATOR)) ERL_FILES= \ $(MODULES:=.erl) \ @@ -124,8 +125,6 @@ TARGET_FILES = \ $(BEAM_FILES) \ $(APP_TARGET) $(APPUP_TARGET) -BEHAVIOUR_TARGET_FILES= $(BEHAVIOUR_MODULES:%=$(EBIN)/%.$(EMULATOR)) - APP_FILE= common_test.app APP_SRC= $(APP_FILE).src APP_TARGET= $(EBIN)/$(APP_FILE) @@ -160,7 +159,7 @@ release_spec: opt $(INSTALL_DIR) "$(RELSYSDIR)/src" $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) "$(RELSYSDIR)/src" $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(BEHAVIOUR_TARGET_FILES) $(TARGET_FILES) "$(RELSYSDIR)/ebin" + $(INSTALL_DATA) $(TARGET_FILES) "$(RELSYSDIR)/ebin" $(INSTALL_DIR) "$(RELSYSDIR)/include" $(INSTALL_DATA) $(EXTERNAL_HRL_FILES) "$(RELSYSDIR)/include" From 84c92b9b776c6d0f094d7ef9faa02be95fcdd806 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 21 Oct 2020 01:01:21 +0100 Subject: [PATCH 27/29] Respect xmllint --- lib/common_test/doc/src/ct_suite.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index 048c766d3213..09a85b23ba85 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -42,31 +42,31 @@ - The name of the testcase function. +

The name of the testcase function.

 - The name of the test group. +

The name of the test group.

 - The configuration data that can be modified. +

The configuration data that can be modified.

 - The status value for a nested subgroup. +

The status value for a nested subgroup.

 - The test group definition. +

The test group definition.

 - The test suite definition. +

The test suite definition.

 - The test suite information. +

The test suite information.

 From ab1f547f5f0fe44b86cc574bc136ee79a8b03f04 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 21 Oct 2020 02:11:14 +0100 Subject: [PATCH 28/29] Adopt a new approach to doc. hyperlinks --- lib/common_test/doc/src/ct_suite.xml | 166 ++++++++++++++++++++------- 1 file changed, 124 insertions(+), 42 deletions(-) diff --git a/lib/common_test/doc/src/ct_suite.xml b/lib/common_test/doc/src/ct_suite.xml index 09a85b23ba85..0289a569f02b 100644 --- a/lib/common_test/doc/src/ct_suite.xml +++ b/lib/common_test/doc/src/ct_suite.xml @@ -57,16 +57,16 @@

The status value for a nested subgroup.

 - -

The test group definition.

 + ct_group_def() +

The test group definition, as returned by Module:groups/0.

 - -

The test suite definition.

 + ct_test_def() +

The test suite definition, as returned by Module:all/0.

 - -

The test suite information.

 + ct_info() +

The test suite information, as returned by Module:suite/0, Module:group/1 and Module:Testcase/0.

 @@ -81,11 +81,19 @@ - Module:all() -> [TestDef] | {skip, Reason} + Module:all() -> [ct_test_def()] | {skip, Reason} Returns the list of all test case groups and test cases in the module. - TestDef = ct_test_def() + ct_test_def() = TestCase | {group, GroupName} | {group, GroupName, Properties} | {group, GroupName, Properties, SubGroups} + TestCase = ct_testname() + GroupName = ct_groupname() + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] | default + SubGroups = [{GroupName, Properties} | {GroupName, Properties, SubGroups}] + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever Reason = term() @@ -100,12 +108,12 @@ indicating that the test case shall be repeated. A test case group is represented by a group tuple, where GroupName, an atom, is the name of the group (defined in - groups/0). + Module:groups/0). Execution properties for groups can also be specified, both for a top-level group and for any of its subgroups. Group execution properties specified here override properties in the group definition (see - groups/0). + Module:groups/0). (With value default, the group definition properties are used).

@@ -121,10 +129,18 @@ - Module:groups() -> [GroupDef] + Module:groups() -> [ct_group_def()] Returns a list of test case group definitions. - GroupDef = ct_group_def() + ct_group_def() = {GroupName, Properties, GroupsAndTestCases} + GroupName = ct_groupname() + Properties = [parallel | sequence | Shuffle | {RepeatType, N}] + GroupsAndTestCases = [Group | {group, GroupName} | TestCase] + TestCase = ct_testname() + Shuffle = shuffle | {shuffle, Seed} + Seed = {integer(), integer(), integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever @@ -137,11 +153,33 @@ - Module:suite() -> [Info] + Module:suite() -> [ct_info()] Test suite info function (providing default data for the suite). - Info = ct_info() + ct_info() = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | +         {CTHModule, CTHInitArgs} | +         {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + CTHPriority = integer() @@ -153,9 +191,9 @@

Tag timetrap sets the maximum time that each test case is allowed to execute (including - init_per_testcase/2 + Module:init_per_testcase/2 and - end_per_testcase/2). + Module:end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. A TimeFunc function can be used to set a new timetrap by returning a TimeVal. It can also be @@ -199,7 +237,7 @@

OPTIONAL; if this function is defined, then end_per_suite/1 + marker="#Module:end_per_suite/1">Module:end_per_suite/1 must also be defined.

This configuration function is called as the first function in the @@ -231,12 +269,12 @@

OPTIONAL; if this function is defined, then init_per_suite/1 + marker="#Module:init_per_suite/1">Module:init_per_suite/1 must also be defined.

This function is called as the last test case in the suite. It is meant to be used for cleaning up after - init_per_suite/1.

+ Module:init_per_suite/1.

For information on save_config, see section Saving Configuration Data in the User's Guide.

@@ -244,13 +282,35 @@ - Module:group(GroupName) -> [Info] + Module:group(GroupName) -> [ct_info()] Test case group information function (providing default data for a test case group, that is, its test cases and subgroups). GroupName = ct_groupname() - Info = ct_info() + ct_info() = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | +         {CTHModule, CTHInitArgs} | +         {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + CTHPriority = integer() @@ -260,15 +320,15 @@ return a list of tagged tuples that specify various properties related to the execution of a test case group (that is, its test cases and subgroups). Properties set by - group/1 override + Module:group/1 override properties with the same key that have been set previously by - suite/0.

+ Module:suite/0.

Tag timetrap sets the maximum time that each test case is allowed to execute (including - init_per_testcase/2 + Module:init_per_testcase/2 and - end_per_testcase/2). + Module:end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. A TimeFunc function can be used to @@ -315,7 +375,7 @@

OPTIONAL; if this function is defined, then end_per_group/2 + marker="#Module:end_per_group/2">Module:end_per_group/2 must also be defined.

This configuration function is called before execution of a @@ -323,7 +383,7 @@ common for all test cases and subgroups in the group, and that must only be performed once. GroupName is the name of the group, as specified in the group definition (see - groups/0). + Module:groups/0). Parameter Config is the configuration data that can be modified. The return value of this function is given as Config @@ -351,15 +411,15 @@

OPTIONAL; if this function is defined, then init_per_group/2 + marker="#Module:init_per_group/2">Module:init_per_group/2 must also be defined.

This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after - init_per_group/2. + Module:init_per_group/2. A status value for a nested subgroup can be returned with {return_group_result, Status}. The status can be retrieved in - end_per_group/2 + Module:end_per_group/2 for the group on the level above. The status is also used by Common Test for deciding if execution of a group is to proceed if property sequence or repeat_until_* @@ -383,7 +443,7 @@

OPTIONAL; if this function is defined, then - end_per_testcase/2 must also be + Module:end_per_testcase/2 must also be defined.

This function is called before each test case. Argument @@ -411,12 +471,12 @@

OPTIONAL; if this function is defined, then - init_per_testcase/2 must also be + Module:init_per_testcase/2 must also be defined.

This function is called after each test case, and can be used to clean up after - init_per_testcase/2 + Module:init_per_testcase/2 and the test case. Any return value (besides {fail, Reason} and {save_config, SaveConfig}) is ignored. By returning {fail, Reason}, TestCase is marked as faulty (even @@ -430,10 +490,32 @@ - Module:Testcase() -> [Info] + Module:Testcase() -> [ct_info()] Test case information function. - Info = ct_info() + ct_info() = {timetrap, Time} | {require, Required} | {require, Name, Required} | {userdata, UserData} | {silent_connections, Conns} | {stylesheet, CSSFile} | {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds, integer()} | {minutes, integer()} | {hours, integer()} + TimeFunc = {Mod, Func, Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key, SubKeys} | {Key, SubKey} | {Key, SubKey, SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | +         {CTHModule, CTHInitArgs} | +         {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + CTHPriority = integer() @@ -444,17 +526,17 @@ return a list of tagged tuples that specify various properties related to the execution of this particular test case. Properties set by - Testcase/0 + Module:Testcase/0 override properties set previously for the test case by - group/1 or - suite/0.

+ Module:group/1 or + Module:suite/0.

Tag timetrap sets the maximum time that the test case is allowed to execute. If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. - init_per_testcase/2 + Module:init_per_testcase/2 and - end_per_testcase/2 + Module:end_per_testcase/2 are included in the timetrap time. A TimeFunc function can be used to set a new timetrap by returning a TimeVal. It can also be @@ -473,8 +555,8 @@

If timetrap or require is not set, the default values specified by - suite/0 (or - group/1) are used.

+ Module:suite/0 (or + Module:group/1) are used.

With userdata, the user can specify any test case-related information that can be read by calling From b48a74f2521052e4e38de94eefcf40c65198813c Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 21 Oct 2020 16:28:44 +0100 Subject: [PATCH 29/29] Fix broken doc. links --- lib/common_test/doc/src/ct.xml | 6 +- lib/common_test/doc/src/ct_hooks.xml | 70 +++++++++---------- lib/common_test/doc/src/ct_hooks_chapter.xml | 36 +++++----- lib/common_test/doc/src/ct_property_test.xml | 2 +- .../doc/src/dependencies_chapter.xml | 10 +-- .../doc/src/test_structure_chapter.xml | 4 +- .../doc/src/write_test_chapter.xml | 22 +++--- 7 files changed, 75 insertions(+), 75 deletions(-) diff --git a/lib/common_test/doc/src/ct.xml b/lib/common_test/doc/src/ct.xml index 0d1c8076971c..c1f638b58054 100644 --- a/lib/common_test/doc/src/ct.xml +++ b/lib/common_test/doc/src/ct.xml @@ -57,9 +57,9 @@

data_dir - Data file directory

priv_dir - Scratch file directory

Whatever added by - init_per_suite/1 + init_per_suite/1 or - init_per_testcase/2 + init_per_testcase/2 in the test suite.

 @@ -1524,7 +1524,7 @@

Returns any data specified with tag userdata in the list of tuples returned from - suite/0.

+ suite/0.

 diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 3c38dcd439be..875efec305a4 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -130,12 +130,12 @@

OPTIONAL

This function is called after - groups/0. + groups/0. It is used to modify the test group definitions, for instance to add or remove groups or change group properties.

GroupDefs is what - groups/0 + groups/0 returned, that is, a list of group definitions.

NewGroupDefs is the possibly modified version of this list.

@@ -146,7 +146,7 @@ in the User's Guide.

Notice that for CTHs that are installed by means of the - suite/0 + suite/0 function, post_groups/2 is called before the init/2 hook function. However, for CTHs that are installed by means @@ -191,19 +191,19 @@

OPTIONAL

This function is called after - all/0. + all/0. It is used to modify the set of test cases and test group to be executed, for instance to add or remove test cases and groups, change group properties, or even skip all tests in the suite.

Return is what - all/0 + all/0 returned, that is, a list of test cases and groups to be executed, or a tuple {skip,Reason}.

GroupDefs is what - groups/0 + groups/0 or the post_groups/2 hook returned, that is, a list of group definitions.

@@ -215,7 +215,7 @@ in the User's Guide.

Notice that for CTHs that are installed by means of the - suite/0 + suite/0 function, post_all/2 is called before the init/2 hook function. However, for CTHs that are installed by means @@ -253,7 +253,7 @@

OPTIONAL

This function is called before - init_per_suite + init_per_suite if it exists. It typically contains initialization/logging that must be done before init_per_suite is called. If {skip,Reason} or {fail,Reason} is returned, @@ -270,11 +270,11 @@

Return is the result of the init_per_suite function. If it is {skip,Reason} or {fail,Reason}, - init_per_suite + init_per_suite is never called, instead the initiation is considered to be skipped or failed, respectively. If a NewConfig list is returned, - init_per_suite + init_per_suite is called with that NewConfig list. For more details, see section Pre Hooks in the User's Guide.

@@ -304,21 +304,21 @@

OPTIONAL

This function is called after - init_per_suite + init_per_suite if it exists. It typically contains extra checks to ensure that all the correct dependencies are started correctly.

Return is what - init_per_suite + init_per_suite returned, that is, {fail,Reason}, {skip,Reason}, a Config list, or a term describing how - init_per_suite + init_per_suite failed.

NewReturn is the possibly modified return value of - init_per_suite. + init_per_suite. To recover from a failure in - init_per_suite, + init_per_suite, return ConfigList with the tc_status element removed. For more details, see Post Hooks in @@ -352,11 +352,11 @@

OPTIONAL

This function is called before - init_per_group + init_per_group if it exists. It behaves the same way as pre_init_per_suite, but for function - init_per_group + init_per_group instead.

If Module:pre_init_per_group/4 is not exported, common_test @@ -385,11 +385,11 @@

OPTIONAL

This function is called after - init_per_group + init_per_group if it exists. It behaves the same way as post_init_per_suite, but for function - init_per_group + init_per_group instead.

If Module:post_init_per_group/5 is not exported, common_test @@ -418,11 +418,11 @@

OPTIONAL

This function is called before - init_per_testcase + init_per_testcase if it exists. It behaves the same way as pre_init_per_suite, but for function - init_per_testcase + init_per_testcase instead.

If Module:pre_init_per_testcase/4 is not exported, common_test @@ -455,11 +455,11 @@

OPTIONAL

This function is called after - init_per_testcase + init_per_testcase if it exists. It behaves the same way as post_init_per_suite, but for function - init_per_testcase + init_per_testcase instead.

If Module:post_init_per_testcase/5 is not exported, common_test @@ -487,11 +487,11 @@

OPTIONAL

This function is called before - end_per_testcase + end_per_testcase if it exists. It behaves the same way as pre_end_per_suite, but for function - end_per_testcase + end_per_testcase instead.

This function cannot change the result of the test case by returning skip or fail @@ -524,11 +524,11 @@

OPTIONAL

This function is called after - end_per_testcase + end_per_testcase if it exists. It behaves the same way as post_end_per_suite, but for function - end_per_testcase + end_per_testcase instead.

If Module:post_end_per_testcase/5 is not exported, common_test @@ -557,11 +557,11 @@

OPTIONAL

This function is called before - end_per_group + end_per_group if it exists. It behaves the same way as pre_init_per_suite, but for function - end_per_group + end_per_group instead.

If Module:pre_end_per_group/4 is not exported, common_test @@ -590,11 +590,11 @@

OPTIONAL

This function is called after - end_per_group + end_per_group if it exists. It behaves the same way as post_init_per_suite, but for function - end_per_group + end_per_group instead.

If Module:post_end_per_group/5 is not exported, common_test @@ -622,11 +622,11 @@

OPTIONAL

This function is called before - end_per_suite + end_per_suite if it exists. It behaves the same way as pre_init_per_suite, but for function - end_per_suite + end_per_suite instead.

 @@ -649,11 +649,11 @@

OPTIONAL

This function is called after - end_per_suite + end_per_suite if it exists. It behaves the same way as post_init_per_suite, but for function - end_per_suite + end_per_suite instead.

diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index e23a8b461a58..a4c72c1fa414 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -84,10 +84,10 @@

CTHs can also be added within a test suite. This is done by returning {ct_hooks,[CTH]} in the configuration list from - suite/0, - + suite/0, + init_per_suite/1, or - + init_per_group/2.

In this case, CTH can either be only the module name of the CTH @@ -157,7 +157,7 @@ the last test suite has been run - suite/0 + suite/0 pre_init_per_suite/3 is called @@ -165,7 +165,7 @@ post_end_per_suite/4 has been called for that test suite - + init_per_suite/1 post_init_per_suite/4 is called @@ -173,7 +173,7 @@ post_end_per_suite/4 has been called for that test suite - + init_per_group/2 post_init_per_group/5 is called @@ -236,12 +236,12 @@ In a CTH, the behavior can be hooked in before the following functions:

- init_per_suite - init_per_group - init_per_testcase - end_per_testcase - end_per_group - end_per_suite + init_per_suite + init_per_group + init_per_testcase + end_per_testcase + end_per_group + end_per_suite

@@ -282,12 +282,12 @@ Post Hooks

In a CTH, behavior can be hooked in after the following functions:

- init_per_suite - init_per_group - init_per_testcase - end_per_testcase - end_per_group - end_per_suite + init_per_suite + init_per_group + init_per_testcase + end_per_testcase + end_per_group + end_per_suite

diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 891e0b475b7c..4c5eede7584e 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -186,7 +186,7 @@ prop_ftp() -> the output from for example proper:run_commands/2 or proper:run_parallel_commands/2 Config = - the Common Test Config in test cases. + the Common Test Config in test cases. Options = [present_option()] present_option() = {print_fun, fun(Format,Args)} diff --git a/lib/common_test/doc/src/dependencies_chapter.xml b/lib/common_test/doc/src/dependencies_chapter.xml index 53a767348928..0a0f4c562fc6 100644 --- a/lib/common_test/doc/src/dependencies_chapter.xml +++ b/lib/common_test/doc/src/dependencies_chapter.xml @@ -87,8 +87,8 @@

To avoid this, we can consider starting and stopping the server for every test. We can thus implement the start and stop action as common functions to be called from - init_per_testcase and - end_per_testcase. + init_per_testcase and + end_per_testcase. (Remember to test the start and stop functionality separately.) The configuration can also be implemented as a common function, maybe grouped with the start function. Finally, the testing of connecting and disconnecting a @@ -194,9 +194,9 @@

To pass data from one test suite to another, the same mechanism is used. The data is to be saved by finction - end_per_suite + end_per_suite and read by function - init_per_suite + init_per_suite in the suite that follows. When passing data between suites, Saver carries the name of the test suite.

@@ -306,7 +306,7 @@ any property, that is, they are not required to also be sequences. If you want the status of the subgroup to affect the sequence on the level above, return {return_group_result,Status} from - end_per_group/2, + end_per_group/2, as described in section Repeated Groups in Writing Test Suites. diff --git a/lib/common_test/doc/src/test_structure_chapter.xml b/lib/common_test/doc/src/test_structure_chapter.xml index 42eb3d6e236b..e041376d7f32 100644 --- a/lib/common_test/doc/src/test_structure_chapter.xml +++ b/lib/common_test/doc/src/test_structure_chapter.xml @@ -60,8 +60,8 @@ Returning {skip,Reason} from function - init_per_testcase/2 or - init_per_suite/1. + init_per_testcase/2 or + init_per_suite/1. Returning {skip,Reason} from the execution clause of the test case. The execution clause is called, so the author diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 3c3790cfc530..661ba04fe887 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -67,7 +67,7 @@

Each test suite module must export function - all/0, + all/0, which returns the list of all test case groups and test cases to be executed in that module.

@@ -83,8 +83,8 @@ Init and End per Suite

Each test suite module can contain the optional configuration functions - init_per_suite/1 - and end_per_suite/1. + init_per_suite/1 + and end_per_suite/1. If the init function is defined, so must the end function be.

@@ -138,8 +138,8 @@ Init and End per Test Case

Each test suite module can contain the optional configuration functions - init_per_testcase/2 - and end_per_testcase/2. + init_per_testcase/2 + and end_per_testcase/2. If the init function is defined, so must the end function be.

If init_per_testcase exists, it is called before each @@ -378,7 +378,7 @@

If timetrap or require, or both, is not set specifically for a particular test case, default values specified by function - suite/0 + suite/0 are used.

@@ -404,7 +404,7 @@ Test Suite Information Function -

Function suite/0 +

Function suite/0 can, for example, be used in a test suite module to set a default timetrap value and to require external configuration data. If a test case, or a group information function also specifies any of the information tags, it @@ -445,7 +445,7 @@

A test case group is a set of test cases sharing configuration functions and execution properties. Test case groups are defined by function - groups/0 + groups/0 according to the following syntax:

  groups() -> GroupDefs
@@ -561,12 +561,12 @@
       execution is immediately stopped and the remaining cases are skipped.

 
     
Before execution of a group begins, the configuration function
-    init_per_group(GroupName, Config) 
+    init_per_group(GroupName, Config) 
     is called. The list of tuples returned from this function is passed to the 
     test cases in the usual manner by argument Config. 
     init_per_group/2 is meant to be used for initializations common 
     for the test cases in the group. After execution of the group is finished, function
-    end_per_group(GroupName, Config) 
+    end_per_group(GroupName, Config) 
     is called. This function is meant to be used for cleaning up after 
     init_per_group/2. If the init function is defined, so must the end function be.

 
@@ -1163,7 +1163,7 @@ ct:pal(?ERROR, "Error report: ~p", [Error])
environment as possible, so that subsequent test cases do not crash because of their execution order. The function - end_per_testcase + end_per_testcase is suitable for this.