Merge branch 'develop' into pr-lowercase-userid

src/feditest/__init__.py

@@ -117,9 +117,11 @@ def load_default_tests() -> None:
     """
     global all_tests
 
-    all_tests['noop2'] = TestFromTestFunction('noop2', 'This denegerate 2-node test does nothing', lambda node1, node2: None )
-    all_tests['noop3'] = TestFromTestFunction('noop3', 'This denegerate 3-node test does nothing', lambda node1, node2, node3: None )
-    all_tests['noop4'] = TestFromTestFunction('noop4', 'This denegerate 4-node test does nothing', lambda node1, node2, node3, node4: None )
+    all_tests['noop0'] = TestFromTestFunction('noop0', 'This denegerate 0-node test does nothing', lambda: None, builtin=True )
+    all_tests['noop1'] = TestFromTestFunction('noop1', 'This denegerate 1-node test does nothing', lambda node1: None, builtin=True )
+    all_tests['noop2'] = TestFromTestFunction('noop2', 'This denegerate 2-node test does nothing', lambda node1, node2: None, builtin=True )
+    all_tests['noop3'] = TestFromTestFunction('noop3', 'This denegerate 3-node test does nothing', lambda node1, node2, node3: None, builtin=True )
+    all_tests['noop4'] = TestFromTestFunction('noop4', 'This denegerate 4-node test does nothing', lambda node1, node2, node3, node4: None, builtin=True )
     # Do not replace those lambda parameters with _: we need to look up their names for role mapping
 
 

src/feditest/cli/__init__.py

@@ -2,7 +2,7 @@
 Main entry point for CLI invocation
 """
 
-from argparse import ArgumentParser, Action
+from argparse import ArgumentError, ArgumentParser, Action
 import importlib
 import sys
 import traceback
@@ -39,6 +39,8 @@ def main() -> None:
             ret = cmds[cmd_name].run(cmd_sub_parsers[cmd_name], args, remaining)
             sys.exit( ret )
 
+        except ArgumentError as e:
+            fatal(e.message)
         except Exception as e: # pylint: disable=broad-exception-caught
             if args.verbose > 1:
                 traceback.print_exception( e )

src/feditest/cli/commands/create_constellation.py

@@ -2,10 +2,9 @@
 Combine node definitions into a constellation.
 """
 
-from argparse import ArgumentParser, Namespace, _SubParsersAction
+from argparse import ArgumentError, ArgumentParser, Namespace, _SubParsersAction
 
 from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode
-from feditest.reporting import fatal
 
 
 def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int:
@@ -22,11 +21,11 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int:
     for nodepair in args.node:
         rolename, nodefile = nodepair.split('=', 1)
         if not rolename:
-            fatal('Rolename component must not be empty:', nodepair)
+            raise ArgumentError(None, f'Rolename component of --node must not be empty: "{ nodepair }".')
         if rolename in roles:
-            fatal('Role is already taken:', rolename)
+            raise ArgumentError(None, f'Role is already taken: "{ rolename }".')
         if not nodefile:
-            fatal('Filename component must not be empty:', nodepair)
+            raise ArgumentError(None, f'Filename component must not be empty: "{ nodepair }".')
         node = TestPlanConstellationNode.load(nodefile)
         roles[rolename] = node
 

src/feditest/cli/commands/create_session_template.py

@@ -32,6 +32,8 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int:
             test = feditest.all_tests.get(name)
             if test is None: # make linter happy
                 continue
+            if test.builtin:
+                continue
 
             test_plan_spec = TestPlanTestSpec(name)
             test_plan_specs.append(test_plan_spec)

src/feditest/cli/commands/info.py

@@ -42,7 +42,12 @@ def run_info_test(name: str) -> int:
     """
     test = feditest.all_tests.get(name)
     if test:
-        print(format_name_value_string(test.metadata()), end='')
+        test_metadata = test.metadata()
+        needed_role_names = test.needed_local_role_names()
+        if needed_role_names:
+            test_metadata['Needed roles:'] = sorted(needed_role_names)
+
+        print(format_name_value_string(test_metadata), end='')
         return 0
 
     warning( 'Test not found:', name)

src/feditest/cli/commands/run.py

@@ -2,12 +2,17 @@
 Run one or more tests
 """
 
-from argparse import ArgumentParser, Namespace, _SubParsersAction
+from argparse import ArgumentError, ArgumentParser, Namespace, _SubParsersAction
+import re
+from typing import Any
+
+from msgspec import ValidationError
 
 import feditest
 from feditest.registry import Registry, set_registry_singleton
 from feditest.reporting import warning
-from feditest.testplan import TestPlan
+from feditest.tests import Test
+from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSession, TestPlanTestSpec
 from feditest.testrun import TestRun
 from feditest.testruncontroller import AutomaticTestRunController, InteractiveTestRunController, TestRunController
 from feditest.testruntranscript import (
@@ -40,7 +45,24 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int:
     if args.domain:
         set_registry_singleton(Registry.create(args.domain)) # overwrite
 
-    plan = TestPlan.load(args.testplan)
+    # Determine testplan. While we are at it, check consistency of arguments.
+    if args.testplan:
+        plan = _create_plan_from_testplan(args)
+    else:
+        session_templates = _create_session_templates(args)
+        constellations = _create_constellations(args)
+
+        sessions = []
+        for session_template in session_templates:
+            for constellation in constellations:
+                session = session_template.instantiate_with_constellation(constellation, constellation.name)
+                sessions.append(session)
+        if sessions:
+            plan = TestPlan(sessions, None)
+            plan.simplify()
+        else: # neither sessions nor testplan specified
+            plan = TestPlan.load("feditest-default.json")
+
     if not plan.is_compatible_type():
         warning(f'Test plan has unexpected type { plan.type }: incompatibilities may occur.')
     if not plan.has_compatible_version():
@@ -86,15 +108,26 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None:
     parent_parser: the parent argparse parser
     cmd_name: name of this command
     """
+    # general flags and options
     parser = parent_parser.add_parser(cmd_name, help='Run one or more tests' )
     parser.add_argument('--testsdir', nargs='*', default=['tests'], help='Directory or directories where to find tests')
-    parser.add_argument('--testplan', default='feditest-default.json', help='Name of the file that contains the test plan to run')
     parser.add_argument('--nodedriversdir', action='append', help='Directory or directories where to find extra drivers for nodes that can be tested')
     parser.add_argument('--domain', type=hostname_validate, help='Local-only DNS domain for the DNS hostnames that are auto-generated for nodes')
     parser.add_argument('--interactive', action="store_true",
                         help="Run the tests interactively")
     parser.add_argument('--who', action='store_true',
                         help="Record who ran the test plan on what host.")
+
+    # test plan options. We do not use argparse groups, as the situation is more complicated than argparse seems to support
+    parser.add_argument('--testplan', help='Name of the file that contains the test plan to run')
+    parser.add_argument('--constellation', nargs='+', help='File(s) each containing a JSON fragment defining a constellation')
+    parser.add_argument('--session', '--session-template', nargs='+', help='File(s) each containing a JSON fragment defining a test session')
+    parser.add_argument('--node', action='append',
+                        help="Use role=file to specify that the node definition in 'file' is supposed to be used for constellation role 'role'")
+    parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression')
+    parser.add_argument('--test', nargs='+', help='Run this/these named tests(s)')
+
+    # output options
     parser.add_argument('--tap', nargs="?", const=True, default=False,
                         help="Write results in TAP format to stdout, or to the provided file (if given).")
     html_group = parser.add_argument_group('html', 'HTML options')
@@ -108,3 +141,108 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None:
                         help="Write summary to stdout, or to the provided file (if given). This is the default if no other output option is given")
 
     return parser
+
+
+def _create_plan_from_testplan(args: Namespace) -> TestPlan:
+    if args.constellation:
+        raise ArgumentError(None, '--testplan already defines --constellation. Do not provide both.')
+    if args.session:
+        raise ArgumentError(None, '--testplan already defines --session-template. Do not provide both.')
+    if args.node:
+        raise ArgumentError(None, '--testplan already defines --node via the contained constellation. Do not provide both.')
+    if args.test:
+        raise ArgumentError(None, '--testplan already defines --test via the contained session. Do not provide both.')
+    plan = TestPlan.load(args.testplan)
+    return plan
+
+
+def _create_session_templates(args: Namespace) -> list[TestPlanSession]:
+    if args.session:
+        if args.filter_regex:
+            raise ArgumentError(None, '--session already defines the tests, do not provide --filter-regex')
+        if args.test:
+            raise ArgumentError(None, '--session already defines --test. Do not provide both.')
+        session_templates = []
+        for session_file in args.session:
+            session_templates.append(TestPlanSession.load(session_file))
+        return session_templates
+
+    test_plan_specs : list[TestPlanTestSpec]= []
+    constellation_role_names : dict[str,Any] = {}
+    constellation_roles: dict[str,TestPlanConstellationNode | None] = {}
+    tests : list[Test]= []
+
+    if args.test:
+        if args.filter_regex:
+            raise ArgumentError(None, '--filter-regex already defines --test. Do not provide both.')
+        for name in args.test:
+            test = feditest.all_tests.get(name)
+            if test is None:
+                raise ArgumentError(None, f'Cannot find test: "{ name }".')
+            tests.append(test)
+
+    elif args.filter_regex:
+        pattern = re.compile(args.filter_regex)
+        for name in sorted(feditest.all_tests.keys()):
+            if pattern.match(name):
+                test = feditest.all_tests.get(name)
+                if test is None: # make linter happy
+                    continue
+                if test.builtin:
+                    continue
+                tests.append(test)
+
+    else:
+        for name in sorted(feditest.all_tests.keys()):
+            test = feditest.all_tests.get(name)
+            if test is None: # make linter happy
+                continue
+            if test.builtin:
+                continue
+            tests.append(test)
+
+    for test in tests:
+        test_plan_spec = TestPlanTestSpec(name)
+        test_plan_specs.append(test_plan_spec)
+
+        for role_name in test.needed_local_role_names():
+            constellation_role_names[role_name] = 1
+            if not test_plan_spec.rolemapping:
+                test_plan_spec.rolemapping = {}
+            test_plan_spec.rolemapping[role_name] = role_name
+
+    for constellation_role_name in constellation_role_names:
+        constellation_roles[constellation_role_name] = None
+
+    session = TestPlanSession(TestPlanConstellation(constellation_roles), test_plan_specs)
+    return [ session ]
+
+
+def _create_constellations(args: Namespace) -> list[TestPlanConstellation]:
+    if args.constellation:
+        if args.node:
+            raise ArgumentError(None, '--constellation already defines --node. Do not provide both.')
+
+        constellations = []
+        for constellation_file in args.constellation:
+            try:
+                constellations.append(TestPlanConstellation.load(constellation_file))
+            except ValidationError as e:
+                raise ArgumentError(None, f'Constellation file { constellation_file }: { e }')
+        return constellations
+
+    # Don't check for empty nodes: we need that for testing feditest
+    roles : dict[str, TestPlanConstellationNode | None] = {}
+    for nodepair in args.node:
+        rolename, nodefile = nodepair.split('=', 1)
+        if not rolename:
+            raise ArgumentError(None, f'Rolename component of --node must not be empty: "{ nodepair }".')
+        if rolename in roles:
+            raise ArgumentError(None, f'Role is already taken: "{ rolename }".')
+        if not nodefile:
+            raise ArgumentError(None, f'Filename component must not be empty: "{ nodepair }".')
+        node = TestPlanConstellationNode.load(nodefile)
+        roles[rolename] = node
+
+    constellation = TestPlanConstellation(roles)
+    return [ constellation ]

src/feditest/nodedrivers/fallback/fediverse.py

@@ -217,16 +217,17 @@ def make_create_note(self, actor_uri: str, content: str, deliver_to: list[str] |
 
 
     # Python 3.12 @override
-    def make_announce_object(self, actor_uri, announced_object_uri: str) -> str:
+    def make_announce_object(self, actor_uri, to_be_announced_object_uri: str) -> str:
         return cast(str, self.prompt_user(
-                f'On FediverseNode "{ self.hostname }", make actor "{ actor_uri }" boost "{ announced_object_uri }"'
-                + ' and enter the Announce object\'s local URI:'))
+                f'On FediverseNode "{ self.hostname }", make actor "{ actor_uri }" boost "{ to_be_announced_object_uri }"'
+                + ' and enter the Announce object\'s local URI:',
+                parse_validate=https_uri_validate))
 
 
     # Python 3.12 @override
-    def make_reply_note(self, actor_uri, replied_object_uri: str, reply_content: str) -> str:
+    def make_reply_note(self, actor_uri, to_be_replied_to_object_uri: str, reply_content: str) -> str:
         return cast(str, self.prompt_user(
-                f'On FediverseNode "{ self.hostname }", make actor "{ actor_uri }" reply to object with "{ replied_object_uri }"'
+                f'On FediverseNode "{ self.hostname }", make actor "{ actor_uri }" reply to object with "{ to_be_replied_to_object_uri }"'
                 + ' and enter the Announce object\'s URI when created.'
                 + f' Reply content:"""\n{ reply_content }\n"""' ))
 
@@ -244,7 +245,7 @@ def make_follow(self, actor_uri: str, to_follow_actor_uri: str) -> None:
     # Python 3.12 @override
     def wait_until_actor_is_following_actor(self, actor_uri: str, to_be_followed_uri: str, max_wait: float = 5.) -> None:
         answer = self.prompt_user(
-                f'On FediverseNode "{ self.hostname }", wait until in actor "{ actor_uri }" is following actor "{ to_be_followed_uri }"'
+                f'On FediverseNode "{ self.hostname }", wait until actor "{ actor_uri }" is following actor "{ to_be_followed_uri }"'
                 + ' and enter "true"; "false" if it didn\'t happen.',
                 parse_validate=boolean_parse_validate)
         if not answer:
@@ -254,13 +255,33 @@ def wait_until_actor_is_following_actor(self, actor_uri: str, to_be_followed_uri
     # Python 3.12 @override
     def wait_until_actor_is_followed_by_actor(self, actor_uri: str, to_be_following_uri: str, max_wait: float = 5.) -> None:
         answer = self.prompt_user(
-                f'On FediverseNode "{ self.hostname }", wait until in actor "{ actor_uri }" is followed by actor "{ to_be_following_uri }"'
+                f'On FediverseNode "{ self.hostname }", wait until actor "{ actor_uri }" is followed by actor "{ to_be_following_uri }"'
                 + ' and enter "true"; "false" if it didn\'t happen.',
                 parse_validate=boolean_parse_validate)
         if not answer:
             raise TimeoutException(f'Actor { actor_uri } not followed by actor { to_be_following_uri}.', max_wait)
 
 
+    # Python 3.12 @override
+    def wait_until_actor_is_unfollowing_actor(self, actor_uri: str, to_be_unfollowed_uri: str, max_wait: float = 5.) -> None:
+        answer = self.prompt_user(
+                f'On FediverseNode "{ self.hostname }", wait until actor "{ actor_uri }" is not following any more actor "{ to_be_unfollowed_uri }"'
+                + ' and enter "true"; "false" if it didn\'t happen.',
+                parse_validate=boolean_parse_validate)
+        if not answer:
+            raise TimeoutException(f'Actor { actor_uri } still following actor { to_be_unfollowed_uri}.', max_wait)
+
+
+    # Python 3.12 @override
+    def wait_until_actor_is_unfollowed_by_actor(self, actor_uri: str, to_be_unfollowing_uri: str, max_wait: float = 5.) -> None:
+        answer = self.prompt_user(
+                f'On FediverseNode "{ self.hostname }", wait until in actor "{ actor_uri }" is not followed any more by actor "{ to_be_unfollowing_uri }"'
+                + ' and enter "true"; "false" if it didn\'t happen.',
+                parse_validate=boolean_parse_validate)
+        if not answer:
+            raise TimeoutException(f'Actor { actor_uri } is still followed by actor { to_be_unfollowing_uri}.', max_wait)
+
+
 class AbstractFallbackFediverseNodeDriver(NodeDriver):
     """
     Abstract superclass of NodeDrivers that support all web server-side protocols but don't