From d2e719403754d5cb59a9cf6dc7b6693f620d3e00 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 9 Feb 2023 12:37:19 +0000 Subject: [PATCH] Add functest-run-module command for centralised tests This command allows an arbitrary module to be loaded and then a function to be run that is passed any remaining parameters from the command. This allows changes to be be to both the command and the runner without having to modify zaza. It is going to be used by the centralised tests stream of work. --- setup.py | 1 + zaza/charm_lifecycle/deploy.py | 13 +++-- zaza/charm_lifecycle/func_test_runner.py | 4 +- zaza/charm_lifecycle/run_module.py | 71 ++++++++++++++++++++++++ zaza/charm_lifecycle/utils.py | 35 ++++++++++-- zaza/utilities/cli.py | 2 +- 6 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 zaza/charm_lifecycle/run_module.py diff --git a/setup.py b/setup.py index 87445e411..d03315fea 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ def run_tests(self): entry_points={ 'console_scripts': [ 'functest-run-suite = zaza.charm_lifecycle.func_test_runner:main', + 'functest-run-module = zaza.charm_lifecycle.run_module:main', 'functest-before-deploy = zaza.charm_lifecycle.before_deploy:main', 'functest-deploy = zaza.charm_lifecycle.deploy:main', 'functest-configure = zaza.charm_lifecycle.configure:main', diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index dc2d7b301..eb9fa929c 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -153,9 +153,15 @@ def get_template(target_file, template_dir=None): :param target_dir: Limit template loading to this directory. :type target_dir: str :returns: Template object used to generate target_file - :rtype: jinja2.Template + :rtype: Optional[jinja2.Template] """ jinja2_env = get_jinja2_env(template_dir=template_dir) + # first see if the non .j2 extension exists; if so then use that + try: + template = jinja2_env.get_template(os.path.basename(target_file)) + return template + except jinja2.exceptions.TemplateNotFound: + pass try: template = jinja2_env.get_template(get_template_name(target_file)) except jinja2.exceptions.TemplateNotFound: @@ -346,11 +352,6 @@ def deploy_bundle(bundle, model, model_ctxt=None, force=False, trust=False): bundle, template_dir=os.path.dirname(bundle)) if bundle_template: - if os.path.exists(bundle): - raise zaza_exceptions.TemplateConflict( - "Found bundle template ({}) and bundle ({})".format( - bundle_template.filename, - bundle)) bundle_out = '{}/{}'.format(tmpdirname, os.path.basename(bundle)) render_template(bundle_template, bundle_out, model_ctxt=model_ctxt) cmd.append(bundle_out) diff --git a/zaza/charm_lifecycle/func_test_runner.py b/zaza/charm_lifecycle/func_test_runner.py index 814357132..2aba3cbba 100644 --- a/zaza/charm_lifecycle/func_test_runner.py +++ b/zaza/charm_lifecycle/func_test_runner.py @@ -266,7 +266,7 @@ def func_test_runner(keep_last_model=False, keep_all_models=False, if '_bundles' in name: all_bundles[name] = values matching_bundles = set() - for _name, bundles in all_bundles.items(): + for bundles in all_bundles.values(): if bundles: for tests_bundle in bundles: if isinstance(tests_bundle, dict): @@ -277,7 +277,7 @@ def func_test_runner(keep_last_model=False, keep_all_models=False, if len(set(matching_bundles)) == 1: model_alias = matching_bundles.pop() else: - logging.info('Could not determine correct model alias' + logging.info('Could not determine correct model alias ' 'from tests.yaml, using default') model_alias = utils.DEFAULT_MODEL_ALIAS deploy[model_alias] = bundle diff --git a/zaza/charm_lifecycle/run_module.py b/zaza/charm_lifecycle/run_module.py new file mode 100644 index 000000000..eb507c533 --- /dev/null +++ b/zaza/charm_lifecycle/run_module.py @@ -0,0 +1,71 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run an arbitrary module with parameters. + +The function allows the caller to specify a specific module to call and pass +arguments to. + +The module is specified as a dotted list of valid python modules with the last +one being the function to call in that module. e.g. + + mod1.mod2.mod3.function +""" +import argparse +import asyncio +import logging +import sys + +import zaza +import zaza.utilities.cli as cli_utils +import zaza.charm_lifecycle.utils as utils + + +def parse_args(args): + """Parse command line arguments. + + :param args: List of configure functions functions + :type list: [str1, str2,...] List of command line arguments + :returns: Parsed arguments + :rtype: Tuple[Namespace, List[str]] + """ + parser = argparse.ArgumentParser() + parser.add_argument('module', + help=('The module to run.')) + parser.add_argument('--log', dest='loglevel', + help='Loglevel [DEBUG|INFO|WARN|ERROR|CRITICAL]') + parser.set_defaults(loglevel='INFO') + return parser.parse_known_args(args) + + +def main(): + """Execute full test run.""" + # known_args are the remaining args to pass to the module function that is + # being run. + args, known_args = parse_args(sys.argv[1:]) + + cli_utils.setup_logging(log_level=args.loglevel.upper()) + + # now find the module, load it, and then pass control to it. + function = None + try: + function = utils.load_module_and_getattr(args.module) + except AttributeError: + logging.error("Couldn't find function %s", args.module) + if function is not None: + try: + function(known_args) + finally: + zaza.clean_up_libjuju_thread() + asyncio.get_event_loop().close() diff --git a/zaza/charm_lifecycle/utils.py b/zaza/charm_lifecycle/utils.py index 0e9117996..ea30b06c1 100644 --- a/zaza/charm_lifecycle/utils.py +++ b/zaza/charm_lifecycle/utils.py @@ -14,6 +14,7 @@ """Utilities to support running lifecycle phases.""" import collections +import collections.abc import copy import importlib import logging @@ -539,13 +540,39 @@ def get_class(class_str): :returns: Test class :rtype: class """ + return load_module_and_getattr(class_str, syspath_prepend=['.']) + + +def load_module_and_getattr(path_str, syspath_prepend=None): + """Load a module and get the attribute at the end of the dotted string. + + This parses a string and attempts to load the module, and assumes the last + part of the string is an attribute to return. The path is assumed to be + the current path of the executable. Pass `insert_path=['.']` to prepend a + the working directory (the default for `get_class`). + + example. + load_module_and_getattr('zaza.openstack.charm_tests.aodh.tests.AodhTest') + + will reture zaza.openstack.charm_tests.aodh.tests.AoghTest + + :param path_str: the path to load, appended with a attribute to return. + :type path_str: str + :param syspath_prepend: optional paths to prepend to the syspath. + :type syspath_prepend: Optional[List[str]] + :returns: the attribute at the end of the dotted str. + :rtype: Any + """ old_syspath = sys.path - sys.path.insert(0, '.') - module_name = '.'.join(class_str.split('.')[:-1]) - class_name = class_str.split('.')[-1] + if syspath_prepend is not None: + assert type(syspath_prepend) is not str, "Must pass a string!" + assert isinstance(syspath_prepend, collections.abc.Iterable) + sys.path[0:0] = syspath_prepend + module_name = '.'.join(path_str.split('.')[:-1]) + attr_name = path_str.split('.')[-1] module = importlib.import_module(module_name) sys.path = old_syspath - return getattr(module, class_name) + return getattr(module, attr_name) def generate_model_name(): diff --git a/zaza/utilities/cli.py b/zaza/utilities/cli.py index ca74de27b..f6d80e084 100644 --- a/zaza/utilities/cli.py +++ b/zaza/utilities/cli.py @@ -43,7 +43,7 @@ def parse_arg(options, arg, multiargs=False): def setup_logging(log_level='INFO'): """Do setup for logging. - :returns: Nothing: This fucntion is executed for its sideffect + :returns: Nothing: This function is executed for its sideffect :rtype: None """ level = getattr(logging, log_level.upper(), None)