From a2ea2f20ce893c9cee945678d4bfbc82067a9105 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Tue, 12 Mar 2019 13:34:10 +0200 Subject: [PATCH 01/17] Add slash.ignored_warnings context --- doc/changelog.rst | 1 + doc/warnings.rst | 6 ++++++ slash/__init__.py | 2 +- slash/warnings.py | 17 ++++++++++++++++- tests/test_warning_ignored.py | 16 ++++++++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index f174c351..7f8d62df 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,7 @@ Changelog ========= +* :feature:`-` Add ``slash.ignored_warnings`` context * :bug:`930` Restore behavior of exceptions propagating out of the test_start or test_end hooks. Correct behavior is for those to fail the test (thanks @pierreluctg) * :bug:`934` Parallel sessions now honor fatal exceptions encountered in worker sessions * :bug:`928` Fixed a bug causing requirements to leak across sibling test classes diff --git a/doc/warnings.rst b/doc/warnings.rst index bf78628e..14a499e3 100644 --- a/doc/warnings.rst +++ b/doc/warnings.rst @@ -26,3 +26,9 @@ For example, you may want to include code in your project's ``.slashrc`` as foll .. note:: Filter arguments to ignore_warnings are treated as though they are ``and``ed together. This means that a filter for a specific filename and a specific category would only ignore warnings coming from the specified file *and* having the specified category. + +For ignoring warnings in specific code-block, one can use the `slash.ignored_warnings` context: +.. code-block:: python + + with slash.ignore_warnings(category=DeprecationWarning, filename='/some/bad/file.py'): + ... diff --git a/slash/__init__.py b/slash/__init__.py index 9b30ece4..036822dd 100644 --- a/slash/__init__.py +++ b/slash/__init__.py @@ -41,7 +41,7 @@ from .core.requirements import requires from .utils import skip_test, skipped, add_error, add_failure, set_test_detail, repeat, register_skip_exception from .utils.interactive import start_interactive_shell -from .warnings import ignore_warnings, clear_ignored_warnings +from .warnings import ignore_warnings, ignored_warnings, clear_ignored_warnings from .runner import run_tests import logbook logger = logbook.Logger(__name__) diff --git a/slash/warnings.py b/slash/warnings.py index a7bb4220..86d26162 100644 --- a/slash/warnings.py +++ b/slash/warnings.py @@ -8,6 +8,7 @@ from . import hooks from .utils.warning_capture import warning_callback_context from .ctx import context +from .exception_handling import handling_exceptions from contextlib import contextmanager @@ -188,7 +189,21 @@ def ignore_warnings(category=None, message=None, filename=None, lineno=None): .. note:: Calling ignore_warnings() with no arguments will ignore **all** warnings """ - _ignored_warnings.append(_IgnoredWarning(category=category, filename=filename, lineno=lineno, message=message)) + iw = _IgnoredWarning(category=category, filename=filename, lineno=lineno, message=message) + _ignored_warnings.append(iw) + return iw + + +@contextmanager +def ignored_warnings(**kwargs): + iw = ignore_warnings(**kwargs) + with handling_exceptions(): + yield + try: + _ignored_warnings.remove(iw) + except ValueError: + # In case clear_warnings was called inside this context, ValueError will be raised + pass def clear_ignored_warnings(): diff --git a/tests/test_warning_ignored.py b/tests/test_warning_ignored.py index e6c076f8..318eb9b0 100644 --- a/tests/test_warning_ignored.py +++ b/tests/test_warning_ignored.py @@ -74,6 +74,22 @@ def test_ignore_warnigns_with_no_parameter(): assert not session.warnings +@pytest.mark.parametrize('should_clear_warnings', [True, False]) +def test_ignored_warnings(should_clear_warnings): + message = 'my warning' + with slash.Session() as session: + with slash.ignored_warnings(message=message): + warnings.warn(message=message) + assert not session.warnings + if should_clear_warnings: + slash.clear_ignored_warnings() + + warnings.warn(message=message) + assert len(session.warnings) == 1 + [caught] = list(session.warnings) + assert caught.message == message + + @pytest.fixture(autouse=True) def warnings_cleanup(request): request.addfinalizer(slash.clear_ignored_warnings) From c6caebe263193b684c9e1b5a94372cde95b17511 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Tue, 19 Mar 2019 11:13:58 +0200 Subject: [PATCH 02/17] Add bors --- .travis.yml | 6 ++++++ bors.toml | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 bors.toml diff --git a/.travis.yml b/.travis.yml index a0570c74..f347f648 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,12 @@ after_success: after_failure: - .env/bin/pip freeze +branches: + except: + - trying.tmp + - staging.tmp + + deploy: - provider: pypi user: vmalloc diff --git a/bors.toml b/bors.toml new file mode 100644 index 00000000..5279fe72 --- /dev/null +++ b/bors.toml @@ -0,0 +1,3 @@ +status = [ + "continuous-integration/travis-ci/push" +] From 9ee34acdff630197b3989ddbaddea0ac98be6d93 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Tue, 19 Mar 2019 11:07:05 +0200 Subject: [PATCH 03/17] Add repr method to _IgnoredWarning --- slash/warnings.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/slash/warnings.py b/slash/warnings.py index 86d26162..7a9d67ed 100644 --- a/slash/warnings.py +++ b/slash/warnings.py @@ -174,6 +174,20 @@ def matches(self, warning): return True + def __repr__(self): + extras = "" + if self.category: + extras += "category: {} ".format(self.category) + if self.message: + extras += "message: {} ".format(self.message) + if self.lineno: + extras += "lineno: {} ".format(self.lineno) + if self.filename: + extras += "filename: {} ".format(self.filename) + if not extras: + extras = "-" + return "".format(extras.strip()) + _ignored_warnings = [] From ab2312b587d9b43900c0119904dafd371e8bb34a Mon Sep 17 00:00:00 2001 From: Maor Marcus Date: Tue, 12 Mar 2019 11:14:42 +0200 Subject: [PATCH 04/17] Refactor parallel to use worker_wrapper --- slash/frontend/slash_run.py | 5 +- slash/parallel/parallel_manager.py | 96 +++++--------------- slash/parallel/server.py | 4 +- slash/parallel/worker_configuration.py | 119 ++++++++++++++++++++++++- tests/test_parallel.py | 3 +- 5 files changed, 143 insertions(+), 84 deletions(-) diff --git a/slash/frontend/slash_run.py b/slash/frontend/slash_run.py index 12158e34..3c4c4720 100644 --- a/slash/frontend/slash_run.py +++ b/slash/frontend/slash_run.py @@ -41,7 +41,9 @@ def slash_run(args, report_stream=None, resume=False, rerun=False, app_callback= if config.root.tmux.enabled and not is_child(): _logger.notice("About to start slash in new tmux session...") run_slash_in_tmux(args) - if is_child(): + if is_parent(): + app.session.parallel_manager = ParallelManager(args) + elif is_child(): worker = Worker(config.root.parallel.worker_id, app.session.id) worker.connect_to_server() if resume or rerun: @@ -62,7 +64,6 @@ def slash_run(args, report_stream=None, resume=False, rerun=False, app_callback= with app.session.get_started_context(): report_tests_to_backslash(collected) if is_parent(): - app.session.parallel_manager = ParallelManager(args) app.session.parallel_manager.start_server_in_thread(collected) app.session.parallel_manager.start() else: diff --git a/slash/parallel/parallel_manager.py b/slash/parallel/parallel_manager.py index 1b4e4966..8b6a2c61 100644 --- a/slash/parallel/parallel_manager.py +++ b/slash/parallel/parallel_manager.py @@ -1,8 +1,4 @@ -import sys -import errno import os -import signal -import subprocess import time import logbook import threading @@ -11,11 +7,8 @@ from .. import log from ..exceptions import INTERRUPTION_EXCEPTIONS, ParallelServerIsDown, ParallelTimeout from ..conf import config -from ..ctx import context from .server import Server, ServerStates, KeepaliveServer -from ..utils.tmux_utils import create_new_window, create_new_pane -from .worker_configuration import WorkerConfiguration -from .. import hooks +from .worker_configuration import TmuxWorkerConfiguration, ProcessWorkerConfiguration _logger = logbook.Logger(__name__) log.set_log_color(_logger.name, logbook.NOTICE, 'blue') @@ -26,29 +19,26 @@ def get_xmlrpc_proxy(address, port): return xmlrpc_client.ServerProxy('http://{}:{}'.format(address, port)) -def is_process_running(pid): - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - return False - else: - raise - return True class ParallelManager(object): def __init__(self, args): super(ParallelManager, self).__init__() self.server = None self.workers_error_dircetory = mkdtemp() - self.args = [sys.executable, '-m', 'slash.frontend.main', 'run', '--parallel-parent-session-id', context.session.id, \ - '--workers-error-dir', self.workers_error_dircetory] + args + self.args = args self.workers_num = config.root.parallel.num_workers self.workers = {} - self.max_worker_id = 1 self.server_thread = None self.keepalive_server = None self.keepalive_server_thread = None + self._create_workers() + + def _create_workers(self): + for index in range(1, self.workers_num+1): + _logger.debug("Creating worker number {}", index) + index_str = str(index) + worker_cls = TmuxWorkerConfiguration if config.root.tmux.enabled else ProcessWorkerConfiguration + self.workers[index_str] = worker_cls(self.args, index_str) def try_connect(self): for _ in range(MAX_CONNECTION_RETRIES): @@ -57,48 +47,17 @@ def try_connect(self): time.sleep(0.1) raise ParallelServerIsDown("Cannot connect to XML_RPC server") - def start_worker(self): - worker_id = str(self.max_worker_id) - _logger.notice("Starting worker number {}", worker_id) - new_args = self.args[:] + ["--parallel-worker-id", worker_id] - worker_config = WorkerConfiguration(new_args) - hooks.before_worker_start(worker_config=worker_config) # pylint: disable=no-member - if config.root.tmux.enabled: - worker_config.argv.append(';$SHELL') - command = ' '.join(worker_config.argv) - if config.root.tmux.use_panes: - self.workers[worker_id] = create_new_pane(command) - else: - self.workers[worker_id] = create_new_window("worker {}".format(worker_id), command) - else: - with open(os.devnull, 'w') as devnull: - proc = subprocess.Popen(worker_config.argv, stdin=devnull, stdout=devnull, stderr=devnull) - self.workers[worker_id] = proc - _logger.trace("Worker #{} command: {}", worker_id, ' '.join(worker_config.argv)) - self.max_worker_id += 1 - - def start_server_in_thread(self, collected): self.server = Server(collected) - self.server_thread = threading.Thread(target=self.server.serve, args=()) - self.server_thread.setDaemon(True) + self.server_thread = threading.Thread(target=self.server.serve, daemon=True) self.server_thread.start() self.keepalive_server = KeepaliveServer() - self.keepalive_server_thread = threading.Thread(target=self.keepalive_server.serve, args=()) - self.keepalive_server_thread.setDaemon(True) + self.keepalive_server_thread = threading.Thread(target=self.keepalive_server.serve, daemon=True) self.keepalive_server_thread.start() def kill_workers(self): - if config.root.tmux.enabled: - for worker_pid in self.server.worker_pids: - try: - os.kill(worker_pid, signal.SIGTERM) - except OSError as err: - if err.errno != errno.ESRCH: - raise - else: - for worker in self.workers.values(): - worker.send_signal(signal.SIGTERM) + for worker in self.workers.values(): + worker.kill() def report_worker_error_logs(self): found_worker_errors_file = False @@ -135,11 +94,7 @@ def check_worker_timed_out(self): if time.time() - worker_last_connection_time > config.root.parallel.communication_timeout_secs: _logger.error("Worker {} is down, terminating session", worker_id, extra={'capture': False}) self.report_worker_error_logs() - if not config.root.tmux.enabled: - if self.workers[worker_id].poll() is None: - self.workers[worker_id].kill() - elif not config.root.tmux.use_panes: - self.workers[worker_id].rename_window('stopped_client_{}'.format(worker_id)) + self.workers[worker_id].handle_timeout() get_xmlrpc_proxy(config.root.parallel.server_addr, self.server.port).report_client_failure(worker_id) def check_no_requests_timeout(self): @@ -159,13 +114,9 @@ def check_no_requests_timeout(self): def start(self): self.try_connect() - if not config.root.parallel.server_port: - self.args.extend(['--parallel-port', str(self.server.port)]) - if not config.root.parallel.keepalive_port: - self.args.extend(['--keepalive-port', str(self.keepalive_server.port)]) try: - for _ in range(self.workers_num): - self.start_worker() + for worker_config in self.workers.values(): + worker_config.start() self.wait_all_workers_to_connect() while self.server.should_wait_for_request(): self.check_worker_timed_out() @@ -177,16 +128,9 @@ def start(self): self.kill_workers() raise finally: - if not config.root.tmux.enabled: - for worker in self.workers.values(): - worker.wait() - else: - for worker_pid in self.server.worker_pids: - for _ in range(10): - if not is_process_running(worker_pid): - break - else: - time.sleep(0.5) + for worker in self.workers.values(): + worker.wait_to_finish() + get_xmlrpc_proxy(config.root.parallel.server_addr, self.server.port).stop_serve() get_xmlrpc_proxy(config.root.parallel.server_addr, self.keepalive_server.port).stop_serve() self.server_thread.join() diff --git a/slash/parallel/server.py b/slash/parallel/server.py index 8687131f..a6dd5f9a 100644 --- a/slash/parallel/server.py +++ b/slash/parallel/server.py @@ -76,7 +76,7 @@ def __init__(self, tests): self.unstarted_tests = queue.Queue() self.num_collections_validated = 0 self.start_time = time.time() - self.worker_pids = [] + self.worker_to_pid = {} for i in range(len(tests)): self.unstarted_tests.put(i) self.connected_clients = set() @@ -125,7 +125,7 @@ def connect(self, client_id, client_pid): context.session.logging.create_worker_symlink(self._get_worker_session_id(client_id), client_session_id) hooks.worker_connected(session_id=client_session_id) # pylint: disable=no-member self.worker_session_ids.append(client_session_id) - self.worker_pids.append(client_pid) + self.worker_to_pid[client_id] = client_pid self.executing_tests[client_id] = None if len(self.connected_clients) >= config.root.parallel.num_workers: _logger.notice("All workers connected to server") diff --git a/slash/parallel/worker_configuration.py b/slash/parallel/worker_configuration.py index 933193ed..e934c2f9 100644 --- a/slash/parallel/worker_configuration.py +++ b/slash/parallel/worker_configuration.py @@ -1,3 +1,118 @@ +from ..conf import config +from .. import hooks +import os +import logbook +from ..utils.tmux_utils import create_new_window, create_new_pane +from slash import ctx +import subprocess +import time +import signal +import errno +import sys + +_logger = logbook.Logger(__name__) + +def is_process_running(pid): + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: + return False + else: + raise + return True + class WorkerConfiguration(object): - def __init__(self, argv): - self.argv = argv + def __init__(self, args, worker_id): + self._worker_id = worker_id + self.argv = [sys.executable, '-m', 'slash.frontend.main', 'run', '--parallel-parent-session-id', ctx.session.id] + \ + args + ["--parallel-worker-id", str(worker_id)] + self._excluded_tests = set() + self._pid = None + + def _start(self): + raise NotImplementedError() + + def start(self): + hooks.before_worker_start(worker_config=self) # pylint: disable=no-member + _logger.notice("Starting worker number {}", self._worker_id) + self.argv += ['--parallel-port', str(ctx.session.parallel_manager.server.port),\ + '--keepalive-port', str(ctx.session.parallel_manager.keepalive_server.port), \ + '--workers-error-dir', ctx.session.parallel_manager.workers_error_dircetory] + self._start() + + def kill(self): + raise NotImplementedError() + + def handle_timeout(self): + raise NotImplementedError() + + def exclude_test(self, test_index): + self._excluded_tests.add(test_index) + + def get_pid(self): + return self._pid + + def is_active(self): + raise NotImplementedError() + + def wait_to_finish(self): + raise NotImplementedError() + +class ProcessWorkerConfiguration(WorkerConfiguration): + def _start(self): + with open(os.devnull, 'w') as devnull: + self._pid = subprocess.Popen(self.argv, stdin=devnull, stdout=devnull, stderr=devnull) + + def kill(self): + if self._pid is not None: + self._pid.send_signal(signal.SIGTERM) + + def handle_timeout(self): + if self.is_active(): + self._pid.kill() + + def is_active(self): + return self._pid.poll() is None + + def wait_to_finish(self): + self._pid.wait() + +class TmuxWorkerConfiguration(WorkerConfiguration): + def __init__(self, basic_args, worker_id): + super(TmuxWorkerConfiguration, self).__init__(basic_args, worker_id) + self.argv += [';$SHELL'] + self._window_handle = None + + def _start(self): + command = ' '.join(self.argv) + if config.root.tmux.use_panes: + self._window_handle = create_new_pane(command) + else: + self._window_handle = create_new_window("worker {}".format(self._worker_id), command) + + def get_pid(self): + if self._pid is None: + self._pid = ctx.session.parallel_manager.server.worker_to_pid.get(self._worker_id) + return self._pid + + def handle_timeout(self): + if not config.root.tmux.use_panes: + self._window_handle.rename_window('stopped_client_{}'.format(self._worker_id)) + + def is_active(self): + return is_process_running(self.get_pid()) + + def kill(self): + try: + os.kill(self.get_pid(), signal.SIGTERM) + except OSError as err: + if err.errno != errno.ESRCH: + raise + + def wait_to_finish(self): + for _ in range(10): + if not self.is_active(): + break + else: + time.sleep(0.5) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 6242f7f8..01acd960 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -91,8 +91,7 @@ def simulate_ctrl_c(session_id): # pylint: disable=unused-variable, unused-argu @slash.hooks.session_interrupt.register # pylint: disable=no-member def check_workers_and_server_down(): # pylint: disable=unused-variable for worker in slash.context.session.parallel_manager.workers.values(): - ret = worker.poll() - assert not ret is None + assert not worker.is_active() assert slash.context.session.parallel_manager.server.interrupted assert not slash.context.session.parallel_manager.server.finished_tests From b010ba07e089262981f70c4978c6c3c4d2e9c428 Mon Sep 17 00:00:00 2001 From: Maor Marcus Date: Wed, 13 Mar 2019 11:42:07 +0200 Subject: [PATCH 05/17] Add tests_distributer to allow forcing and excluding tests for specific worker --- slash/core/metadata.py | 1 + slash/loader.py | 2 + slash/parallel/parallel_manager.py | 16 ++--- slash/parallel/server.py | 28 +++++---- slash/parallel/tests_distributer.py | 46 ++++++++++++++ slash/parallel/worker.py | 2 +- slash/parallel/worker_configuration.py | 42 ++++++++++++- tests/test_parallel.py | 84 +++++++++++++++++++++++++- 8 files changed, 195 insertions(+), 26 deletions(-) create mode 100644 slash/parallel/tests_distributer.py diff --git a/slash/core/metadata.py b/slash/core/metadata.py index 805530e6..fa53170d 100644 --- a/slash/core/metadata.py +++ b/slash/core/metadata.py @@ -22,6 +22,7 @@ def __init__(self, factory, test): self.tags = test.get_tags() self._sort_key = next(_sort_key_generator) self.repeat_all_index = 0 + self.parallel_index = None if factory is not None: #: The path to the file from which this test was loaded self.module_name = factory.get_module_name() diff --git a/slash/loader.py b/slash/loader.py index 9d6fb996..27fb5ac8 100644 --- a/slash/loader.py +++ b/slash/loader.py @@ -64,6 +64,8 @@ def get_runnables(self, paths, prepend_interactive=False): if prepend_interactive: returned.insert(0, generate_interactive_test()) + for index, test in enumerate(returned): + test.__slash__.parallel_index = index hooks.tests_loaded(tests=returned) # pylint: disable=no-member returned.sort(key=lambda test: ( test.__slash__.repeat_all_index, test.__slash__.get_sort_key() diff --git a/slash/parallel/parallel_manager.py b/slash/parallel/parallel_manager.py index 8b6a2c61..ca33282a 100644 --- a/slash/parallel/parallel_manager.py +++ b/slash/parallel/parallel_manager.py @@ -49,14 +49,16 @@ def try_connect(self): def start_server_in_thread(self, collected): self.server = Server(collected) - self.server_thread = threading.Thread(target=self.server.serve, daemon=True) + self.server_thread = threading.Thread(target=self.server.serve, args=()) + self.server_thread.setDaemon(True) self.server_thread.start() self.keepalive_server = KeepaliveServer() - self.keepalive_server_thread = threading.Thread(target=self.keepalive_server.serve, daemon=True) + self.keepalive_server_thread = threading.Thread(target=self.keepalive_server.serve, args=()) + self.keepalive_server_thread.setDaemon(True) self.keepalive_server_thread.start() def kill_workers(self): - for worker in self.workers.values(): + for worker in list(self.workers.values()): worker.kill() def report_worker_error_logs(self): @@ -105,7 +107,7 @@ def check_no_requests_timeout(self): _logger.error("Clients that are still connected to server: {}", self.server.connected_clients, extra={'capture': False}) if self.server.has_more_tests(): - _logger.error("Number of unstarted tests: {}", self.server.unstarted_tests.qsize(), + _logger.error("Number of unstarted tests: {}", len(self.server.get_unstarted_tests()), extra={'capture': False}) if self.server.executing_tests: _logger.error("Currently executed tests indexes: {}", self.server.executing_tests.values(), @@ -115,8 +117,8 @@ def check_no_requests_timeout(self): def start(self): self.try_connect() try: - for worker_config in self.workers.values(): - worker_config.start() + for worker in list(self.workers.values()): + worker.start() self.wait_all_workers_to_connect() while self.server.should_wait_for_request(): self.check_worker_timed_out() @@ -128,7 +130,7 @@ def start(self): self.kill_workers() raise finally: - for worker in self.workers.values(): + for worker in list(self.workers.values()): worker.wait_to_finish() get_xmlrpc_proxy(config.root.parallel.server_addr, self.server.port).stop_serve() diff --git a/slash/parallel/server.py b/slash/parallel/server.py index a6dd5f9a..7a1095ac 100644 --- a/slash/parallel/server.py +++ b/slash/parallel/server.py @@ -3,7 +3,6 @@ import time import threading from enum import Enum -from six.moves import queue from six.moves import xmlrpc_server from ..utils.python import unpickle from .. import log @@ -11,6 +10,7 @@ from ..runner import _get_test_context from .. import hooks from ..conf import config +from .tests_distributer import TestsDistributer _logger = logbook.Logger(__name__) log.set_log_color(_logger.name, logbook.NOTICE, 'blue') @@ -73,21 +73,16 @@ def __init__(self, tests): self.worker_session_ids = [] self.executing_tests = {} self.finished_tests = [] - self.unstarted_tests = queue.Queue() self.num_collections_validated = 0 self.start_time = time.time() self.worker_to_pid = {} - for i in range(len(tests)): - self.unstarted_tests.put(i) self.connected_clients = set() self.collection = [[test.__slash__.file_path, test.__slash__.function_name, test.__slash__.variation.dump_variation_dict()] for test in self.tests] self._sorted_collection = sorted(self.collection) - - def _has_unstarted_tests(self): - return not self.unstarted_tests.empty() + self._tests_distrubuter = TestsDistributer(len(self._sorted_collection)) def has_connected_clients(self): return len(self.connected_clients) > 0 @@ -108,12 +103,16 @@ def report_client_failure(self, client_id): self._mark_unrun_tests() self.worker_error_reported = True + def get_unstarted_tests(self): + return self._tests_distrubuter.get_unstarted_tests() + def _mark_unrun_tests(self): - while self._has_unstarted_tests(): - test_index = self.unstarted_tests.get() + unstarted_tests_indexes = self._tests_distrubuter.get_unstarted_tests() + for test_index in unstarted_tests_indexes: with _get_test_context(self.tests[test_index], logging=False): pass self.finished_tests.append(test_index) + self._tests_distrubuter.clear_unstarted_tests() def _get_worker_session_id(self, client_id): return "worker_{}".format(client_id) @@ -142,10 +141,11 @@ def validate_collection(self, client_id, sorted_client_collection): self.state = ServerStates.SERVE_TESTS return True - def disconnect(self, client_id): + def disconnect(self, client_id, has_failure=False): _logger.notice("Client {} sent disconnect", client_id) self.connected_clients.remove(client_id) - self.state = ServerStates.STOP_TESTS_SERVING + if has_failure: + self.state = ServerStates.STOP_TESTS_SERVING def get_test(self, client_id): if not self.executing_tests[client_id] is None: @@ -156,8 +156,10 @@ def get_test(self, client_id): return NO_MORE_TESTS elif self.state in [ServerStates.WAIT_FOR_CLIENTS, ServerStates.WAIT_FOR_COLLECTION_VALIDATION]: return WAITING_FOR_CLIENTS - elif self.state == ServerStates.SERVE_TESTS and self._has_unstarted_tests(): - test_index = self.unstarted_tests.get() + elif self.state == ServerStates.SERVE_TESTS and self._tests_distrubuter.has_unstarted_tests(): + test_index = self._tests_distrubuter.get_next_test_for_client(client_id) + if test_index is None: #we have omre tests but current worker cannot execute them + return NO_MORE_TESTS test = self.tests[test_index] self.executing_tests[client_id] = test_index hooks.test_distributed(test_logical_id=test.__slash__.id, worker_session_id=self._get_worker_session_id(client_id)) # pylint: disable=no-member diff --git a/slash/parallel/tests_distributer.py b/slash/parallel/tests_distributer.py new file mode 100644 index 00000000..de450761 --- /dev/null +++ b/slash/parallel/tests_distributer.py @@ -0,0 +1,46 @@ +from slash import ctx +from logbook import Logger + + +_logger = Logger(__name__) + +class TestsDistributer(object): + def __init__(self, num_tests): + self._unstarted_tests_indices = [i for i in range(num_tests)] + self._current_tests_index = 0 + workers_config = ctx.session.parallel_manager.workers.values() + self._workers_excluded_tests = {config.get_worker_id(): config.get_excluded_tests() \ + for config in workers_config} + self._forced_tests_dict = {} + for config in workers_config: + for test_index in config.get_forced_tests(): + self._forced_tests_dict[test_index] = config.get_worker_id() + _logger.debug("forced_tests_dict: {}", self._forced_tests_dict) + + def _can_execute_test(self, test_index, client_id): + _logger.debug("_can_execute_test - client_id: {}, test_index: {}", client_id, test_index) + forced_worker = self._forced_tests_dict.get(test_index) + if forced_worker is not None: + return forced_worker == client_id + return test_index not in self._workers_excluded_tests[client_id] + + def get_next_test_for_client(self, client_id): + ret = None + for test_index in self._unstarted_tests_indices: + if self._can_execute_test(test_index, client_id): + ret = test_index + break + else: + _logger.debug('worker id {} cannot execute test number {}, searching another', client_id, test_index) + if ret is not None: + self._unstarted_tests_indices.remove(ret) + return ret + + def clear_unstarted_tests(self): + self._unstarted_tests_indices = [] + + def get_unstarted_tests(self): + return list(self._unstarted_tests_indices) + + def has_unstarted_tests(self): + return len(self._unstarted_tests_indices) > 0 diff --git a/slash/parallel/worker.py b/slash/parallel/worker.py index 974f2b97..aa70499d 100644 --- a/slash/parallel/worker.py +++ b/slash/parallel/worker.py @@ -87,7 +87,7 @@ def start_execution(self, app, collected_tests): _logger.error("Collections of worker id {} and master don't match, worker terminates", self.client_id, extra={'capture': False}) self._stop_keepalive_thread() - self.client.disconnect(self.client_id) + self.client.disconnect(self.client_id, has_failure=True) return should_stop = False diff --git a/slash/parallel/worker_configuration.py b/slash/parallel/worker_configuration.py index e934c2f9..f651c15c 100644 --- a/slash/parallel/worker_configuration.py +++ b/slash/parallel/worker_configuration.py @@ -28,8 +28,18 @@ def __init__(self, args, worker_id): self.argv = [sys.executable, '-m', 'slash.frontend.main', 'run', '--parallel-parent-session-id', ctx.session.id] + \ args + ["--parallel-worker-id", str(worker_id)] self._excluded_tests = set() + self._forced_tests = set() self._pid = None + def get_forced_tests(self): + return self._forced_tests + + def get_excluded_tests(self): + return self._excluded_tests + + def get_worker_id(self): + return self._worker_id + def _start(self): raise NotImplementedError() @@ -47,8 +57,34 @@ def kill(self): def handle_timeout(self): raise NotImplementedError() - def exclude_test(self, test_index): - self._excluded_tests.add(test_index) + def _test_or_index_to_index(self, test_or_test_index): + test_index = test_or_test_index.__slash__.parallel_index if hasattr(test_or_test_index, '__slash__') else test_or_test_index + assert isinstance(test_index, int) + return test_index + + def exclude_test(self, test_or_test_index): + """Prevents from the test/test_index to be executed on the specific worker + """ + test_index = self._test_or_index_to_index(test_or_test_index) + if test_index in self._forced_tests: + raise RuntimeError('Cannot exclude test_index {} for client {} since it has been forced before'.format(test_index, + self._worker_id)) + if test_index in self._excluded_tests: + _logger.warning('test_index {} already in exclude list for worker {}, not adding it', test_index, self._worker_id) + else: + self._excluded_tests.add(test_index) + + def force_test(self, test_or_test_index): + """Forces the test/test_index to be executed on the specific worker + """ + test_index = self._test_or_index_to_index(test_or_test_index) + if test_index in self._excluded_tests: + raise RuntimeError('Cannot force test_index {} for client {} since it has been excluded before'.format(test_index, + self._worker_id)) + if test_index in self._forced_tests: + _logger.warning('test_index {} already in forced list for worker {}, not adding it', test_index, self._worker_id) + else: + self._forced_tests.add(test_index) def get_pid(self): return self._pid @@ -81,10 +117,10 @@ def wait_to_finish(self): class TmuxWorkerConfiguration(WorkerConfiguration): def __init__(self, basic_args, worker_id): super(TmuxWorkerConfiguration, self).__init__(basic_args, worker_id) - self.argv += [';$SHELL'] self._window_handle = None def _start(self): + self.argv += [';$SHELL'] command = ' '.join(self.argv) if config.root.tmux.use_panes: self._window_handle = create_new_pane(command) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 01acd960..a872b4ae 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -334,9 +334,10 @@ def test_children_session_ids(parallel_suite): def test_timeout_no_request_to_server(config_override, runnable_test_dir): config_override("parallel.no_request_timeout", 1) - with Session(): + with Session() as session: runnables = Loader().get_runnables(str(runnable_test_dir)) parallel_manager = ParallelManager([]) + session.parallel_manager = parallel_manager parallel_manager.start_server_in_thread(runnables) parallel_manager.try_connect() parallel_manager.server.state = ServerStates.SERVE_TESTS @@ -348,9 +349,10 @@ def test_timeout_no_request_to_server(config_override, runnable_test_dir): def test_children_not_connected_timeout(runnable_test_dir, config_override): config_override("parallel.worker_connect_timeout", 0) config_override("parallel.num_workers", 1) - with Session(): + with Session() as session: runnables = Loader().get_runnables(str(runnable_test_dir)) parallel_manager = ParallelManager([]) + session.parallel_manager = parallel_manager parallel_manager.start_server_in_thread(runnables) time.sleep(0.1) with slash.assert_raises(ParallelTimeout) as caught: @@ -390,3 +392,81 @@ def test_distributed(test_logical_id, worker_session_id): # pylint: disable=un suite = Suite(debug_info=False, is_parallel=True) suite.populate(num_tests=2) suite.run(num_workers=1) + +def test_force_worker(parallel_suite): + for test in parallel_suite: + test.append_line("from slash import config") + test.append_line("slash.context.result.data.setdefault('worker_id', config.root.parallel.worker_id)") + + @slash.hooks.tests_loaded.register # pylint: disable=no-member, unused-argument + def tests_loaded(tests): # pylint: disable=unused-variable, unused-argument + if slash.utils.parallel_utils.is_parent_session(): + from slash import ctx + workers = ctx.session.parallel_manager.workers + for index in range(len(tests)): + worker_id = '1' if index%2 == 0 else '2' + workers[worker_id].force_test(index) + + summary = parallel_suite.run(num_workers=2) + assert summary.session.results.is_success() + for index, test in enumerate(parallel_suite): + [result] = summary.get_all_results_for_test(test) + assert result.data['worker_id'] if index%2 == 0 else '2' + +def test_force_on_one_worker(parallel_suite): + @slash.hooks.tests_loaded.register # pylint: disable=no-member, unused-argument + def tests_loaded(tests): # pylint: disable=unused-variable, unused-argument + if slash.utils.parallel_utils.is_parent_session(): + from slash import ctx + worker = list(ctx.session.parallel_manager.workers.values())[0] + for index in range(len(tests)): + worker.force_test(index) + + summary = parallel_suite.run(num_workers=2) + assert summary.session.results.is_success() + +@pytest.mark.parametrize('num_workers', [1, 2]) +def test_exclude_on_one_worker(parallel_suite, config_override, num_workers): + config_override("parallel.no_request_timeout", 2) + + @slash.hooks.tests_loaded.register # pylint: disable=no-member, unused-argument + def tests_loaded(tests): # pylint: disable=unused-variable, unused-argument + if slash.utils.parallel_utils.is_parent_session(): + from slash import ctx + worker = list(ctx.session.parallel_manager.workers.values())[0] + for index in range(len(tests)): + worker.exclude_test(index) + if num_workers == 1: + summary = parallel_suite.run(num_workers=num_workers, verify=False) + assert summary.session.results.get_num_started() == 0 + else: + summary = parallel_suite.run(num_workers=num_workers) + assert summary.session.results.is_success() + +@pytest.mark.parametrize('use_test_index', [True, False]) +def test_exclude_test_on_all_workers_causes_timeout(parallel_suite, config_override, use_test_index): + config_override("parallel.no_request_timeout", 2) + + @slash.hooks.tests_loaded.register # pylint: disable=no-member, unused-argument + def tests_loaded(tests): # pylint: disable=unused-variable, unused-argument + if slash.utils.parallel_utils.is_parent_session(): + from slash import ctx + for worker in ctx.session.parallel_manager.workers.values(): + if use_test_index: + worker.exclude_test(0) + else: + worker.exclude_test(tests[0]) + summary = parallel_suite.run(num_workers=2, verify=False) + assert summary.session.results.get_num_started() == len(parallel_suite) - 1 + assert not summary.get_all_results_for_test(parallel_suite[0]) + +def test_exclude_and_force_on_same_worker_raises_runtime_err(parallel_suite): + @slash.hooks.tests_loaded.register # pylint: disable=no-member, unused-argument + def tests_loaded(tests): # pylint: disable=unused-variable, unused-argument + if slash.utils.parallel_utils.is_parent_session(): + from slash import ctx + worker = list(ctx.session.parallel_manager.workers.values())[0] + worker.exclude_test(0) + worker.force_test(0) + summary = parallel_suite.run(num_workers=2, verify=False) + assert "Cannot force test_index 0" in summary.session.results.global_result.get_errors()[0].exception_str From a699ea65ea14d98d6cb5646bd29a0e6423cce2df Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Sun, 24 Mar 2019 13:03:41 +0200 Subject: [PATCH 06/17] Fix bors toml --- bors.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bors.toml b/bors.toml index 5279fe72..1658139d 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,5 @@ +timeout-sec = 14400 +delete_merged_branches = true status = [ "continuous-integration/travis-ci/push" ] From aafce9d8c59c02cdec79af948b52e828f3e91675 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Tue, 26 Mar 2019 12:52:17 +0200 Subject: [PATCH 07/17] Improve skip reason for excluded tests --- slash/runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/slash/runner.py b/slash/runner.py index bbb186b4..23fe02ec 100644 --- a/slash/runner.py +++ b/slash/runner.py @@ -150,8 +150,9 @@ def _process_requirements_and_exclusions(test): def _process_exclusions(test): if is_excluded(test): - context.result.add_skip('Excluded') - hooks.test_avoided(reason='Excluded') # pylint: disable=no-member + reason = 'Excluded due to parameter combination exclusion rules' + context.result.add_skip(reason) + hooks.test_avoided(reason=reason) # pylint: disable=no-member return False return True From d9ec32dcb0909c99adbcf92dc535e60abc4593e8 Mon Sep 17 00:00:00 2001 From: Ronen Hoffer Date: Tue, 23 Apr 2019 16:45:15 +0300 Subject: [PATCH 08/17] Add test execution time to xunit report --- doc/changelog.rst | 2 +- slash/core/result.py | 15 ++++++++++++++- slash/plugins/builtin/xunit.py | 7 +++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 7f8d62df..8f3f2d33 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,6 @@ Changelog ========= - +* :feature:`-` Add tests and suite execution time to xunit plugin * :feature:`-` Add ``slash.ignored_warnings`` context * :bug:`930` Restore behavior of exceptions propagating out of the test_start or test_end hooks. Correct behavior is for those to fail the test (thanks @pierreluctg) * :bug:`934` Parallel sessions now honor fatal exceptions encountered in worker sessions diff --git a/slash/core/result.py b/slash/core/result.py index 2ffaa47a..d58b2ac2 100644 --- a/slash/core/result.py +++ b/slash/core/result.py @@ -4,9 +4,9 @@ import pickle import sys from numbers import Number - import gossip import logbook +from datetime import datetime, timedelta from vintage import deprecated from .. import hooks @@ -35,6 +35,8 @@ def __init__(self, test_metadata=None): self.test_metadata = test_metadata #: dictionary to be use by tests and plugins to store result-related information for later analysis self.data = {} + self._start_time = None + self._end_time = None self._errors = [] self._failures = [] self._skips = [] @@ -155,6 +157,7 @@ def is_not_run(self): return not self.is_started() and not self.has_errors_or_failures() def mark_started(self): + self._start_time = datetime.now() self._started = True def is_error(self): @@ -191,6 +194,7 @@ def is_finished(self): return self._finished def mark_finished(self): + self._end_time = datetime.now() self._finished = True def mark_interrupted(self): @@ -246,6 +250,15 @@ def add_skip(self, reason, append=True): self._skips.append(reason) context.reporter.report_test_skip_added(context.test, reason) + def get_duration(self): + """Returns the test duration time as timedelta object + + :return: timedelta + """ + if self._end_time is None or self._start_time is None: + return timedelta() + return self._end_time - self._start_time + def get_errors(self): """Returns the list of errors recorded for this result diff --git a/slash/plugins/builtin/xunit.py b/slash/plugins/builtin/xunit.py index 8e9dae31..bd538445 100644 --- a/slash/plugins/builtin/xunit.py +++ b/slash/plugins/builtin/xunit.py @@ -75,11 +75,14 @@ def session_end(self): if config.root.parallel.worker_id is not None: return + suite_time = sum([result.get_duration() for result in context.session.results.iter_test_results()], + datetime.timedelta()).total_seconds() + e = E('testsuite', { "name": "slash-suite", "hostname": socket.getfqdn(), "timestamp": self._start_time.isoformat().rsplit(".", 1)[0], - "time": "0", + "time": str(suite_time), "tests": str(context.session.results.get_num_results()), "errors": str(context.session.results.get_num_errors()), "failures": str(context.session.results.get_num_failures()), @@ -90,7 +93,7 @@ def session_end(self): test = E("testcase", { "name": result.test_metadata.address, "classname": result.test_metadata.class_name or '', - "time": "0" + "time": str(result.get_duration().total_seconds()) }) self._add_errors(test, result) From d2cda2cde34537ed35a8c0ff3937c5a91bcd3dd3 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Tue, 28 May 2019 23:19:11 +0300 Subject: [PATCH 09/17] Added log for handling fatal error (closes #950) --- doc/changelog.rst | 1 + slash/exception_handling.py | 5 ++++- tests/test_exception_handling.py | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 71ce0a5c..19192c14 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :feature:`950` Added log for handling fatal error * :feature:`-` Add tests and suite execution time to xunit plugin * :feature:`-` Add ``slash.ignored_warnings`` context * :release:`1.7.10 <30-04-2019>` diff --git a/slash/exception_handling.py b/slash/exception_handling.py index 5db23599..5754d3a6 100644 --- a/slash/exception_handling.py +++ b/slash/exception_handling.py @@ -149,12 +149,15 @@ def handle_exception(exc_info, context=None): It also adds the exception to its correct place in the current result, be it a failure, an error or a skip """ - already_handled = is_exception_handled(exc_info[1]) + exc_value = exc_info[1] + already_handled = is_exception_handled(exc_value) msg = "Handling exception" if context is not None: msg += " (Context: {0})" if already_handled: msg += " (already handled)" + if is_exception_fatal(exc_value): + msg += " FATAL" _logger.debug(msg, context, exc_info=exc_info if not already_handled else None) if not already_handled: diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py index 76be16a6..5bed10b8 100644 --- a/tests/test_exception_handling.py +++ b/tests/test_exception_handling.py @@ -13,6 +13,32 @@ from .utils import CustomException, TestCase +@pytest.mark.parametrize('is_fatal', [True, False]) +@pytest.mark.parametrize('context', [None, 'Something']) +def test_handling_exceptions_log(context, is_fatal): + raised = CustomException() + if is_fatal: + exception_handling.mark_exception_fatal(raised) + with slash.Session(): + with logbook.TestHandler() as handler: + with exception_handling.handling_exceptions(context=context, swallow=True): + raise raised + assert len(handler.records) == 3 + assert handler.records[1].message.startswith('Error added') + assert handler.records[2].message.startswith('Swallowing') + handle_exc_msg = handler.records[0].message + assert handle_exc_msg.startswith('Handling exception') + + if context: + assert 'Context: {}'.format(context) in handle_exc_msg + else: + assert 'Context' not in handle_exc_msg + if is_fatal: + assert 'FATAL' in handle_exc_msg + else: + assert 'FATAL' not in handle_exc_msg + + def test_handling_exceptions_swallow_skip_test(suite, suite_test): @suite_test.append_body From 95a40b00edb402cfd08d6034b92ccc0ab1c39351 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Fri, 31 May 2019 10:17:09 +0300 Subject: [PATCH 10/17] Small changelog fix --- doc/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 19192c14..9a621410 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,6 @@ Changelog ========= -* :feature:`950` Added log for handling fatal error +* :feature:`950` Slash now emits a log record when handling fatal errors * :feature:`-` Add tests and suite execution time to xunit plugin * :feature:`-` Add ``slash.ignored_warnings`` context * :release:`1.7.10 <30-04-2019>` From 8f116ecf926cc4f9b04b97173f0b84f7a1b53830 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Sun, 2 Jun 2019 10:28:47 +0300 Subject: [PATCH 11/17] Support light background mode (closes #925) --- doc/changelog.rst | 1 + slash/conf.py | 3 +++ slash/log.py | 5 ++++- slash/reporting/console_reporter.py | 9 ++++++--- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 9a621410..18aa9e02 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :feature:`925` Support was added for terminals with light backgrounds by changing ``log.console_theme.dark_background`` configuration * :feature:`950` Slash now emits a log record when handling fatal errors * :feature:`-` Add tests and suite execution time to xunit plugin * :feature:`-` Add ``slash.ignored_warnings`` context diff --git a/slash/conf.py b/slash/conf.py index 93099207..a11fd2ab 100644 --- a/slash/conf.py +++ b/slash/conf.py @@ -18,6 +18,7 @@ "log": { "colorize": False // Doc("Emit log colors to files"), "console_theme": { + 'dark_background': True, 'inline-file-end-fail': 'red', 'inline-file-end-skip': 'yellow', 'inline-file-end-success': 'green', @@ -26,8 +27,10 @@ 'error-cause-marker': 'white/bold', 'fancy-message': 'yellow/bold', 'frame-local-varname': 'yellow/bold', + 'num-collected': 'white/bold', 'session-summary-success': 'green/bold', 'session-summary-failure': 'red/bold', + 'session-start': 'white/bold', 'error-separator-dash': 'red', 'tb-error-message': 'red/bold', 'tb-error': 'red/bold', diff --git a/slash/log.py b/slash/log.py index 8267be89..cdfc908d 100644 --- a/slash/log.py +++ b/slash/log.py @@ -67,7 +67,10 @@ def get_color(self, record): elif record.level >= logbook.WARNING: return 'yellow' elif record.level >= logbook.NOTICE: - return 'white' + if config.root.log.console_theme.dark_background: + return 'white' + else: + return 'black' return None # default class ColorizedFileHandler(ColorizedHandlerMixin, logbook.FileHandler): diff --git a/slash/reporting/console_reporter.py b/slash/reporting/console_reporter.py index 2d0be039..f7e6befe 100644 --- a/slash/reporting/console_reporter.py +++ b/slash/reporting/console_reporter.py @@ -23,7 +23,10 @@ def theme(name): - return dict((x, True) for x in config['log']['console_theme'][name].split('/')) + returned = dict((x, True) for x in config['log']['console_theme'][name].split('/')) + if not config.root.log.console_theme.dark_background: + returned['black'] = returned.pop('white', False) + return returned def from_verbosity(level): @@ -166,14 +169,14 @@ def _report_num_collected(self, collected, stillworking): return self._terminal.write('{} tests collected{}'.format( - len(collected), '...' if stillworking else ' \n'), white=True, bold=True) + len(collected), '...' if stillworking else ' \n'), **theme('num-collected')) def _is_verbose(self, level): return self._level <= level @from_verbosity(VERBOSITIES.ERROR) def report_session_start(self, session): - self._terminal.sep('=', 'Session starts', white=True, bold=True) + self._terminal.sep('=', 'Session starts', **theme('session-start')) def report_session_end(self, session): From f83e2034638bcca3a940d79cd7460351880235e3 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Fri, 31 May 2019 13:23:56 +0300 Subject: [PATCH 12/17] Add get_current_scope() (fixes #952) --- doc/changelog.rst | 1 + slash/__init__.py | 1 + slash/core/scope_manager.py | 15 ++++++++++++++- tests/test_scope_manager.py | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 18aa9e02..e0154cb6 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :feature:`952` Added support for getting the currently active scope (``test``, ``module`` or ``session``) through the new ``get_current_scope`` API. ``session.scope_manager.current_scope`` is also available. * :feature:`925` Support was added for terminals with light backgrounds by changing ``log.console_theme.dark_background`` configuration * :feature:`950` Slash now emits a log record when handling fatal errors * :feature:`-` Add tests and suite execution time to xunit plugin diff --git a/slash/__init__.py b/slash/__init__.py index 036822dd..e8dda5e6 100644 --- a/slash/__init__.py +++ b/slash/__init__.py @@ -3,6 +3,7 @@ from .conf import config from .ctx import context from .ctx import g, session, test +from .core.scope_manager import get_current_scope from .core.session import Session from .core.tagging import tag # assertions diff --git a/slash/core/scope_manager.py b/slash/core/scope_manager.py index 8a26e1d5..7f820ee8 100644 --- a/slash/core/scope_manager.py +++ b/slash/core/scope_manager.py @@ -2,6 +2,7 @@ import logbook +from ..ctx import context from ..exceptions import SlashInternalError from ..utils.python import call_all_raise_first @@ -15,6 +16,12 @@ def __init__(self, session): self._scopes = [] self._last_module = self._last_test = None + @property + def current_scope(self): + if not self._scopes: + return None + return self._scopes[-1] + def begin_test(self, test): test_module = test.__slash__.module_name if not test_module: @@ -28,7 +35,7 @@ def begin_test(self, test): if self._last_module is not None: _logger.trace('Module scope has changed. Popping previous module scope') self._pop_scope('module') - assert self._scopes[-1] != 'module' + assert self.current_scope != 'module' self._push_scope('module') self._last_module = test_module self._push_scope('test') @@ -66,3 +73,9 @@ def flush_remaining_scopes(self): _logger.trace('Flushing remaining scopes: {}', self._scopes) call_all_raise_first([functools.partial(self._pop_scope, s) for s in self._scopes[::-1]]) + + +def get_current_scope(): + if context.session is None or context.session.scope_manager is None: + return None + return context.session.scope_manager.current_scope diff --git a/tests/test_scope_manager.py b/tests/test_scope_manager.py index e3165a08..3166892c 100644 --- a/tests/test_scope_manager.py +++ b/tests/test_scope_manager.py @@ -6,7 +6,7 @@ import pytest import slash from slash._compat import iteritems -from slash.core.scope_manager import ScopeManager +from slash.core.scope_manager import ScopeManager, get_current_scope from .utils import make_runnable_tests from .utils.suite_writer import Suite @@ -56,6 +56,37 @@ def test_scope_manager(dummy_fixture_store, scope_manager, tests_by_module): assert not dummy_fixture_store._scopes +def test_get_current_scope(suite_builder): + + @suite_builder.first_file.add_code + def __code__(): + # pylint: disable=unused-variable,redefined-outer-name,reimported + import slash + import gossip + + TOKEN = 'testing-current-scope-token' + + def _validate_current_scope(expected_scope): + assert slash.get_current_scope() == expected_scope + + @gossip.register('slash.after_session_start', token=TOKEN) + def session_validation(): + assert slash.get_current_scope() == 'session' + + @gossip.register('slash.configure', token=TOKEN) + @gossip.register('slash.app_quit', token=TOKEN) + def _no_scope(): + assert slash.get_current_scope() is None + + def test_something(): + assert slash.get_current_scope() == 'test' + + gossip.unregister_token(TOKEN) + + suite_builder.build().run().assert_success(1) + assert get_current_scope() is None + + @pytest.fixture def scope_manager(dummy_fixture_store, forge): session = slash.Session() From df397f0bf1808a677848ac21a12b689a15041b94 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Sun, 16 Jun 2019 17:41:35 +0300 Subject: [PATCH 13/17] Drop support for old-style assertion (closes #452) --- doc/changelog.rst | 1 + slash/__init__.py | 26 +--------- slash/assertions.py | 107 +-------------------------------------- tests/test_assertions.py | 95 ++-------------------------------- 4 files changed, 7 insertions(+), 222 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index e0154cb6..21186110 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :feature:`452` Drop support for old-style assertions * :feature:`952` Added support for getting the currently active scope (``test``, ``module`` or ``session``) through the new ``get_current_scope`` API. ``session.scope_manager.current_scope`` is also available. * :feature:`925` Support was added for terminals with light backgrounds by changing ``log.console_theme.dark_background`` configuration * :feature:`950` Slash now emits a log record when handling fatal errors diff --git a/slash/__init__.py b/slash/__init__.py index e8dda5e6..2017d191 100644 --- a/slash/__init__.py +++ b/slash/__init__.py @@ -1,3 +1,4 @@ +# pylint: disable=unused-import from .__version__ import __version__ from .cleanups import add_cleanup, add_critical_cleanup, add_success_only_cleanup, get_current_cleanup_phase, is_in_cleanup from .conf import config @@ -9,30 +10,7 @@ # assertions from . import assertions should = assertions -from .assertions import ( - allowing_exceptions, - assert_almost_equal, - assert_contains, - assert_equal, - assert_equals, - assert_false, - assert_in, - assert_is, - assert_is_none, - assert_empty, - assert_not_empty, - assert_is_not, - assert_is_not_none, - assert_isinstance, - assert_not_contain, - assert_not_contains, - assert_not_equal, - assert_not_equals, - assert_not_in, - assert_not_isinstance, - assert_raises, - assert_true, - ) +from .assertions import allowing_exceptions, assert_almost_equal, assert_raises from .core.test import Test from .core.test import abstract_test_class from .core.exclusions import exclude diff --git a/slash/assertions.py b/slash/assertions.py index 12264ca4..565b04a4 100644 --- a/slash/assertions.py +++ b/slash/assertions.py @@ -1,117 +1,19 @@ -import operator import sys import logbook -from vintage import deprecated from . import exception_handling from ._compat import PY2 -from .exceptions import TestFailed, ExpectedExceptionNotCaught -from .utils import operator_information +from .exceptions import ExpectedExceptionNotCaught -sys.modules["slash.should"] = sys.modules[__name__] _logger = logbook.Logger(__name__) -def _deprecated(func, message=None): - return deprecated(since='0.19.0', what='slash.should.{.__name__}'.format(func), - message=message or 'Use plain assertions instead')(func) - - -def _binary_assertion(name, operator_func): - op = operator_information.get_operator_by_func(operator_func) - - def _assertion(a, b, msg=None): - if not op(a, b): - msg = _get_message(msg, operator_information.get_operator_by_func( - op.inverse_func).to_expression(a, b)) - raise TestFailed(msg) - _assertion.__name__ = name - _assertion.__doc__ = "Asserts **{}**".format( - op.to_expression("ARG1", "ARG2")) - _assertion = _deprecated(_assertion) - return _assertion - - -def _unary_assertion(name, operator_func): - op = operator_information.get_operator_by_func(operator_func) - - def _assertion(a, msg=None): - if not op(a): - msg = _get_message(msg, operator_information.get_operator_by_func( - op.inverse_func).to_expression(a)) - raise TestFailed(msg) - _assertion.__name__ = name - _assertion.__doc__ = "Asserts **{}**".format(op.to_expression("ARG")) - _assertion = _deprecated(_assertion) - return _assertion - def _get_message(msg, description): if msg is None: return description return "{} ({})".format(msg, description) -equal = _binary_assertion("equal", operator.eq) -assert_equal = assert_equals = equal = equal - -not_equal = _binary_assertion("not_equal", operator.ne) -assert_not_equal = assert_not_equals = not_equals = not_equal - -be_a = _binary_assertion("be_a", operator_information.safe_isinstance) -assert_isinstance = be_a - -not_be_a = _binary_assertion( - "not_be_a", operator_information.safe_not_isinstance) -assert_not_isinstance = not_be_a - -be_none = _unary_assertion("be_none", operator_information.is_none) -assert_is_none = be_none - -not_be_none = _unary_assertion("not_be_none", operator_information.is_not_none) -assert_is_not_none = not_be_none - -be = _binary_assertion("be", operator.is_) -assert_is = be - -not_be = _binary_assertion("not_be", operator.is_not) -assert_is_not = not_be - -be_true = _unary_assertion("be_true", operator.truth) -assert_true = be_true - -be_false = _unary_assertion("be_false", operator.not_) -assert_false = be_false - -be_empty = _unary_assertion("be_empty", operator_information.is_empty) -assert_empty = assert_is_empty = be_empty - -not_be_empty = _unary_assertion( - "not_be_empty", operator_information.is_not_empty) -assert_not_empty = assert_is_not_empty = not_be_empty - -contain = _binary_assertion("contain", operator.contains) -assert_contains = contains = contain - -not_contain = _binary_assertion( - "not_contain", operator_information.not_contains) -assert_not_contains = assert_not_contain = not_contains = not_contain - - -def be_in(a, b, msg=None): - """ - Asserts **ARG1 in ARG2** - """ - return contain(b, a, msg) -assert_in = be_in - - -def not_be_in(a, b, msg=None): - """ - Asserts **ARG1 not in ARG2** - """ - return not_contain(b, a, msg) -assert_not_in = not_be_in - class _CaughtContext(object): @@ -174,13 +76,6 @@ def allowing_exceptions(exception_class, msg=None): return _CaughtContext(msg, exception_class, ensure_caught=False) -@deprecated(since='0.19.0', what='slash.should.raise_exception', message='Use slash.assert_raises instead') -def raise_exception(exception_class, msg=None): - return assert_raises(exception_class, msg=msg) - -raise_exception.__doc__ = assert_raises.__doc__.replace("assert_raises", "raise_exception") - - def assert_almost_equal(a, b, delta=0.00000001): """Asserts that abs(a - b) <= delta """ diff --git a/tests/test_assertions.py b/tests/test_assertions.py index 3f6d28cc..c488ffa8 100644 --- a/tests/test_assertions.py +++ b/tests/test_assertions.py @@ -6,7 +6,6 @@ import pytest import slash -from slash import should from slash.exceptions import TestFailed as _TestFailed, ExpectedExceptionNotCaught @@ -14,7 +13,6 @@ (1, 1), (1, 1.00000001), ]) -@pytest.mark.usefixtures('disable_vintage_deprecations') def test_assert_almost_equal_positive(pair): a, b = pair slash.assert_almost_equal(a, b) @@ -26,7 +24,6 @@ def test_assert_almost_equal_positive(pair): (1, 1.1, 0.5), (1.0001, 1.00009, 0.00002), ]) -@pytest.mark.usefixtures('disable_vintage_deprecations') def test_assert_almost_equal_positive_with_delta(combination): a, b, delta = combination slash.assert_almost_equal(a, b, delta) @@ -37,84 +34,13 @@ def test_assert_almost_equal_positive_with_delta(combination): (1, 1.1, 0.00001), (1.0001, 1.00009, 0.000001), ]) -@pytest.mark.usefixtures('disable_vintage_deprecations') def test_assert_almost_equal_negative_with_delta(combination): a, b, delta = combination with pytest.raises(AssertionError): slash.assert_almost_equal(a, b, delta) -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_assert_equals(): - with checking(should.equal, should.not_equal): - good(1, 1) - bad(1, 2) - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_assert_empty(): - with checking(should.be_empty, should.not_be_empty): - good([]) - good({}) - good(set()) - bad([1, 2, 3]) - bad({1: 2}) - bad(set([1, 2, 3])) - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_assert_isinstance(): - with checking(should.be_a, should.not_be_a): - good(1, int) - good("a", str) - good({}, dict) - bad(1, 1) - bad(1, str) - bad(None, str) - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_is_none(): - with checking(should.be_none, should.not_be_none): - good(None) - bad("None") - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_is(): - obj = object() - with checking(should.be, should.not_be): - good(obj, obj) - bad(obj, object()) - bad({}, {}) - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_truth(): - with checking(should.be_true, should.be_false): - good(True) - good("hello") - bad(False) - bad("") - bad({}) - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_in(): - with checking(should.be_in, should.not_be_in): - good(1, [1, 2, 3]) - good("e", "hello") - bad(1, []) - bad("e", "fdfd") - with checking(should.contain, should.not_contain): - good([1, 2, 3], 1) - good("hello", "e") - bad("fdfdfd", "e") - - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -@pytest.mark.parametrize('func', [should.raise_exception, slash.assert_raises]) +@pytest.mark.parametrize('func', [slash.assert_raises]) def test_assert_raises(func): thrown = CustomException() with func(CustomException) as caught: @@ -134,11 +60,10 @@ def test_assert_raises(func): assert " not raised" in str(e) assert e.expected_types == (CustomException,) else: - assert False, "should.raise_exception allowed success" + assert False, "assert_raises allowed success" -@pytest.mark.usefixtures('disable_vintage_deprecations') -@pytest.mark.parametrize('func', [should.raise_exception, slash.assert_raises]) +@pytest.mark.parametrize('func', [slash.assert_raises]) def test_assert_raises_multiple_exceptions(func): class CustomException1(Exception): pass @@ -163,20 +88,6 @@ class CustomException3(Exception): raise value assert caught.value is value - - - -@pytest.mark.usefixtures('disable_vintage_deprecations') -def test_raises_exception_multiple_classes(): - possible_exception_types = (CustomException, OtherException) - for x in possible_exception_types: - with should.raise_exception(possible_exception_types): - raise x() - - with pytest.raises(ExpectedExceptionNotCaught): - with should.raise_exception((CustomException,)): - pass - # boilerplate From b8eb2ef87990b1d8845be740e18fed2840b4dead Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Mon, 17 Jun 2019 08:34:36 +0300 Subject: [PATCH 14/17] Silent unittest of deprecated API --- tests/test_parallel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index a872b4ae..ec4481f4 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -3,6 +3,7 @@ import signal import sys +from vintage import get_no_deprecations_context from .utils.suite_writer import Suite from slash.resuming import get_tests_from_previous_session from slash.exceptions import InteractiveParallelNotAllowed, ParallelTimeout @@ -234,8 +235,9 @@ def test_traceback_vars(parallel_suite): found_failure += 1 assert len(result.get_failures()) == 1 assert len(result.get_failures()[0].traceback.frames) == 3 - assert 'x' in result.get_failures()[0].traceback.frames[2].locals - assert 'num' in result.get_failures()[0].traceback.frames[1].locals + with get_no_deprecations_context(): + assert 'x' in result.get_failures()[0].traceback.frames[2].locals + assert 'num' in result.get_failures()[0].traceback.frames[1].locals assert found_failure == 1 def test_result_data_not_picklable(parallel_suite): From 58299f7f1966e0d6572281daba8eaf4092cc86e0 Mon Sep 17 00:00:00 2001 From: Maor Marcus Date: Mon, 1 Jul 2019 14:14:41 +0300 Subject: [PATCH 15/17] Return copy of connected clients when iterating over them (closes #961) --- slash/parallel/parallel_manager.py | 2 +- slash/parallel/server.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/slash/parallel/parallel_manager.py b/slash/parallel/parallel_manager.py index ca33282a..34c9c64e 100644 --- a/slash/parallel/parallel_manager.py +++ b/slash/parallel/parallel_manager.py @@ -89,7 +89,7 @@ def wait_all_workers_to_connect(self): def check_worker_timed_out(self): workers_last_connection_time = self.keepalive_server.get_workers_last_connection_time() - for worker_id in self.server.connected_clients: + for worker_id in self.server.get_connected_clients(): worker_last_connection_time = workers_last_connection_time.get(worker_id, None) if worker_last_connection_time is None: #worker keepalive thread didn't started yet continue diff --git a/slash/parallel/server.py b/slash/parallel/server.py index 7a1095ac..a2586b32 100644 --- a/slash/parallel/server.py +++ b/slash/parallel/server.py @@ -87,6 +87,9 @@ def __init__(self, tests): def has_connected_clients(self): return len(self.connected_clients) > 0 + def get_connected_clients(self): + return self.connected_clients.copy() + def has_more_tests(self): return len(self.finished_tests) < len(self.tests) From f3a64e6cce44422fb1ba801eb71f3e599a43d9c6 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Wed, 3 Jul 2019 11:00:36 +0300 Subject: [PATCH 16/17] Drop support for deprecated arguments of add_cleanup (closes #945) --- doc/changelog.rst | 1 + slash/core/cleanup_manager.py | 8 ++------ tests/test_cleanups.py | 7 ------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 21186110..c1436451 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :feature:`945` Drop support for deprecated arguments of ``add_cleanup`` * :feature:`452` Drop support for old-style assertions * :feature:`952` Added support for getting the currently active scope (``test``, ``module`` or ``session``) through the new ``get_current_scope`` API. ``session.scope_manager.current_scope`` is also available. * :feature:`925` Support was added for terminals with light backgrounds by changing ``log.console_theme.dark_background`` configuration diff --git a/slash/core/cleanup_manager.py b/slash/core/cleanup_manager.py index fc99e8ba..ce29f71f 100644 --- a/slash/core/cleanup_manager.py +++ b/slash/core/cleanup_manager.py @@ -2,7 +2,6 @@ import logbook from sentinels import Sentinel -from vintage import warn_deprecation from .. import hooks from ..ctx import context @@ -63,11 +62,8 @@ def add_cleanup(self, _func, *args, **kwargs): new_kwargs = kwargs.pop('kwargs', {}).copy() new_args = list(kwargs.pop('args', ())) - if args or kwargs: - warn_deprecation('Passing *args/**kwargs to slash.add_cleanup is deprecated. ' - 'Use args=(...) and/or kwargs={...} instead', frame_correction=+2) - new_args.extend(args) - new_kwargs.update(kwargs) + assert (not args) and (not kwargs), \ + 'Passing *args/**kwargs to slash.add_cleanup is not supported. Use args=(...) and/or kwargs={...} instead' added = _Cleanup(_func, new_args, new_kwargs, critical=critical, success_only=success_only) diff --git a/tests/test_cleanups.py b/tests/test_cleanups.py index 427737f5..8fcc6dd7 100644 --- a/tests/test_cleanups.py +++ b/tests/test_cleanups.py @@ -125,13 +125,6 @@ def test_cleanups(suite, suite_test): assert summary.events[cleanup] -def test_cleanup_args_kwargs_deprecated(): - with slash.Session() as s: - slash.add_cleanup(lambda: None, "arg1", arg2=1) - [w] = s.warnings - assert 'deprecated' in str(w).lower() - - def test_cleanup_ordering(suite, suite_test): cleanup1 = suite_test.add_cleanup() cleanup2 = suite_test.add_cleanup() From 07521a67ab1d38c32ce01b65074aabc04b1bd208 Mon Sep 17 00:00:00 2001 From: Ayala Shachar Date: Wed, 3 Jul 2019 14:39:40 +0300 Subject: [PATCH 17/17] Update changelog --- doc/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index c1436451..58273cee 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,6 @@ Changelog ========= +* :release:`1.8.0 <03-07-2019>` * :feature:`945` Drop support for deprecated arguments of ``add_cleanup`` * :feature:`452` Drop support for old-style assertions * :feature:`952` Added support for getting the currently active scope (``test``, ``module`` or ``session``) through the new ``get_current_scope`` API. ``session.scope_manager.current_scope`` is also available.