Skip to content

Commit

Permalink
Merge pull request #175 from stevetracvc/feat/code-coverage-report
Browse files Browse the repository at this point in the history 
PR: Add code coverage report
  • Loading branch information
@jitseniesen
jitseniesen authored Jun 27, 2022
2 parents 0b25301 + 2e4b88f commit 6bfbc0a
Show file tree
Hide file tree
Showing 23 changed files with 963 additions and 542 deletions.
6 changes: 3 additions & 3 deletions spyder_unittest/backend/noserunner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ def get_versions(self):
" {} {}".format(ep.dist.project_name, ep.dist.version))
return versions

def create_argument_list(self):
def create_argument_list(self, config, cov_path):
"""Create argument list for testing process."""
return [
'-m', self.module, '--with-xunit',
'--xunit-file={}'.format(self.resultfilename)
]
'--xunit-file={}'.format(self.resultfilename),
]

def finished(self):
"""Called when the unit test process has finished."""
Expand Down
60 changes: 54 additions & 6 deletions spyder_unittest/backend/pytestrunner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,20 @@
# Standard library imports
import os
import os.path as osp
import re

# Local imports
from spyder_unittest.backend.runnerbase import Category, RunnerBase, TestResult
from spyder.config.base import get_translation
from spyder_unittest.backend.runnerbase import (Category, RunnerBase,
TestResult, COV_TEST_NAME)
from spyder_unittest.backend.zmqstream import ZmqStreamReader

try:
_ = get_translation('spyder_unittest')
except KeyError:
import gettext
_ = gettext.gettext


class PyTestRunner(RunnerBase):
"""Class for running tests within pytest framework."""
Expand Down Expand Up @@ -40,17 +49,21 @@ def pytest_cmdline_main(self, config):
plugins=[GetPluginVersionsPlugin()])
return versions

def create_argument_list(self):
def create_argument_list(self, config, cov_path):
"""Create argument list for testing process."""
pyfile = os.path.join(os.path.dirname(__file__), 'pytestworker.py')
return [pyfile, str(self.reader.port)]
arguments = [pyfile, str(self.reader.port)]
if config.coverage:
arguments += [f'--cov={cov_path}', '--cov-report=term-missing']
return arguments


def start(self, config, executable, pythonpath):
def start(self, config, cov_path, executable, pythonpath):
"""Start process which will run the unit test suite."""
self.config = config
self.reader = ZmqStreamReader()
self.reader.sig_received.connect(self.process_output)
RunnerBase.start(self, config, executable, pythonpath)
RunnerBase.start(self, config, cov_path, executable, pythonpath)

def process_output(self, output):
"""
Expand All @@ -65,7 +78,6 @@ def process_output(self, output):
collecterror_list = []
starttest_list = []
result_list = []

for result_item in output:
if result_item['event'] == 'config':
self.rootdir = result_item['rootdir']
Expand All @@ -90,6 +102,40 @@ def process_output(self, output):
if result_list:
self.sig_testresult.emit(result_list)

def process_coverage(self, output):
"""Search the output text for coverage details.
Called by the function 'finished' at the very end.
"""
cov_results = re.search(
r'-*? coverage:.*?-*\nTOTAL\s.*?\s(\d*?)\%.*\n=*',
output, flags=re.S)
if cov_results:
total_coverage = cov_results.group(1)
cov_report = TestResult(
Category.COVERAGE, f'{total_coverage}%', COV_TEST_NAME)
# create a fake test, then emit the coverage as the result
# This gives overall test coverage, used in TestDataModel.summary
self.sig_collected.emit([COV_TEST_NAME])
self.sig_testresult.emit([cov_report])

# also build a result for each file's coverage
header = "".join(cov_results.group(0).split("\n")[1:3])
# coverage report columns:
# Name Stmts Miss Cover Missing
for row in re.findall(
r'^((.*?\.py) .*?(\d+%).*?(\d[\d\,\-\ ]*)?)$',
cov_results.group(0), flags=re.M):
lineno = (int(re.search(r'^(\d*)', row[3]).group(1)) - 1
if row[3] else None)
file_cov = TestResult(
Category.COVERAGE, row[2], row[1],
message=_('Missing: {}').format(row[3] if row[3] else _("(none)")),
extra_text=_('{}\n{}').format(header, row[0]), filename=row[1],
lineno=lineno)
self.sig_collected.emit([row[1]])
self.sig_testresult.emit([file_cov])

def finished(self):
"""
Called when the unit test process has finished.
Expand All @@ -98,6 +144,8 @@ def finished(self):
"""
self.reader.close()
output = self.read_all_process_output()
if self.config.coverage:
self.process_coverage(output)
no_tests_ran = "no tests ran" in output.splitlines()[-1]
self.sig_finished.emit([] if no_tests_ran else None, output)

Expand Down
14 changes: 11 additions & 3 deletions spyder_unittest/backend/runnerbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,19 @@
from pkgutil import find_loader as find_spec_or_loader


# if generating coverage report, use this name for the TestResult
# it's here in case we can get coverage results from unittest too
COV_TEST_NAME = 'Total Test Coverage'


class Category:
"""Enum type representing category of test result."""

FAIL = 1
OK = 2
SKIP = 3
PENDING = 4
COVERAGE = 5


class TestResult:
Expand Down Expand Up @@ -160,7 +166,7 @@ def get_versions(self):
"""
raise NotImplementedError

def create_argument_list(self):
def create_argument_list(self, config, cov_path):
"""
Create argument list for testing process (dummy).
Expand Down Expand Up @@ -188,7 +194,7 @@ def _prepare_process(self, config, pythonpath):
process.setProcessEnvironment(env)
return process

def start(self, config, executable, pythonpath):
def start(self, config, cov_path, executable, pythonpath):
"""
Start process which will run the unit test suite.
Expand All @@ -202,6 +208,8 @@ def start(self, config, executable, pythonpath):
----------
config : TestConfig
Unit test configuration.
cov_path : str or None
Path to filter source for coverage report
executable : str
Path to Python executable
pythonpath : list of str
Expand All @@ -213,7 +221,7 @@ def start(self, config, executable, pythonpath):
If process failed to start.
"""
self.process = self._prepare_process(config, pythonpath)
p_args = self.create_argument_list()
p_args = self.create_argument_list(config, cov_path)
try:
os.remove(self.resultfilename)
except OSError:
Expand Down
87 changes: 83 additions & 4 deletions spyder_unittest/backend/tests/test_pytestrunner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
# Local imports
from spyder_unittest.backend.pytestrunner import (PyTestRunner,
logreport_to_testresult)
from spyder_unittest.backend.runnerbase import Category, TestResult
from spyder_unittest.backend.runnerbase import (Category, TestResult,
COV_TEST_NAME)
from spyder_unittest.widgets.configdialog import Config

try:
Expand All @@ -30,6 +31,8 @@ def test_pytestrunner_is_installed():


def test_pytestrunner_create_argument_list(monkeypatch):
config = Config()
cov_path = None
MockZMQStreamReader = Mock()
monkeypatch.setattr(
'spyder_unittest.backend.pytestrunner.ZmqStreamReader',
Expand All @@ -40,7 +43,7 @@ def test_pytestrunner_create_argument_list(monkeypatch):
runner.reader = mock_reader
monkeypatch.setattr('spyder_unittest.backend.pytestrunner.os.path.dirname',
lambda _: 'dir')
pyfile, port = runner.create_argument_list()
pyfile, port, *coverage = runner.create_argument_list(config, cov_path)
assert pyfile == 'dir{}pytestworker.py'.format(os.sep)
assert port == '42'

Expand All @@ -58,13 +61,14 @@ def test_pytestrunner_start(monkeypatch):

runner = PyTestRunner(None, 'results')
config = Config()
runner.start(config, sys.executable, ['pythondir'])
cov_path = None
runner.start(config, cov_path, sys.executable, ['pythondir'])
assert runner.config is config
assert runner.reader is mock_reader
runner.reader.sig_received.connect.assert_called_once_with(
runner.process_output)
MockRunnerBase.start.assert_called_once_with(
runner, config, sys.executable, ['pythondir'])
runner, config, cov_path, sys.executable, ['pythondir'])


def test_pytestrunner_process_output_with_collected(qtbot):
Expand Down Expand Up @@ -110,6 +114,7 @@ def test_pytestrunner_finished(qtbot, output, results):
runner = PyTestRunner(None)
runner.reader = mock_reader
runner.read_all_process_output = lambda: output
runner.config = Config('pytest', None, False)
with qtbot.waitSignal(runner.sig_finished) as blocker:
runner.finished()
assert blocker.args == [results, output]
Expand Down Expand Up @@ -137,6 +142,80 @@ def test_pytestrunner_process_output_with_logreport_passed(qtbot):
assert blocker.args == [expected]


def test_pytestrunner_process_coverage(qtbot):
output = """
============================= test session starts ==============================
platform linux -- Python 3.9.12, pytest-7.1.2, pluggy-1.0.0
PyQt5 5.12.3 -- Qt runtime 5.12.9 -- Qt compiled 5.12.9
rootdir: /TRAC/TRAC-data/spyder-unittest, configfile: setup.cfg
plugins: flaky-3.7.0, cov-3.0.0, qt-4.0.2, mock-3.7.0
collected 152 items
spyder_unittest/backend/tests/test_abbreviator.py ........... [ 7%]
spyder_unittest/backend/tests/test_frameworkregistry.py .. [ 8%]
spyder_unittest/backend/tests/test_noserunner.py ..... [ 11%]
spyder_unittest/backend/tests/test_pytestrunner.py ..................... [ 25%]
.... [ 28%]
spyder_unittest/backend/tests/test_pytestworker.py ..................... [ 42%]
.... [ 44%]
spyder_unittest/backend/tests/test_runnerbase.py ..... [ 48%]
spyder_unittest/backend/tests/test_unittestrunner.py .......... [ 54%]
spyder_unittest/backend/tests/test_zmqstream.py . [ 55%]
spyder_unittest/tests/test_unittestplugin.py s.sss [ 58%]
spyder_unittest/widgets/tests/test_configdialog.py ........... [ 65%]
spyder_unittest/widgets/tests/test_datatree.py ......................... [ 82%]
.. [ 83%]
spyder_unittest/widgets/tests/test_unittestgui.py ...................... [ 98%]
... [100%]
=============================== warnings summary ===============================
---------- coverage: platform linux, python 3.9.12-final-0 -----------
Name Stmts Miss Cover Missing
-------------------------------------------------------------------------
setup.py 26 26 0% 7-53
spyder_unittest/backend/noserunner.py 62 7 89% 17-19, 71-72, 94, 103
spyder_unittest/backend/pytestrunner.py 101 6 94% 100-106
spyder_unittest/backend/pytestworker.py 78 4 95% 36, 40, 44, 152
spyder_unittest/backend/runnerbase.py 87 2 98% 20-21
spyder_unittest/backend/unittestrunner.py 78 5 94% 69, 75, 123, 138, 146
spyder_unittest/unittestplugin.py 119 65 45% 60, 71, 119-123, 136-141, 148-150, 161, 170-173, 183-186, 207-208, 219-226, 240-272, 280-289, 299-301, 313-314
spyder_unittest/widgets/configdialog.py 95 10 89% 28-30, 134-135, 144, 173-176
spyder_unittest/widgets/datatree.py 244 14 94% 26-28, 100, 105, 107, 276-277, 280, 293, 312, 417, 422-424
spyder_unittest/widgets/unittestgui.py 218 35 84% 41-43, 49, 223, 241, 245, 249-256, 271-278, 302-305, 330, 351-352, 468-482
-------------------------------------------------------------------------
TOTAL 1201 174 86%
6 files skipped due to complete coverage.
================= 148 passed, 4 skipped, 242 warnings in 4.25s =================
"""
cov_text = """
---------- coverage: platform linux, python 3.9.12-final-0 -----------
Name Stmts Miss Cover Missing
-------------------------------------------------------------------------
setup.py 26 26 0% 7-53
spyder_unittest/backend/noserunner.py 62 7 89% 17-19, 71-72, 94, 103
spyder_unittest/backend/pytestrunner.py 101 6 94% 100-106
spyder_unittest/backend/pytestworker.py 78 4 95% 36, 40, 44, 152
spyder_unittest/backend/runnerbase.py 87 2 98% 20-21
spyder_unittest/backend/unittestrunner.py 78 5 94% 69, 75, 123, 138, 146
spyder_unittest/unittestplugin.py 119 65 45% 60, 71, 119-123, 136-141, 148-150, 161, 170-173, 183-186, 207-208, 219-226, 240-272, 280-289, 299-301, 313-314
spyder_unittest/widgets/configdialog.py 95 10 89% 28-30, 134-135, 144, 173-176
spyder_unittest/widgets/datatree.py 244 14 94% 26-28, 100, 105, 107, 276-277, 280, 293, 312, 417, 422-424
spyder_unittest/widgets/unittestgui.py 218 35 84% 41-43, 49, 223, 241, 245, 249-256, 271-278, 302-305, 330, 351-352, 468-482
-------------------------------------------------------------------------
TOTAL 1201 174 86%
6 files skipped due to complete coverage."""
runner = PyTestRunner(None)
runner.rootdir = 'ham'
with qtbot.waitSignal(runner.sig_testresult) as blocker:
runner.process_coverage(output)
expected = TestResult(
Category.COVERAGE, "86%", COV_TEST_NAME, extra_text=cov_text)


@pytest.mark.parametrize('outcome,witherror,category', [
('passed', True, Category.FAIL),
('passed', False, Category.OK),
Expand Down
19 changes: 16 additions & 3 deletions spyder_unittest/backend/tests/test_runnerbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@ class FooRunner(RunnerBase):

assert not FooRunner.is_installed()

foo_runner = FooRunner(None)
config = Config(foo_runner.module, 'wdir', True)

with pytest.raises(NotImplementedError):
foo_runner.create_argument_list(config, 'cov_path')

with pytest.raises(NotImplementedError):
foo_runner.get_versions()

with pytest.raises(NotImplementedError):
foo_runner.finished()


@pytest.mark.parametrize('pythonpath,env_pythonpath', [
([], None),
Expand Down Expand Up @@ -78,11 +90,12 @@ def test_runnerbase_start(monkeypatch):

runner = RunnerBase(None, 'results')
runner._prepare_process = lambda c, p: mock_process
runner.create_argument_list = lambda: ['arg1', 'arg2']
config = Config('pytest', 'wdir')
runner.create_argument_list = lambda c, cp: ['arg1', 'arg2']
config = Config('pytest', 'wdir', False)
cov_path = None
mock_process.waitForStarted = lambda: False
with pytest.raises(RuntimeError):
runner.start(config, 'python_exec', ['pythondir'])
runner.start(config, cov_path, 'python_exec', ['pythondir'])

mock_process.start.assert_called_once_with('python_exec', ['arg1', 'arg2'])
mock_remove.assert_called_once_with('results')
2 changes: 1 addition & 1 deletion spyder_unittest/backend/unittestrunner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def get_versions(self):
import platform
return ['unittest {}'.format(platform.python_version())]

def create_argument_list(self):
def create_argument_list(self, config, cov_path):
"""Create argument list for testing process."""
return ['-m', self.module, 'discover', '-v']

Expand Down
Loading

0 comments on commit 6bfbc0a

Please sign in to comment.