diff --git a/Jenkinsfile b/Jenkinsfile index d276dbbc56e..0577f741882 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1075,7 +1075,8 @@ def dumpFilteredEnvironment(){ echo "PROCESSES: ${env.PROCESSES}" echo "TIMEOUT: ${env.TIMEOUT}" echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" + echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go index 196eba36d48..2a195a41c7d 100644 --- a/dev-tools/mage/check.go +++ b/dev-tools/mage/check.go @@ -43,12 +43,12 @@ import ( // if it finds any modifications. If executed in in verbose mode it will write // the results of 'git diff' to stdout to indicate what changes have been made. // -// It checks the file permissions of nosetests test cases and YAML files. +// It checks the file permissions of python test cases and YAML files. // It checks .go source files using 'go vet'. func Check() error { fmt.Println(">> check: Checking source code for common problems") - mg.Deps(GoVet, CheckNosetestsNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat) + mg.Deps(GoVet, CheckPythonTestNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat) changes, err := GitDiffIndex() if err != nil { @@ -124,16 +124,15 @@ func GitDiff() error { return err } -// CheckNosetestsNotExecutable checks that none of the nosetests files are -// executable. Nosetests silently skips executable .py files and we don't want -// this to happen. -func CheckNosetestsNotExecutable() error { +// CheckPythonTestNotExecutable checks that none of the python test files are +// executable. They are silently skipped and we don't want this to happen. +func CheckPythonTestNotExecutable() error { if runtime.GOOS == "windows" { // Skip windows because it doesn't have POSIX permissions. return nil } - tests, err := FindFiles(nosetestsTestFiles...) + tests, err := FindFiles(pythonTestFiles...) if err != nil { return err } @@ -151,7 +150,7 @@ func CheckNosetestsNotExecutable() error { } if len(executableTestFiles) > 0 { - return errors.Errorf("nosetests files cannot be executable because "+ + return errors.Errorf("python test files cannot be executable because "+ "they will be skipped. Fix permissions of %v", executableTestFiles) } return nil diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index c8a12fdfab6..e562fdef95b 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -56,7 +56,7 @@ var ( pythonVirtualenvLock sync.Mutex // More globs may be needed in the future if tests are added in more places. - nosetestsTestFiles = []string{ + pythonTestFiles = []string{ "tests/system/test_*.py", "module/*/test_*.py", "module/*/*/test_*.py", @@ -77,11 +77,11 @@ func init() { } // PythonTestArgs are the arguments used for the "python*Test" targets and they -// define how "nosetests" is invoked. +// define how python tests are invoked. type PythonTestArgs struct { TestName string // Test name used in logging. Env map[string]string // Env vars to add to the current env. - Files []string // Globs used by nosetests to find tests. + Files []string // Globs used to find tests. XUnitReportFile string // File to write the XUnit XML test report to. CoverageProfileFile string // Test coverage profile file. } @@ -109,8 +109,8 @@ func DefaultPythonTestUnitArgs() PythonTestArgs { return makePythonTestArgs("Uni // checking for INTEGRATION_TEST=1 in the test code. func DefaultPythonTestIntegrationArgs() PythonTestArgs { return makePythonTestArgs("Integration") } -// PythonNoseTest invokes "nosetests" via a Python virtualenv. -func PythonNoseTest(params PythonTestArgs) error { +// PythonTest executes python tests via a Python virtualenv. +func PythonTest(params PythonTestArgs) error { fmt.Println(">> python test:", params.TestName, "Testing") ve, err := PythonVirtualenv() @@ -118,30 +118,31 @@ func PythonNoseTest(params PythonTestArgs) error { return err } - nosetestsEnv := map[string]string{ + pytestEnv := map[string]string{ // activate sets this. Not sure if it's ever needed. "VIRTUAL_ENV": ve, } if IsInIntegTestEnv() { - nosetestsEnv["INTEGRATION_TESTS"] = "1" + pytestEnv["INTEGRATION_TESTS"] = "1" } for k, v := range params.Env { - nosetestsEnv[k] = v + pytestEnv[k] = v } - nosetestsOptions := []string{ - "--process-timeout=90", - "--with-timer", - "-v", + pytestOptions := []string{ + "--timeout=90", + "--durations=20", + } + if mg.Verbose() { + pytestOptions = append(pytestOptions, "-v") } if params.XUnitReportFile != "" { - nosetestsOptions = append(nosetestsOptions, - "--with-xunit", - "--xunit-file="+createDir(params.XUnitReportFile), + pytestOptions = append(pytestOptions, + "--junit-xml="+createDir(params.XUnitReportFile), ) } - files := nosetestsTestFiles + files := pythonTestFiles if len(params.Files) > 0 { files = params.Files } @@ -157,23 +158,23 @@ func PythonNoseTest(params PythonTestArgs) error { // We check both the VE and the normal PATH because on Windows if the // requirements are met by the globally installed package they are not // installed to the VE. - nosetestsPath, err := LookVirtualenvPath(ve, "nosetests") + pytestPath, err := LookVirtualenvPath(ve, "pytest") if err != nil { return err } defer fmt.Println(">> python test:", params.TestName, "Testing Complete") - _, err = sh.Exec(nosetestsEnv, os.Stdout, os.Stderr, nosetestsPath, append(nosetestsOptions, testFiles...)...) + _, err = sh.Exec(pytestEnv, os.Stdout, os.Stderr, pytestPath, append(pytestOptions, testFiles...)...) return err // TODO: Aggregate all the individual code coverage reports and generate // and HTML report. } -// PythonNoseTestForModule executes python system tests for modules. +// PythonTestForModule executes python system tests for modules. // // Use `MODULE=module` to run only tests for `module`. -func PythonNoseTestForModule(params PythonTestArgs) error { +func PythonTestForModule(params PythonTestArgs) error { if module := EnvOr("MODULE", ""); module != "" { params.Files = []string{ fmt.Sprintf("module/%s/test_*.py", module), @@ -181,7 +182,7 @@ func PythonNoseTestForModule(params PythonTestArgs) error { } params.TestName += "-" + module } - return PythonNoseTest(params) + return PythonTest(params) } // PythonVirtualenv constructs a virtualenv that contains the given modules as diff --git a/dev-tools/mage/target/integtest/integtest.go b/dev-tools/mage/target/integtest/integtest.go index 62d601cea6d..7b7cae52819 100644 --- a/dev-tools/mage/target/integtest/integtest.go +++ b/dev-tools/mage/target/integtest/integtest.go @@ -74,19 +74,18 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(pythonTestDeps...) } - runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("NOSE_")...)...) + runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("PYTEST_")...)...) if err != nil { return err } return runner.Test("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs()) }) } diff --git a/dev-tools/mage/target/unittest/unittest.go b/dev-tools/mage/target/unittest/unittest.go index 859849bac1e..891b5c23309 100644 --- a/dev-tools/mage/target/unittest/unittest.go +++ b/dev-tools/mage/target/unittest/unittest.go @@ -61,5 +61,5 @@ func GoUnitTest(ctx context.Context) error { func PythonUnitTest() error { mg.SerialDeps(pythonTestDeps...) mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } diff --git a/docs/devguide/modules-dev-guide.asciidoc b/docs/devguide/modules-dev-guide.asciidoc index 40b6fe2c0b9..0c0c99f3ddf 100644 --- a/docs/devguide/modules-dev-guide.asciidoc +++ b/docs/devguide/modules-dev-guide.asciidoc @@ -488,4 +488,4 @@ locally for a specific module, using the following procedure under Filebeat dire . Create python env: `make python-env` . Source python env: `./build/python-env/bin/activate` . Create the testing binary: `make filebeat.test` -. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx nosetests tests/system/test_modules.py` +. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx pytest tests/system/test_modules.py` diff --git a/docs/devguide/python.asciidoc b/docs/devguide/python.asciidoc index 4f9902af205..923166fe62b 100644 --- a/docs/devguide/python.asciidoc +++ b/docs/devguide/python.asciidoc @@ -52,14 +52,14 @@ of these virtual environments: is created by `make` or `mage` targets when needed. Virtual environments can also be used without `make` or `mage`, this is usual -for example when running individual system tests with `nosetests`. There are two +for example when running individual system tests with `pytest`. There are two ways to run commands from the virtual environment: * "Activating" the virtual environment in your current terminal running `source ./build/python-env/bin/activate`. Virtual environment can be deactivated by running `deactivate`. * Directly running commands from the virtual environment path. For example - `nosetests` can be executed as `./build/python-env/bin/nosetests`. + `pytest` can be executed as `./build/python-env/bin/pytest`. To recreate a virtual environment, remove its directory. All virtual environments are also removed with `make clean`. diff --git a/docs/devguide/testing.asciidoc b/docs/devguide/testing.asciidoc index a21a8d65ce0..75a476d5c5a 100644 --- a/docs/devguide/testing.asciidoc +++ b/docs/devguide/testing.asciidoc @@ -8,7 +8,8 @@ In general there are two major test suites: * Tests written in Go * Tests written in Python -The tests written in Go use the https://golang.org/pkg/testing/[Go Testing package]. The tests written in Python depend on http://nose.readthedocs.io/en/latest/[nosetests] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs. +The tests written in Go use the https://golang.org/pkg/testing/[Go Testing +package]. The tests written in Python depend on https://docs.pytest.org/en/latest/[pytest] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs. For both of the above test suites so called integration tests exists. Integration tests in Beats are tests which require an external system like Elasticsearch to test if the integration with this service works as expected. Beats provides in its testsuite docker containers and docker-compose files to start these environments but a developer can run the required services also locally. @@ -23,7 +24,7 @@ All Go tests are in the same package as the tested code itself and have the post ==== Running Python Tests -The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `nosetests` has to be run inside `tests/system`. +The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `pytest` has to be run inside `tests/system`. To automate all these steps into one `make system-tests` can be run. This creates the binary, the environment and runs all tests which do not require and external service. diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 6760476cd2b..9b90cc68d9b 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -215,7 +215,7 @@ func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.Deps(Fields) } - runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "NOSE_"), "GENERATE")...) + runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "PYTEST_"), "GENERATE")...) if err != nil { return err } @@ -223,6 +223,6 @@ func PythonIntegTest(ctx context.Context) error { mg.Deps(devtools.BuildSystemTestBinary) args := devtools.DefaultPythonTestIntegrationArgs() args.Env["MODULES_PATH"] = devtools.CWD("module") - return devtools.PythonNoseTest(args) + return devtools.PythonTest(args) }) } diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 2b22e7e830b..f3b5d0877a6 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- - -from filebeat import BaseTest - import codecs import os +import shutil import time import unittest -from nose.plugins.skip import Skip, SkipTest -import shutil +from filebeat import BaseTest # Additional tests to be added: # * Check what happens when file renamed -> no recrawling should happen @@ -767,7 +764,7 @@ def test_file_no_permission(self): """ if os.name != "nt" and os.geteuid() == 0: # root ignores permission flags, so we have to skip the test - raise SkipTest + raise unittest.SkipTest self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", @@ -789,7 +786,7 @@ def test_file_no_permission(self): if os.name == "nt": - raise SkipTest + raise unittest.SkipTest # TODO: Currently skipping this test on windows as it requires `pip install win32api` # which seems to have windows only dependencies. # To solve this problem a requirements_windows.txt could be introduced which would diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index b53a8d18032..6a35a6ce250 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -1,12 +1,12 @@ -from filebeat import BaseTest import os import logging import logging.handlers import json import time import unittest -from nose.plugins.skip import Skip, SkipTest -from nose.plugins.attrib import attr +import pytest + +from filebeat import BaseTest """ Test filebeat under different load scenarios @@ -25,7 +25,7 @@ def test_no_missing_events(self): if os.name == "nt": # This test is currently skipped on windows because very fast file # rotation cannot happen when harvester has file handler still open. - raise SkipTest + raise unittest.SkipTest log_file = self.working_dir + "/log/test.log" os.mkdir(self.working_dir + "/log/") @@ -63,7 +63,7 @@ def test_no_missing_events(self): for i in range(total_lines): # Make sure each line has the same length line = format(i, str(line_length - 1)) - logger.debug("%d", i) + logger.warning("%d", i) # wait until all lines are read self.wait_until( @@ -106,7 +106,7 @@ def test_no_missing_events(self): assert len(entry_list) == total_lines @unittest.skipUnless(LOAD_TESTS, "load test") - @attr('load') + @pytest.mark.load def test_large_number_of_files(self): """ Tests the number of files filebeat can open on startup @@ -148,7 +148,7 @@ def test_large_number_of_files(self): assert len(data) == number_of_files @unittest.skipUnless(LOAD_TESTS, "load test") - @attr('load') + @pytest.mark.load def test_concurrent_harvesters(self): """ Test large number of files on startup if harvester overlap happens and would create too many events diff --git a/filebeat/tests/system/test_publisher.py b/filebeat/tests/system/test_publisher.py index 227f09d1050..3f018cab6f3 100644 --- a/filebeat/tests/system/test_publisher.py +++ b/filebeat/tests/system/test_publisher.py @@ -1,11 +1,9 @@ -from filebeat import BaseTest - import os import platform import time import shutil import json -from nose.plugins.skip import Skip, SkipTest +from filebeat import BaseTest # Additional tests: to be implemented diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 5dda351a5f9..55d815e9c92 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -7,9 +7,7 @@ import stat import time import unittest - from filebeat import BaseTest -from nose.plugins.skip import SkipTest # Additional tests: to be implemented @@ -163,7 +161,7 @@ def test_registry_file_default_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry" registry_path = os.path.join(registry_home, "filebeat") @@ -197,7 +195,7 @@ def test_registry_file_custom_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry" registry_path = os.path.join(registry_home, "filebeat") @@ -232,7 +230,7 @@ def test_registry_file_update_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry_x" registry_path = os.path.join(registry_home, "filebeat") @@ -363,7 +361,7 @@ def test_rotating_file_inode(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" @@ -453,7 +451,7 @@ def test_restart_continue(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" @@ -527,7 +525,7 @@ def test_rotating_file_with_restart(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" diff --git a/filebeat/tests/system/test_registrar_upgrade.py b/filebeat/tests/system/test_registrar_upgrade.py index 7a077caea51..91e0664d214 100644 --- a/filebeat/tests/system/test_registrar_upgrade.py +++ b/filebeat/tests/system/test_registrar_upgrade.py @@ -3,8 +3,7 @@ import os import json - -from nose.plugins.skip import Skip, SkipTest +import unittest from filebeat import BaseTest @@ -55,7 +54,7 @@ def prepare_log(self): # test is current skipped on windows, due to FileStateOS must match the # current OS format. if os.name == "nt": - raise SkipTest + raise unittest.SkipTest self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*" diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index 83813a5ccaa..4001fd863c9 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -1,9 +1,10 @@ -from filebeat import BaseTest import socket import ssl import unittest +import pytest + +from filebeat import BaseTest from os import path -from nose.tools import raises, assert_raises NUMBER_OF_EVENTS = 2 @@ -86,7 +87,6 @@ def test_tcp_over_tls_and_verify_valid_server_without_mutual_auth(self): sock.close() - @raises(ssl.SSLError) def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): """ Test filebeat TCP with TLS with an invalid cacert and not requiring mutual auth. @@ -123,9 +123,10 @@ def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP tls = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=CERTIFICATE2, do_handshake_on_connect=True) - tls.connect((config.get('host'), config.get('port'))) - @raises(ssl.SSLError) + with pytest.raises(ssl.SSLError): + tls.connect((config.get('host'), config.get('port'))) + def test_tcp_over_tls_mutual_auth_fails(self): """ Test filebeat TCP with TLS with default setting to enforce client auth, with bad client certificates @@ -162,12 +163,13 @@ def test_tcp_over_tls_mutual_auth_fails(self): tls = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=CERTIFICATE1, do_handshake_on_connect=True) - tls.connect((config.get('host'), config.get('port'))) - # In TLS 1.3 authentication failures are not detected by the initial - # connection and handshake. For the client to detect that authentication - # has failed (at least in python) it must wait for a server response - # so that the failure can be reported as an exception when it arrives. - tls.recv(1) + with pytest.raises(ssl.SSLError): + tls.connect((config.get('host'), config.get('port'))) + # In TLS 1.3 authentication failures are not detected by the initial + # connection and handshake. For the client to detect that authentication + # has failed (at least in python) it must wait for a server response + # so that the failure can be reported as an exception when it arrives. + tls.recv(1) def test_tcp_over_tls_mutual_auth_succeed(self): """ @@ -265,7 +267,7 @@ def test_tcp_tls_with_a_plain_text_socket(self): # The TLS handshake will close the connection, resulting in a broken pipe. # no events should be written on disk. - with assert_raises(IOError): + with pytest.raises(IOError): for n in range(0, 100000): sock.send(bytes("Hello World: " + str(n) + "\n", "utf-8")) diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 4e4468e240b..e60d1096919 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -7950,7 +7950,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- @@ -7961,7 +7961,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- diff --git a/heartbeat/tests/system/heartbeat.py b/heartbeat/tests/system/heartbeat.py index 3c93396c7ed..70b7faf1ac3 100644 --- a/heartbeat/tests/system/heartbeat.py +++ b/heartbeat/tests/system/heartbeat.py @@ -2,7 +2,6 @@ import sys import http.server import threading -import nose.tools sys.path.append(os.path.join(os.path.dirname( __file__), '../../../libbeat/tests/system')) @@ -69,7 +68,7 @@ def monitors_dir(self): return self.working_dir + "/monitors.d/" def assert_last_status(self, status): - nose.tools.eq_(self.last_output_line()["monitor.status"], status) + self.assertEqual(self.last_output_line()["monitor.status"], status) def setup_dynamic(self, extra_beat_args=[]): os.mkdir(self.monitors_dir()) diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 683ffde212a..876c9ee43dd 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -4,7 +4,6 @@ from heartbeat import BaseTest from elasticsearch import Elasticsearch from beat.beat import INTEGRATION_TESTS -import nose.tools class Test(BaseTest): @@ -180,7 +179,7 @@ def test_dataset(self): heartbeat_proc.check_kill_and_wait() for output in self.read_output(): - nose.tools.assert_equal( + self.assertEqual( output["event.dataset"], "uptime", "Check for event.dataset in {} failed".format(output) diff --git a/heartbeat/tests/system/test_icmp.py b/heartbeat/tests/system/test_icmp.py index 5e7fc551552..7f61a7430f8 100644 --- a/heartbeat/tests/system/test_icmp.py +++ b/heartbeat/tests/system/test_icmp.py @@ -1,15 +1,14 @@ +import logging import os -import unittest import platform import socket -import sys -from heartbeat import BaseTest -from elasticsearch import Elasticsearch -from beat.beat import INTEGRATION_TESTS -import nose.tools -import logging import subprocess +import sys import time +import unittest +from beat.beat import INTEGRATION_TESTS +from elasticsearch import Elasticsearch +from heartbeat import BaseTest class Test(BaseTest): diff --git a/heartbeat/tests/system/test_monitor.py b/heartbeat/tests/system/test_monitor.py index 9f5aa8d43c1..98ace615bd3 100644 --- a/heartbeat/tests/system/test_monitor.py +++ b/heartbeat/tests/system/test_monitor.py @@ -1,8 +1,8 @@ +import os +import unittest + from heartbeat import BaseTest from parameterized import parameterized -import os -from nose.plugins.skip import SkipTest -import nose.tools class Test(BaseTest): @@ -34,7 +34,7 @@ def test_http(self, status_code): if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) @parameterized.expand([ @@ -64,7 +64,7 @@ def test_http_with_hosts_config(self, status_code): if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) def test_http_delayed(self): @@ -83,7 +83,7 @@ def test_http_delayed(self): try: proc = self.start_beat() self.wait_until(lambda: self.output_has(lines=1)) - nose.tools.assert_greater_equal( + self.assertGreaterEqual( self.last_output_line()['http.rtt.total.us'], delay) finally: proc.check_kill_and_wait() @@ -125,7 +125,7 @@ def test_http_json(self, expected_status, body): self.assert_last_status(expected_status) if expected_status == "down": - nose.tools.eq_(self.last_output_line()["http.response.body.content"], body) + self.assertEqual(self.last_output_line()["http.response.body.content"], body) else: assert "http.response.body.content" not in self.last_output_line() finally: @@ -201,7 +201,7 @@ def test_tcp(self, url, status): self.assert_last_status(status) if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) finally: server.shutdown() diff --git a/heartbeat/tests/system/test_telemetry.py b/heartbeat/tests/system/test_telemetry.py index d6420c2c56b..1575a5eed57 100644 --- a/heartbeat/tests/system/test_telemetry.py +++ b/heartbeat/tests/system/test_telemetry.py @@ -1,11 +1,10 @@ -from heartbeat import BaseTest +import unittest import urllib.request import urllib.error import urllib.parse import json -import nose.tools import os -from nose.plugins.skip import SkipTest +from heartbeat import BaseTest class Test(BaseTest): @@ -18,12 +17,12 @@ def test_telemetry(self): Test that telemetry metrics are correctly registered and increment / decrement """ # This test is flaky https://github.com/elastic/beats/issues/8966 - raise SkipTest + raise unittest.SkipTest if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest server = self.start_server("hello world", 200) try: @@ -90,8 +89,7 @@ def test_telemetry(self): self.proc.check_kill_and_wait() server.shutdown() - @staticmethod - def assert_state(expected={}): + def assert_state(self, expected={}): stats = json.loads(urllib.request.urlopen( "http://localhost:5066/state").read()) @@ -104,23 +102,22 @@ def assert_state(expected={}): endpoints = proto_expected.get("endpoints", 0) total_monitors += monitors total_endpoints += endpoints - nose.tools.assert_dict_equal(stats['heartbeat'][proto], { + self.assertDictEqual(stats['heartbeat'][proto], { 'monitors': monitors, 'endpoints': endpoints, }) - nose.tools.assert_equal(stats['heartbeat']['monitors'], total_monitors) - nose.tools.assert_equal( + self.assertEqual(stats['heartbeat']['monitors'], total_monitors) + self.assertEqual( stats['heartbeat']['endpoints'], total_endpoints) - @staticmethod - def assert_stats(expected={}): + def assert_stats(self, expected={}): stats = json.loads(urllib.request.urlopen( "http://localhost:5066/stats").read()) for proto in ("http", "tcp", "icmp"): proto_expected = expected.get(proto, {}) - nose.tools.assert_dict_equal(stats['heartbeat'][proto], { + self.assertDictEqual(stats['heartbeat'][proto], { 'monitor_starts': proto_expected.get("monitor_starts", 0), 'monitor_stops': proto_expected.get("monitor_stops", 0), 'endpoint_starts': proto_expected.get("endpoint_starts", 0), diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 7aaeffad64f..e17c37bed34 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -81,7 +81,7 @@ REVIEWDOG_REPO?=github.com/reviewdog/reviewdog/cmd/reviewdog PROCESSES?= 4 TIMEOUT?= 90 PYTHON_TEST_FILES?=$(shell find . -type f -name 'test_*.py' -not -path "*/build/*" -not -path "*/vendor/*" 2>/dev/null) -NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling nosetests +PYTEST_OPTIONS?=--timeout=$(TIMEOUT) --durations=20 --junit-xml=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests @@ -234,7 +234,7 @@ integration-tests-environment: prepare-tests build-image .PHONY: system-tests system-tests: ## @testing Runs the system tests system-tests: prepare-tests ${BEAT_NAME}.test python-env - . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} nosetests ${PYTHON_TEST_FILES} ${NOSETESTS_OPTIONS} + . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} pytest ${PYTHON_TEST_FILES} ${PYTEST_OPTIONS} ${PYTHON_ENV_EXE} ${ES_BEATS}/dev-tools/aggregate_coverage.py -o ${COVERAGE_DIR}/system.cov ${BUILD_DIR}/system-tests/run # Runs the system tests @@ -252,7 +252,7 @@ system-tests-environment: prepare-tests build-image .PHONY: fast-system-tests fast-system-tests: ## @testing Runs system tests without coverage reports and in parallel fast-system-tests: ${BEAT_NAME}.test python-env - . ${PYTHON_ENV}/bin/activate; nosetests ${PYTHON_TEST_FILES} ${NOSETESTS_OPTIONS} + . ${PYTHON_ENV}/bin/activate; pytest ${PYTHON_TEST_FILES} ${PYTEST_OPTIONS} # Runs the go based stress tests .PHONY: stress-tests @@ -271,7 +271,7 @@ benchmark-tests: ## @testing Runs benchmarks (NOT YET IMPLEMENTED) # Run load tests .PHONY: load-tests load-tests: ## @testing Runs load tests - . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 nosetests ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a 'load' + . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 pytest ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --timeout=$(TIMEOUT) -a 'load' # Sets up the virtual python environment .PHONY: python-env diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 199f8b2c23a..1d52b2b8ec4 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -134,14 +134,11 @@ def setUpClass(self): if not hasattr(self, 'test_binary'): self.test_binary = os.path.abspath(self.beat_path + "/" + self.beat_name + ".test") - template_paths = [ - self.beat_path, - os.path.abspath(os.path.join(self.beat_path, "../libbeat")) - ] if not hasattr(self, 'template_paths'): - self.template_paths = template_paths - else: - self.template_paths.append(template_paths) + self.template_paths = [ + self.beat_path, + os.path.abspath(os.path.join(self.beat_path, "../libbeat")) + ] # Create build path build_dir = self.beat_path + "/build" diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 095533f7f89..864d134a9c9 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -1,7 +1,7 @@ -from elasticsearch import NotFoundError -from nose.tools import raises import datetime import unittest +import pytest +from elasticsearch import NotFoundError class IdxMgmt(unittest.TestCase): @@ -52,9 +52,9 @@ def delete_policy(self, policy): except NotFoundError: pass - @raises(NotFoundError) def assert_index_template_not_loaded(self, template): - self._client.transport.perform_request('GET', '/_template/' + template) + with pytest.raises(NotFoundError): + self._client.transport.perform_request('GET', '/_template/' + template) def assert_index_template_loaded(self, template): resp = self._client.transport.perform_request('GET', '/_template/' + template) @@ -86,9 +86,9 @@ def assert_alias_created(self, alias, pattern=None): assert name in resp assert resp[name]["aliases"][alias]["is_write_index"] == True - @raises(NotFoundError) def assert_policy_not_created(self, policy): - self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) + with pytest.raises(NotFoundError): + self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) def assert_policy_created(self, policy): resp = self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 2a030093b33..00eff11c5d9 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,8 +1,10 @@ +attrs==19.3.0 autopep8==1.3.5 backports.ssl-match-hostname==3.5.0.1 cached-property==1.4.2 certifi==2018.1.18 chardet==3.0.4 +deepdiff==4.2.0 docker==4.1.0 docker-compose==1.25.3 docker-pycreds==0.4.0 @@ -11,25 +13,35 @@ docopt==0.6.2 elasticsearch==7.8.1 enum34==1.1.6 idna==2.6 +importlib-metadata==1.7.0 +iniconfig==1.0.1 ipaddress==1.0.19 -Jinja2==2.10.1 -jsonschema==2.6.0 -MarkupSafe==1.0 -nose==1.3.7 -nose-timer==0.7.1 +Jinja2==2.11.2 +jsondiff==1.1.2 +jsonschema==3.2.0 +kafka-python==1.4.3 +MarkupSafe==1.1.1 +more-itertools==8.4.0 +ordered-set==3.1.1 +packaging==20.4 +parameterized==0.7.0 +pluggy==0.13.1 +py==1.9.0 pycodestyle==2.4.0 -PyYAML==4.2b1 +pyparsing==2.4.7 +pyrsistent==0.16.0 +pytest==6.0.1 +pytest-timeout==1.3.4 +PyYAML==5.3.1 redis==2.10.6 requests==2.20.0 -six==1.11.0 +semver==2.8.1 +six==1.15.0 +stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 +toml==0.10.1 urllib3==1.24.2 +wcwidth==0.2.5 websocket-client==0.47.0 -parameterized==0.7.0 -jsondiff==1.1.2 -semver==2.8.1 -stomp.py==4.1.22 -ordered-set==3.1.1 -deepdiff==4.2.0 -kafka-python==1.4.3 +zipp>=1.2.0,<=3.1.0 diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index 1e0dd6f6a43..4c1480b82a0 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -1,11 +1,10 @@ -from base import BaseTest -from idxmgmt import IdxMgmt +import logging import os -from nose.plugins.attrib import attr +import pytest import unittest -import logging -from nose.tools import raises +from base import BaseTest from elasticsearch import RequestError +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -16,7 +15,7 @@ class TestCAPinning(BaseTest): """ @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_sending_events_with_a_good_sha256(self): """ Test Sending events while using ca pinning with a good sha256 @@ -48,7 +47,7 @@ def test_sending_events_with_a_good_sha256(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_sending_events_with_a_bad_sha256(self): """ Test Sending events while using ca pinning with a bad sha256 diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 1ce764d640e..8a5b6100fad 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -1,11 +1,11 @@ -from base import BaseTest -from idxmgmt import IdxMgmt +import logging import os -from nose.plugins.attrib import attr +import pytest import unittest -import logging -from nose.tools import raises + +from base import BaseTest from elasticsearch import RequestError +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -47,7 +47,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_default(self): """ Test setup --index-management with default config @@ -64,8 +64,7 @@ def test_setup_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') - @raises(RequestError) + @pytest.mark.tag('integration') def test_setup_default(self): """ Test setup --index-management with default config @@ -80,10 +79,11 @@ def test_setup_default(self): self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) # try deleting policy needs to raise an error as it is in use - self.idxmgmt.delete_policy(self.policy_name) + with pytest.raises(RequestError): + self.idxmgmt.delete_policy(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_disabled(self): """ Test setup --index-management when ilm disabled @@ -99,7 +99,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_ilm_disabled(self): """ Test setup --index-management when ilm disabled @@ -115,7 +115,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_policy_name(self): """ Test setup --index-management when policy_name is configured @@ -130,7 +130,7 @@ def test_setup_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_ilm_policy_no_overwrite(self): """ Test setup --index-management respects overwrite configuration @@ -177,7 +177,7 @@ def test_setup_ilm_policy_no_overwrite(self): assert "hot" in resp[policy_name]["policy"]["phases"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_rollover_alias(self): """ Test setup --index-management when ilm.rollover_alias is configured @@ -193,7 +193,7 @@ def test_setup_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_rollover_alias_with_fieldref(self): """ Test setup --index-management when ilm.rollover_alias is configured and using field reference. @@ -212,7 +212,7 @@ def test_setup_rollover_alias_with_fieldref(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_name_and_pattern(self): """ Test setup --index-management ignores template.name and template.pattern when ilm is enabled @@ -229,7 +229,7 @@ def test_setup_template_name_and_pattern(self): self.idxmgmt.assert_alias_created(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_name_and_pattern_on_ilm_disabled(self): """ Test setup --index-management respects template.name and template.pattern when ilm is disabled @@ -248,7 +248,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_with_opts(self): """ Test setup --index-management with config options @@ -270,7 +270,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_overwrite_template_on_ilm_policy_created(self): """ Test setup --index-management overwrites template when new ilm policy is created diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index 77b2d8f4b64..38f15ef095f 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -1,8 +1,8 @@ -from base import BaseTest import os import logging import unittest -from nose.plugins.attrib import attr +import pytest +from base import BaseTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -41,7 +41,7 @@ def test_bad_config(self): assert self.log_contains("Config OK") is False @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_output(self): """ Test test output works diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 2ed79fd2743..a7b3145c1e7 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -1,5 +1,6 @@ import os import os.path +import pytest import re import requests import semver @@ -7,8 +8,6 @@ import unittest from base import BaseTest -from nose.plugins.attrib import attr -from unittest import SkipTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -16,7 +15,7 @@ class Test(BaseTest): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_load_without_dashboard(self): """ Test loading without dashboards @@ -41,7 +40,7 @@ def test_load_without_dashboard(self): assert self.log_contains("Skipping loading dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_load_dashboard(self): """ Test loading dashboards @@ -66,7 +65,7 @@ def test_load_dashboard(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_load_dashboard_into_space(self, create_space=True): """ Test loading dashboards into Kibana space @@ -74,7 +73,7 @@ def test_load_dashboard_into_space(self, create_space=True): version = self.get_version() if semver.compare(version, "6.5.0") == -1: # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available - raise SkipTest + raise unittest.SkipTest self.render_config_template() if create_space: @@ -100,7 +99,7 @@ def test_load_dashboard_into_space(self, create_space=True): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_load_only_index_patterns(self): """ Test loading dashboards @@ -126,7 +125,7 @@ def test_load_only_index_patterns(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): """ Test testbeat export dashboard can export dashboards @@ -150,7 +149,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id(self): """ Test testbeat export dashboard can export dashboards @@ -172,7 +171,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): """ Test testbeat export dashboard fails gracefully when dashboard with unknown ID is requested @@ -194,7 +193,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): assert self.log_contains(expected_error) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_from_yml(self): """ Test testbeat export dashboard can export dashboards from dashboards YAML file @@ -227,7 +226,7 @@ def test_export_dashboard_cmd_export_dashboard_from_yml(self): os.remove(exported_dashboard_path) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): """ Test testbeat export dashboard fails gracefully when cannot find YAML file @@ -249,7 +248,7 @@ def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): assert self.log_contains("error opening the list of dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id(self): """ Test dev-tools/cmd/dashboards exports dashboard and removes unsupported characters @@ -275,7 +274,7 @@ def test_dev_tool_export_dashboard_by_id(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id_unknown_id(self): """ Test dev-tools/cmd/dashboards fails gracefully when dashboard with unknown ID is requested @@ -291,7 +290,7 @@ def test_dev_tool_export_dashboard_by_id_unknown_id(self): assert p.returncode != 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id_from_space(self): """ Test dev-tools/cmd/dashboards exports dashboard from Kibana space @@ -300,7 +299,7 @@ def test_dev_tool_export_dashboard_by_id_from_space(self): version = self.get_version() if semver.compare(version, "6.5.0") == -1: # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available - raise SkipTest + raise unittest.SkipTest self.test_load_dashboard_into_space(False) @@ -322,7 +321,7 @@ def test_dev_tool_export_dashboard_by_id_from_space(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_from_yml(self): """ Test dev-tools/cmd/dashboards exports dashboard from dashboards YAML file diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 293544f8dd6..3d37125f6e4 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -1,12 +1,13 @@ -from base import BaseTest -from idxmgmt import IdxMgmt -import os -from nose.plugins.attrib import attr -import unittest -import shutil import datetime -import logging import json +import logging +import os +import pytest +import shutil +import unittest + +from base import BaseTest +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -37,7 +38,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_ilm_default(self): """ Test ilm default settings to load ilm policy, write alias and ilm template @@ -55,7 +56,7 @@ def test_ilm_default(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_ilm_disabled(self): """ Test ilm disabled to not load ilm related components @@ -72,7 +73,7 @@ def test_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_policy_name(self): """ Test setting ilm policy name @@ -92,7 +93,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_rollover_alias(self): """ Test settings ilm rollover alias @@ -111,7 +112,7 @@ def test_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_pattern(self): """ Test setting ilm pattern @@ -131,7 +132,7 @@ def test_pattern(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_pattern_date(self): """ Test setting ilm pattern with date @@ -187,7 +188,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_ilm_policy_and_template(self): """ Test combination of ilm policy and template setup @@ -204,7 +205,7 @@ def test_setup_ilm_policy_and_template(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_ilm_default(self): """ Test ilm policy setup with default config @@ -221,7 +222,7 @@ def test_setup_ilm_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_ilm_disabled(self): """ Test ilm policy setup when ilm disabled @@ -238,7 +239,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_policy_name(self): """ Test ilm policy setup when policy_name is configured @@ -254,7 +255,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_rollover_alias(self): """ Test ilm policy setup when rollover_alias is configured diff --git a/libbeat/tests/system/test_migration.py b/libbeat/tests/system/test_migration.py index fec98ef39f7..81ab3b679dd 100644 --- a/libbeat/tests/system/test_migration.py +++ b/libbeat/tests/system/test_migration.py @@ -1,11 +1,9 @@ -from base import BaseTest -from nose.plugins.attrib import attr -from elasticsearch import Elasticsearch, TransportError - import logging import os import shutil import unittest +from base import BaseTest +from elasticsearch import Elasticsearch, TransportError INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index b339ac2bd94..61c1c62c29e 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -1,12 +1,13 @@ -from base import BaseTest import os -from elasticsearch import Elasticsearch +import pytest +import random import re -from nose.plugins.attrib import attr -import unittest import requests -import random import string +import unittest + +from base import BaseTest +from elasticsearch import Elasticsearch INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -21,7 +22,7 @@ def setUp(self): self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_via_output_cluster(self): """ Test shipping monitoring data via the elasticsearch output cluster. @@ -58,7 +59,7 @@ def test_via_output_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_direct_to_monitoring_cluster(self): """ Test shipping monitoring data directly to the monitoring cluster. @@ -92,7 +93,7 @@ def test_direct_to_monitoring_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_compare(self): """ Test that monitoring docs are the same, regardless of how they are shipped. @@ -155,7 +156,7 @@ def test_compare(self): self.assert_same_structure(indirect_beats_stats_doc['beats_stats'], direct_beats_stats_doc['beats_stats']) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_cluster_uuid_setting(self): """ Test that monitoring.cluster_uuid setting may be set without any other monitoring.* settings @@ -178,7 +179,7 @@ def test_cluster_uuid_setting(self): self.assertEqual(test_cluster_uuid, state["monitoring"]["cluster_uuid"]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_cluster_uuid_setting_monitoring_disabled(self): """ Test that monitoring.cluster_uuid setting may be set with monitoring.enabled explicitly set to false diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index c1c4f5ea32c..67a34457484 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -1,11 +1,12 @@ -from base import BaseTest -from idxmgmt import IdxMgmt +import json +import logging import os -from nose.plugins.attrib import attr -import unittest +import pytest import shutil -import logging -import json +import unittest + +from base import BaseTest +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -83,7 +84,7 @@ def test_index_with_pattern_name(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_json_template(self): """ Test loading of json based template @@ -140,7 +141,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_template_default(self): """ Test run cmd with default settings for template @@ -157,7 +158,7 @@ def test_template_default(self): self.idxmgmt.assert_docs_written_to_alias(self.index_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_template_disabled(self): """ Test run cmd does not load template when disabled in config @@ -201,7 +202,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup(self): """ Test setup cmd with template and ilm-policy subcommands @@ -216,7 +217,7 @@ def test_setup(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_default(self): """ Test template setup with default config @@ -235,7 +236,7 @@ def test_setup_template_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_disabled(self): """ Test template setup when ilm disabled @@ -254,7 +255,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_with_opts(self): """ Test template setup with config options @@ -275,7 +276,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_setup_template_with_ilm_changed_pattern(self): """ Test template setup with changed ilm.rollover_alias config @@ -290,7 +291,7 @@ def test_setup_template_with_ilm_changed_pattern(self): self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_template_created_on_ilm_policy_created(self): """ Test template setup overwrites template when new ilm policy is created diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 84474e39fe5..661239410b2 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -191,19 +191,18 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). // Use MODULE=module to run only tests for `module`. -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } - runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("NOSE_")...) + runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("PYTEST_")...) if err != nil { return err } return runner.Test("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTestForModule(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTestForModule(devtools.DefaultPythonTestIntegrationArgs()) }) } diff --git a/metricbeat/module/apache/test_apache.py b/metricbeat/module/apache/test_apache.py index f47d046bdae..396c60e3847 100644 --- a/metricbeat/module/apache/test_apache.py +++ b/metricbeat/module/apache/test_apache.py @@ -1,9 +1,9 @@ import os import unittest -from nose.plugins.attrib import attr import urllib.request import urllib.error import urllib.parse +import pytest import time import semver import sys @@ -39,7 +39,7 @@ class ApacheStatusTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['apache'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_output(self): """ Apache module outputs an event. diff --git a/metricbeat/module/consul/test_consul.py b/metricbeat/module/consul/test_consul.py index 5ee0f5ac2a8..2702c144c3a 100644 --- a/metricbeat/module/consul/test_consul.py +++ b/metricbeat/module/consul/test_consul.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -27,7 +27,7 @@ class ConsulAgentTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['consul'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_output(self): """ Consul agent module outputs an event. diff --git a/metricbeat/module/docker/test_docker.py b/metricbeat/module/docker/test_docker.py index 22a86784e02..8eacc289358 100644 --- a/metricbeat/module/docker/test_docker.py +++ b/metricbeat/module/docker/test_docker.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index 88e064c6d11..2ccf3b0fddd 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -2,14 +2,13 @@ import sys import os import unittest -from elasticsearch import Elasticsearch, TransportError, client -from parameterized import parameterized -from nose.plugins.skip import SkipTest import urllib.request import urllib.error import urllib.parse import json import semver +from elasticsearch import Elasticsearch, TransportError, client +from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) @@ -317,10 +316,12 @@ def start_basic(self): def check_skip(self, metricset): if metricset == 'ccr' and not self.is_ccr_available(): - raise SkipTest("elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") + raise unittest.SkipTest( + "elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") if metricset == 'enrich' and not self.is_enrich_available(): - raise SkipTest("elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") + raise unittest.SkipTest( + "elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") def is_ccr_available(self): es_version = self.get_version() diff --git a/metricbeat/module/haproxy/test_haproxy.py b/metricbeat/module/haproxy/test_haproxy.py index 40ab58f847f..39ec3cfe6d9 100644 --- a/metricbeat/module/haproxy/test_haproxy.py +++ b/metricbeat/module/haproxy/test_haproxy.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/http/test_http.py b/metricbeat/module/http/test_http.py index ce9e1c81b27..92aa693552b 100644 --- a/metricbeat/module/http/test_http.py +++ b/metricbeat/module/http/test_http.py @@ -3,7 +3,6 @@ import sys import time import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/jolokia/test_jolokia.py b/metricbeat/module/jolokia/test_jolokia.py index 509b9f5681e..db6055fe91e 100644 --- a/metricbeat/module/jolokia/test_jolokia.py +++ b/metricbeat/module/jolokia/test_jolokia.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) diff --git a/metricbeat/module/kibana/test_kibana.py b/metricbeat/module/kibana/test_kibana.py index 08e7657a1e8..cbd4f365979 100644 --- a/metricbeat/module/kibana/test_kibana.py +++ b/metricbeat/module/kibana/test_kibana.py @@ -6,7 +6,6 @@ import urllib.error import urllib.parse import urllib.request -from nose.plugins.skip import SkipTest sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -27,12 +26,12 @@ def test_status(self): if env == "2x" or env == "5x": # Skip for 5.x and 2.x tests as Kibana endpoint not available - raise SkipTest + raise unittest.SkipTest version = self.get_version() if semver.compare(version, "6.4.0") == -1: # Skip for Kibana versions < 6.4.0 as Kibana endpoint not available - raise SkipTest + raise unittest.SkipTest self.render_config_template(modules=[{ "name": "kibana", diff --git a/metricbeat/module/logstash/test_logstash.py b/metricbeat/module/logstash/test_logstash.py index 4d5f7c233fc..b526515fdf0 100644 --- a/metricbeat/module/logstash/test_logstash.py +++ b/metricbeat/module/logstash/test_logstash.py @@ -7,7 +7,6 @@ import urllib.error import urllib.parse import urllib.request -from nose.plugins.skip import SkipTest sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -41,7 +40,7 @@ def test_xpack(self): version = self.get_version() if semver.compare(version, "7.3.0") == -1: # Skip for Logstash versions < 7.3.0 as necessary APIs not available - raise SkipTest + raise unittest.SkipTest self.render_config_template(modules=[{ "name": "logstash", diff --git a/metricbeat/module/mongodb/test_mongodb.py b/metricbeat/module/mongodb/test_mongodb.py index c62801b9853..bd53edd1060 100644 --- a/metricbeat/module/mongodb/test_mongodb.py +++ b/metricbeat/module/mongodb/test_mongodb.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -15,7 +15,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mongodb'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_status(self): """ MongoDB module outputs an event. diff --git a/metricbeat/module/mysql/test_mysql.py b/metricbeat/module/mysql/test_mysql.py index 2422422a685..4a8b34728ed 100644 --- a/metricbeat/module/mysql/test_mysql.py +++ b/metricbeat/module/mysql/test_mysql.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -18,7 +18,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mysql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_status(self): """ MySQL module outputs an event. diff --git a/metricbeat/module/postgresql/test_postgresql.py b/metricbeat/module/postgresql/test_postgresql.py index 8dded22cd32..a8d286d5685 100644 --- a/metricbeat/module/postgresql/test_postgresql.py +++ b/metricbeat/module/postgresql/test_postgresql.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -32,7 +32,7 @@ def get_hosts(self): ) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_activity(self): """ PostgreSQL module outputs an event. @@ -59,7 +59,7 @@ def test_activity(self): assert "state" in evt["postgresql"]["activity"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_database(self): """ PostgreSQL module outputs an event. @@ -89,7 +89,7 @@ def test_database(self): assert "deadlocks" in evt["postgresql"]["database"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_bgwriter(self): """ PostgreSQL module outputs an event. diff --git a/metricbeat/module/redis/test_redis.py b/metricbeat/module/redis/test_redis.py index 98ffb982ca3..1f6082b5626 100644 --- a/metricbeat/module/redis/test_redis.py +++ b/metricbeat/module/redis/test_redis.py @@ -1,8 +1,8 @@ import os +import pytest import redis import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -29,7 +29,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['redis'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_info(self): """ Test redis info metricset @@ -59,7 +59,7 @@ def test_info(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_keyspace(self): """ Test redis keyspace metricset @@ -95,7 +95,7 @@ def test_keyspace(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_key(self): """ Test redis key metricset @@ -133,7 +133,7 @@ def test_key(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_module_processors(self): """ Test local processors for Redis info event. diff --git a/metricbeat/module/uwsgi/test_uwsgi.py b/metricbeat/module/uwsgi/test_uwsgi.py index 6f58c0d6911..1659a7e3e4f 100644 --- a/metricbeat/module/uwsgi/test_uwsgi.py +++ b/metricbeat/module/uwsgi/test_uwsgi.py @@ -1,8 +1,8 @@ import logging import os +import pytest import sys import unittest -from nose.plugins.attrib import attr from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) @@ -56,7 +56,7 @@ def common_checks(self, output): @parameterized.expand(["http", "tcp"]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_status(self, proto): """ uWSGI module outputs an event. diff --git a/metricbeat/module/zookeeper/test_zookeeper.py b/metricbeat/module/zookeeper/test_zookeeper.py index 65db80bf48b..99184119cea 100644 --- a/metricbeat/module/zookeeper/test_zookeeper.py +++ b/metricbeat/module/zookeeper/test_zookeeper.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -21,7 +21,7 @@ class ZooKeeperMntrTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['zookeeper'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_output(self): """ ZooKeeper mntr module outputs an event. @@ -55,7 +55,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_output(self): """ ZooKeeper server module outputs an event. @@ -83,7 +83,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_connection(self): """ ZooKeeper server module outputs an event. diff --git a/metricbeat/tests/system/test_autodiscover_jolokia.py b/metricbeat/tests/system/test_autodiscover_jolokia.py index 727e4e62807..51d5dddd5a8 100644 --- a/metricbeat/tests/system/test_autodiscover_jolokia.py +++ b/metricbeat/tests/system/test_autodiscover_jolokia.py @@ -1,7 +1,6 @@ import os import metricbeat import unittest -from nose.plugins.attrib import attr class Test(metricbeat.BaseTest): diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index 4f680d29172..c54be3244eb 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -1,11 +1,14 @@ +import os +import pytest import re +import shutil import sys import unittest -import os -import shutil + from metricbeat import BaseTest -from elasticsearch import Elasticsearch + from beat.beat import INTEGRATION_TESTS +from elasticsearch import Elasticsearch class Test(BaseTest): @@ -53,6 +56,7 @@ def test_template(self): assert len(es.cat.templates(name='metricbeat-*', h='name')) > 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.timeout(180, func_only=True) def test_dashboards(self): """ Test that the dashboards can be loaded with `setup --dashboards` diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py index d1cd2702265..a09a23f5ede 100644 --- a/metricbeat/tests/system/test_config.py +++ b/metricbeat/tests/system/test_config.py @@ -1,7 +1,6 @@ import os from metricbeat import BaseTest import unittest -from nose.plugins.attrib import attr import urllib.request import urllib.error import urllib.parse diff --git a/metricbeat/tests/system/test_template.py b/metricbeat/tests/system/test_template.py index 5899796524d..75b366ab15d 100644 --- a/metricbeat/tests/system/test_template.py +++ b/metricbeat/tests/system/test_template.py @@ -1,7 +1,7 @@ import os import metricbeat import json -from nose.plugins.skip import SkipTest +import unittest class Test(metricbeat.BaseTest): @@ -12,7 +12,7 @@ def test_export_template(self): """ if os.name == "nt": - raise SkipTest + raise unittest.SkipTest self.render_config_template("metricbeat", os.path.join(self.working_dir, diff --git a/packetbeat/SUPPORT_PROTOCOL.md b/packetbeat/SUPPORT_PROTOCOL.md index 035eb793b85..5bc304f6745 100644 --- a/packetbeat/SUPPORT_PROTOCOL.md +++ b/packetbeat/SUPPORT_PROTOCOL.md @@ -16,7 +16,7 @@ Test suites are based on [pcap files](./tests/system/pcaps), that are dumps from tcpdump -s 0 port 27017 -i docker0 -w tests/system/pcaps/mongodb_find.pcap -# Nosetests +# Python tests The 'tests' directory contains tests written in python that run the full packetbeat program. You can add some of yours based on the pcaps files of you test dataset. diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 2f373603817..f82836c1342 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -10482,7 +10482,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- @@ -10493,7 +10493,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- @@ -10754,7 +10754,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- @@ -10765,7 +10765,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- diff --git a/packetbeat/tests/system/README.md b/packetbeat/tests/system/README.md index 370ba15b4e5..05f8e479bbb 100644 --- a/packetbeat/tests/system/README.md +++ b/packetbeat/tests/system/README.md @@ -1,7 +1,7 @@ # System tests for Packetbeat This folder contains the system tests for Packetbeat. The system tests -are written in Python and they make use of the nose framework. +are written in Python and they make use of the pytest framework. ## Running @@ -13,4 +13,4 @@ the setup and run all the tests with: Running a single test, e.g.: . env/bin/activate - nosetests test_0002_thrift_basics.py:Test.test_thrift_integration + pytest test_0002_thrift_basics.py:Test.test_thrift_integration diff --git a/packetbeat/tests/system/test_0015_udpjson.py b/packetbeat/tests/system/test_0015_udpjson.py index 454dd3a3523..06e38f49990 100644 --- a/packetbeat/tests/system/test_0015_udpjson.py +++ b/packetbeat/tests/system/test_0015_udpjson.py @@ -1,11 +1,11 @@ -from packetbeat import BaseTest -from nose.tools import nottest import socket +import unittest +from packetbeat import BaseTest +@unittest.skip("udpjson not supported anymore") class Test(BaseTest): - @nottest def test_udpjson_config(self): """ Should start with sniffer and udpjson inputs configured. @@ -21,7 +21,6 @@ def test_udpjson_config(self): assert all([o["type"] == "mysql" for o in objs]) assert len(objs) == 7 - @nottest def test_only_udpjson_config(self): """ It should be possible to start without the sniffer configured. @@ -39,7 +38,6 @@ def test_only_udpjson_config(self): packetbeat.kill_and_wait() - @nottest def test_send_udpjson_msg(self): """ It should be possible to send a UDP message and read it from diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..cc816886a6e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,10 @@ +[pytest] +junit_family=xunit1 + +addopts = --strict-markers +markers = + load: Load tests + tag(name): Tag tests with Go-like semantics + +# Ignore setup and teardown for the timeout +timeout_func_only = True diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index 691977b89eb..456bd382e1c 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -165,7 +165,7 @@ func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.Deps(Fields) } - runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "NOSE_"), "GENERATE")...) + runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "PYTEST_"), "GENERATE")...) if err != nil { return err } @@ -173,6 +173,6 @@ func PythonIntegTest(ctx context.Context) error { mg.Deps(devtools.BuildSystemTestBinary) args := devtools.DefaultPythonTestIntegrationArgs() args.Env["MODULES_PATH"] = devtools.CWD("module") - return devtools.PythonNoseTest(args) + return devtools.PythonTest(args) }) } diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index 2aa9317dc25..8161c16b9b8 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -146,19 +146,18 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). // Use MODULE=module to run only tests for `module`. -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } - runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("NOSE_")...) + runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("PYTEST_")...) if err != nil { return err } return runner.Test("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTestForModule(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs()) }) } diff --git a/x-pack/metricbeat/module/mssql/test_mssql.py b/x-pack/metricbeat/module/mssql/test_mssql.py index fe839cb8969..225dde74fb2 100644 --- a/x-pack/metricbeat/module/mssql/test_mssql.py +++ b/x-pack/metricbeat/module/mssql/test_mssql.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) from xpack_metricbeat import XPackTest, metricbeat @@ -18,7 +18,7 @@ class Test(XPackTest): COMPOSE_SERVICES = ['mssql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_status(self): """ MSSQL module outputs an event. @@ -47,7 +47,7 @@ def test_status(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.tag('integration') def test_performance(self): """ MSSQL module outputs an event.