From 279c24059f110bf0924cf10b20e43328afd6affc Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Thu, 22 Jul 2021 09:32:43 -0500 Subject: [PATCH] [UnitTests] Expose TVM pytest helpers as plugin Previously, pytest helper utilities such as automatic parametrization of `target`/`dev`, or `tvm.testing.parameter` were only available for tests within the `${TVM_HOME}/tests` directory. This PR extracts the helper utilities into an importable plugin, which can be used in external tests (e.g. one-off debugging). --- conftest.py | 33 +------- pytest.ini => python/tvm/testing/__init__.py | 13 +-- python/tvm/testing/plugin.py | 82 +++++++++++++++++++ python/tvm/{ => testing}/testing.py | 0 .../unittest/test_tvm_testing_features.py | 2 +- 5 files changed, 87 insertions(+), 43 deletions(-) rename pytest.ini => python/tvm/testing/__init__.py (68%) create mode 100644 python/tvm/testing/plugin.py rename python/tvm/{ => testing}/testing.py (100%) diff --git a/conftest.py b/conftest.py index f591fe970de88..28859fd4a17ba 100644 --- a/conftest.py +++ b/conftest.py @@ -14,36 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import pytest -from pytest import ExitCode -import tvm -import tvm.testing - - -def pytest_configure(config): - print("enabled targets:", "; ".join(map(lambda x: x[0], tvm.testing.enabled_targets()))) - print("pytest marker:", config.option.markexpr) - - -@pytest.fixture -def dev(target): - return tvm.device(target) - - -def pytest_generate_tests(metafunc): - tvm.testing._auto_parametrize_target(metafunc) - tvm.testing._parametrize_correlated_parameters(metafunc) - - -def pytest_collection_modifyitems(config, items): - tvm.testing._count_num_fixture_uses(items) - tvm.testing._remove_global_fixture_definitions(items) - - -def pytest_sessionfinish(session, exitstatus): - # Don't exit with an error if we select a subset of tests that doesn't - # include anything - if session.config.option.markexpr != "": - if exitstatus == ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = ExitCode.OK +pytest_plugins = ["tvm.testing.plugin"] diff --git a/pytest.ini b/python/tvm/testing/__init__.py similarity index 68% rename from pytest.ini rename to python/tvm/testing/__init__.py index 675f8fe9b5a00..7e24c4309908d 100644 --- a/pytest.ini +++ b/python/tvm/testing/__init__.py @@ -14,13 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -[pytest] -markers = - gpu: mark a test as requiring a gpu - tensorcore: mark a test as requiring a tensorcore - cuda: mark a test as requiring cuda - opencl: mark a test as requiring opencl - rocm: mark a test as requiring rocm - vulkan: mark a test as requiring vulkan - metal: mark a test as requiring metal - llvm: mark a test as requiring llvm +"""Namespace for tvm testing utilities""" + +from .testing import * diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py new file mode 100644 index 0000000000000..12a8949eae3dd --- /dev/null +++ b/python/tvm/testing/plugin.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Pytest plugin for using tvm testing extensions. + +TVM provides utilities for testing across all supported targets, and +to more easily parametrize across many inputs. For more information +on usage of these features, see documentation in the tvm.testing +module. + +These are enabled by default in all pytests provided by tvm, but may +be useful externally for one-off testing. To enable, add the +following line to the test script, or to the conftest.py in the same +directory as the test scripts. + + pytest_plugins = ['tvm.testing.plugin'] + +""" + +import pytest + +import tvm.testing.testing + + +def pytest_configure(config): + """Runs at pytest configure time, defines marks to be used later.""" + markers = { + "gpu": "mark a test as requiring a gpu", + "tensorcore": "mark a test as requiring a tensorcore", + "cuda": "mark a test as requiring cuda", + "opencl": "mark a test as requiring opencl", + "rocm": "mark a test as requiring rocm", + "vulkan": "mark a test as requiring vulkan", + "metal": "mark a test as requiring metal", + "llvm": "mark a test as requiring llvm", + } + for markername, desc in markers.items(): + config.addinivalue_line("markers", "{}: {}".format(markername, desc)) + + print("enabled targets:", "; ".join(map(lambda x: x[0], tvm.testing.enabled_targets()))) + print("pytest marker:", config.option.markexpr) + + +def pytest_generate_tests(metafunc): + """Called once per unit test, modifies/parametrizes it as needed.""" + tvm.testing.testing._auto_parametrize_target(metafunc) + tvm.testing.testing._parametrize_correlated_parameters(metafunc) + + +def pytest_collection_modifyitems(config, items): + """Called after all tests are chosen, currently used for bookkeeping.""" + # pylint: disable=unused-argument + tvm.testing.testing._count_num_fixture_uses(items) + tvm.testing.testing._remove_global_fixture_definitions(items) + + +@pytest.fixture +def dev(target): + """Give access to the device to tests that need it.""" + return tvm.device(target) + + +def pytest_sessionfinish(session, exitstatus): + # Don't exit with an error if we select a subset of tests that doesn't + # include anything + if session.config.option.markexpr != "": + if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK diff --git a/python/tvm/testing.py b/python/tvm/testing/testing.py similarity index 100% rename from python/tvm/testing.py rename to python/tvm/testing/testing.py diff --git a/tests/python/unittest/test_tvm_testing_features.py b/tests/python/unittest/test_tvm_testing_features.py index 4b699096e96ae..944db58fb6e1f 100644 --- a/tests/python/unittest/test_tvm_testing_features.py +++ b/tests/python/unittest/test_tvm_testing_features.py @@ -183,7 +183,7 @@ def test_num_uses_cached(self): class TestAutomaticMarks: @staticmethod def check_marks(request, target): - parameter = tvm.testing._pytest_target_params([target])[0] + parameter = tvm.testing.testing._pytest_target_params([target])[0] required_marks = [decorator.mark for decorator in parameter.marks] applied_marks = list(request.node.iter_markers())