Add option --order-scope-level to set the scope at directory level

- defines a scope based on directory depth, between session and module scope
- closes pytest-dev#8

CHANGELOG.md

@@ -5,6 +5,9 @@
 ### New features
 - added support for class level relative markers,
   see [#7](https://github.com/pytest-dev/pytest-order/issues/7)
+- added option `--order-scope-level` which allows to groups tests on the
+  same directory level, 
+  see [#8](https://github.com/pytest-dev/pytest-order/issues/8)
 
 ### Fixes
 - fixed sorting of dependency markers that depend on an item with the same

docs/source/index.rst

@@ -556,6 +556,120 @@ Here is what you get using session and module-scoped sorting:
     tests/test_module2.py:5: test2 PASSED
 
 
+``--order-scope-level``
+-----------------------
+This is an alternative option to define the order scope. It defines the
+directory level which is used as the order scope, counting from the root
+directory. The resulting scope is between the session and module
+scopes defined via ``--order-scope``, where ``--order-scope-level=0`` is the
+same as session scope, while setting the level to the number of test
+directory levels would result in module scope.
+
+Consider the following directory structure:
+
+::
+
+  order_scope_level
+    feature1
+      __init__.py
+      test_a.py
+      test_b.py
+    feature2
+       __init__.py
+       test_a.py
+       test_b.py
+
+with the test contents:
+
+**test_a.py**:
+
+.. code::
+
+  import pytest
+
+  @pytest.mark.order(4)
+  def test_four():
+      pass
+
+  @pytest.mark.order(3)
+  def test_three():
+      pass
+
+**test_b.py**:
+
+.. code::
+
+  import pytest
+
+  @pytest.mark.order(2)
+  def test_two():
+      pass
+
+  @pytest.mark.order(1)
+  def test_one():
+      pass
+
+The idea here is to test each feature separately, while ordering the tests
+across the test modules for each feature.
+
+If we use session scope, we get:
+
+::
+
+    $ pytest -v order_scope_level
+    ============================= test session starts ==============================
+    ...
+
+    order_scope_level/feature1/test_a.py::test_one PASSED
+    order_scope_level/feature2/test_a.py::test_one PASSED
+    order_scope_level/feature1/test_a.py::test_two PASSED
+    order_scope_level/feature2/test_a.py::test_two PASSED
+    order_scope_level/feature1/test_b.py::test_three PASSED
+    order_scope_level/feature2/test_b.py::test_three PASSED
+    order_scope_level/feature1/test_b.py::test_four PASSED
+    order_scope_level/feature2/test_b.py::test_four PASSED
+
+which mixes the features.
+
+Using module scope instead separates the features, but does not order the
+modules as wanted:
+
+::
+
+    $ pytest -v --order-scope=module order_scope_level
+    ============================= test session starts ==============================
+    ...
+
+    order_scope_level/feature1/test_a.py::test_three PASSED
+    order_scope_level/feature1/test_a.py::test_four PASSED
+    order_scope_level/feature1/test_b.py::test_one PASSED
+    order_scope_level/feature1/test_b.py::test_two PASSED
+    order_scope_level/feature2/test_a.py::test_three PASSED
+    order_scope_level/feature2/test_a.py::test_four PASSED
+    order_scope_level/feature2/test_b.py::test_one PASSED
+    order_scope_level/feature2/test_b.py::test_two PASSED
+
+To get the wanted behavior, we can use ``--order-scope-level=2``, which keeps
+the first two directory levels:
+
+::
+
+    $ pytest tests -v --order-scope-level=2 order_scope_level
+    ============================= test session starts ==============================
+    ...
+
+    order_scope_level/feature1/test_b.py::test_one PASSED
+    order_scope_level/feature1/test_b.py::test_two PASSED
+    order_scope_level/feature1/test_a.py::test_three PASSED
+    order_scope_level/feature1/test_a.py::test_four PASSED
+    order_scope_level/feature2/test_b.py::test_one PASSED
+    order_scope_level/feature2/test_b.py::test_two PASSED
+    order_scope_level/feature2/test_a.py::test_three PASSED
+    order_scope_level/feature2/test_a.py::test_four PASSED
+
+Note that using a level of 0 or 1 would cause the same result as session
+scope, and any level greater than 2 would emulate module scope.
+
 ``--order-group-scope``
 -----------------------
 This option is also related to the order scope. It defines the scope inside

example/order_scope_level/feature1/test_a.py

@@ -0,0 +1,11 @@
+import pytest
+
+
+@pytest.mark.order(2)
+def test_two():
+    pass
+
+
+@pytest.mark.order(1)
+def test_one():
+    pass

example/order_scope_level/feature1/test_b.py

@@ -0,0 +1,11 @@
+import pytest
+
+
+@pytest.mark.order(4)
+def test_four():
+    pass
+
+
+@pytest.mark.order(3)
+def test_three():
+    pass

example/order_scope_level/feature2/test_a.py

@@ -0,0 +1,11 @@
+import pytest
+
+
+@pytest.mark.order(2)
+def test_two():
+    pass
+
+
+@pytest.mark.order(1)
+def test_one():
+    pass

example/order_scope_level/feature2/test_b.py

@@ -0,0 +1,11 @@
+import pytest
+
+
+@pytest.mark.order(4)
+def test_four():
+    pass
+
+
+@pytest.mark.order(3)
+def test_three():
+    pass

pytest_order/__init__.py

@@ -51,6 +51,13 @@ def pytest_addoption(parser):
                     help="Defines the scope used for ordering. Possible values"
                          "are 'session' (default), 'module', and 'class'."
                          "Ordering is only done inside a scope.")
+    group.addoption("--order-scope-level", action="store", type=int,
+                    dest="order_scope_level",
+                    help="Defines that the given directory level is used as "
+                         "order scope. Cannot be used with --order-scope. "
+                         "The value is a number that defines the "
+                         "hierarchical index of the directories used as "
+                         "order scope, starting with 0 at session scope.")
     group.addoption("--order-group-scope", action="store",
                     dest="order_group_scope",
                     help="Defines the scope used for order groups. Possible "

pytest_order/sorter.py

@@ -46,6 +46,12 @@ def __init__(self, config):
                      "Valid scopes are 'session', 'module' and 'class'."
                      .format(scope))
             self.scope = SESSION
+        scope_level = config.getoption("order_scope_level") or 0
+        if scope_level != 0 and self.scope != SESSION:
+            warn("order-scope-level cannot be used together with "
+                 "--order-scope={}".format(scope))
+            scope_level = 0
+        self.scope_level = scope_level
         group_scope = config.getoption("order_group_scope")
         if group_scope in self.valid_scopes:
             self.group_scope = self.valid_scopes[group_scope]
@@ -69,7 +75,16 @@ def __init__(self, config, items):
 
     def sort_items(self):
         if self.settings.scope == SESSION:
-            sorted_list = ScopeSorter(self.settings, self.items).sort_items()
+            if self.settings.scope_level > 0:
+                dir_groups = directory_item_groups(
+                    self.items, self.settings.scope_level)
+                sorted_list = []
+                for items in dir_groups.values():
+                    sorted_list.extend(
+                        ScopeSorter(self.settings, items).sort_items())
+            else:
+                sorted_list = ScopeSorter(
+                    self.settings, self.items).sort_items()
         elif self.settings.scope == MODULE:
             module_groups = module_item_groups(self.items)
             sorted_list = []
@@ -92,6 +107,13 @@ def module_item_groups(items):
     return module_items
 
 
+def directory_item_groups(items, level):
+    module_items = OrderedDict()
+    for item in items:
+        module_items.setdefault(item.parent_path(level), []).append(item)
+    return module_items
+
+
 def class_item_groups(items):
     class_items = OrderedDict()
     for item in items:
@@ -387,6 +409,9 @@ def __init__(self, item):
     def module_path(self):
         return self.item.nodeid[:self.node_id.index("::")]
 
+    def parent_path(self, level):
+        return "/".join(self.module_path.split("/")[:level])
+
     @property
     def node_id(self):
         if self._node_id is None:

tests/conftest.py

@@ -1,10 +1,8 @@
 import os
 import shutil
 import uuid
-from random import randint
 
 import pytest
-
 from pytest_order.sorter import SESSION
 from tests.utils import write_test
 
@@ -43,6 +41,7 @@ def ignore_settings(mocker):
     settings.return_value.order_dependencies = False
     settings.return_value.scope = SESSION
     settings.return_value.group_scope = SESSION
+    settings.return_value.scope_level = 0
     yield settings
 
 
@@ -76,9 +75,3 @@ def test_node(nodeid):
 
     yield fixture_path
     shutil.rmtree(fixture_path, ignore_errors=True)
-
-
-def pytest_collection_modifyitems(config, items):
-    for item in items:
-        if item.name.startswith("test_performance"):
-            item.add_marker(pytest.mark.order(randint(-100, 100)))