Merge pull request #5 from jbasko/dev

Dev

Author: jbasko

README.rst

@@ -7,21 +7,29 @@ pytest-random-order
 This is a pytest plugin **to randomise the order** in which tests are run **with some control**
 over how much randomness one allows.
 
-It is a good idea to randomise the order in which your tests run
-because a test running early as part of a larger suite of tests may have left
+Why?
+----
+
+It is a good idea to shuffle the order in which your tests run
+because a test running early as part of a larger test suite may be leaving
 the system under test in a particularly fortunate state for a subsequent test to pass.
 
-How Random
-----------
+How?
+----
 
 **pytest-random-order** groups tests in buckets, shuffles them within buckets and then shuffles the buckets.
 
 You can choose from four types of buckets:
 
-    * ``class``
-    * ``module`` - **this is the default setting**
-    * ``package``
-    * ``global`` - all tests fall in the same bucket, full randomness, tests probably take longer to run
+``class``
+
+``module``
+    the default setting
+
+``package``
+
+``global``
+    all tests fall in the same bucket, full randomness, tests probably take longer to run
 
 If you have three buckets of tests ``A``, ``B``, and ``C`` with three tests ``1`` and ``2``, and ``3`` in each of them,
 then here are just two of many potential orderings that non-global randomisation can produce:
@@ -43,6 +51,9 @@ By default, your tests will be randomised at ``module`` level which means that
 tests within a single module X will be executed in no particular order, but tests from
 other modules will not be mixed in between tests of module X.
 
+The plugin also supports **disabling shuffle on module basis** irrespective of the bucket type
+chosen for the test run. See Advanced Options below.
+
 ----
 
 Installation
@@ -56,7 +67,8 @@ Installation
 Usage
 -----
 
-The plugin is enabled by default. To randomise the order of tests within modules, just run pytest as always:
+The plugin **is enabled by default**.
+To randomise the order of tests within modules, just run pytest as always:
 
 ::
 
@@ -87,6 +99,27 @@ pass undeservedly, you can disable it:
     $ pytest -p no:random-order -v
 
 
+Advanced Options
+----------------
+
+Disable Shuffling In a Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can disable shuffling of tests within a single module by placing a pytest marker in the module:
+
+::
+
+    pytest.mark.random_order_disabled = True
+
+    def test_number_one():
+        pass
+
+    def test_number_two():
+        pass
+
+No matter what will be the bucket type for the test run, ``test_number_one`` will always run
+before ``test_number_two``.
+
 License
 -------
 

pytest_random_order.py

@@ -4,6 +4,8 @@
 import sys
 import traceback
 
+import operator
+
 
 def pytest_addoption(parser):
     group = parser.getgroup('random-order')
@@ -35,7 +37,7 @@ def pytest_report_header(config):
 }
 
 
-def _shuffle_items(items, key=None, preserve_bucket_order=False):
+def _shuffle_items(items, key=None, disable=None, preserve_bucket_order=False):
     """
     Shuffles `items`, a list, in place.
 
@@ -47,26 +49,46 @@ def _shuffle_items(items, key=None, preserve_bucket_order=False):
     Bucket defines the boundaries across which tests will not
     be reordered.
 
+    If `disable` is function and returns True for ALL items
+    in a bucket, items in this bucket will remain in their original order.
+
     `preserve_bucket_order` is only customisable for testing purposes.
     There is no use case for predefined bucket order, is there?
     """
 
     # If `key` is falsey, shuffle is global.
-    if not key:
+    if not key and not disable:
         random.shuffle(items)
         return
 
+    # Use (key(x), disable(x)) as the key because
+    # when we have a bucket type like package over a disabled module, we must
+    # not shuffle the disabled module items.
+    def full_key(x):
+        if key and disable:
+            return key(x), disable(x)
+        elif disable:
+            return disable(x)
+        else:
+            return key(x)
+
     buckets = []
     this_key = '__not_initialised__'
     for item in items:
         prev_key = this_key
-        this_key = key(item)
+        this_key = full_key(item)
         if this_key != prev_key:
             buckets.append([])
         buckets[-1].append(item)
 
-    # Shuffle within bucket
+    # Shuffle within bucket unless disable(item) evaluates to True for
+    # the first item in the bucket.
+    # This assumes that whoever supplied disable function knows this requirement.
+    # Fixation of individual items in an otherwise shuffled bucket
+    # is not supported.
     for bucket in buckets:
+        if callable(disable) and disable(bucket[0]):
+            continue
         random.shuffle(bucket)
 
     # Shuffle buckets
@@ -85,13 +107,28 @@ def _get_set_of_item_ids(items):
         return s
 
 
+_is_random_order_disabled = operator.attrgetter('pytest.mark.random_order_disabled')
+
+
+def _disable(item):
+    try:
+        if _is_random_order_disabled(item.module):
+            # It is not enough to return just True because in case the shuffling
+            # is disabled on module, we must preserve the module unchanged
+            # even when the bucket type for this test run is say package or global.
+            return item.module.__name__
+    except AttributeError:
+        return False
+
+
 def pytest_collection_modifyitems(session, config, items):
     failure = None
+
     item_ids = _get_set_of_item_ids(items)
 
     try:
-        shuffle_mode = config.getoption('random_order_bucket')
-        _shuffle_items(items, key=_random_order_item_keys[shuffle_mode])
+        bucket_type = config.getoption('random_order_bucket')
+        _shuffle_items(items, key=_random_order_item_keys[bucket_type], disable=_disable)
 
     except Exception as e:
         # If the number of items is still the same, we assume that we haven't messed up too hard

setup.py

@@ -13,7 +13,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.3.0',
+    version='0.3.1',
     author='Jazeps Basko',
     author_email='jazeps.basko@gmail.com',
     maintainer='Jazeps Basko',

tests/conftest.py

@@ -1 +1,33 @@
+import collections
+
+import pytest
+
+
 pytest_plugins = 'pytester'
+
+
+Call = collections.namedtuple('Call', field_names=('package', 'module', 'cls', 'name'))
+
+
+def _get_test_calls(result):
+    """
+    Returns a tuple of test calls in the order they were made.
+    """
+    calls = []
+
+    for c in result.reprec.getcalls('pytest_runtest_call'):
+        calls.append(Call(
+            package=c.item.module.__package__,
+            module=c.item.module.__name__,
+            cls=(c.item.module.__name__, c.item.cls.__name__) if c.item.cls else None,
+            name=c.item.name,
+        ))
+    return tuple(calls)
+
+
+@pytest.fixture
+def get_test_calls():
+    """
+    Returns a function to get runtest calls out from testdir.pytestrun result object.
+    """
+    return _get_test_calls

tests/test_actual_test_runs.py

@@ -5,26 +5,6 @@
 import pytest
 
 
-Call = collections.namedtuple('Call', field_names=('package', 'module', 'cls', 'name'))
-
-
-def get_runtest_call_sequence(result):
-    """
-    Returns a tuple of names of test methods that were run
-    in the order they were run.
-    """
-    calls = []
-
-    for c in result.reprec.getcalls('pytest_runtest_call'):
-        calls.append(Call(
-            package=c.item.module.__package__,
-            module=c.item.module.__name__,
-            cls=(c.item.module.__name__, c.item.cls.__name__) if c.item.cls else None,
-            name=c.item.name,
-        ))
-    return tuple(calls)
-
-
 @pytest.fixture
 def tmp_tree_of_tests(testdir):
     """
@@ -156,13 +136,13 @@ def inspect_attr(this_call, prev_call, attr_name):
 
 
 @pytest.mark.parametrize('bucket', ['class', 'module', 'package', 'global'])
-def test_it_works_with_actual_tests(tmp_tree_of_tests, bucket):
+def test_it_works_with_actual_tests(tmp_tree_of_tests, get_test_calls, bucket):
     sequences = set()
 
     for x in range(5):
         result = tmp_tree_of_tests.runpytest('--random-order-bucket={}'.format(bucket), '--verbose')
         result.assert_outcomes(passed=14, failed=3)
-        seq = get_runtest_call_sequence(result)
+        seq = get_test_calls(result)
         check_call_sequence(seq, bucket=bucket)
         assert len(seq) == 17
         sequences.add(seq)

tests/test_markers.py

@@ -0,0 +1,28 @@
+import pytest
+
+
+@pytest.fixture
+def twenty_tests():
+    code = []
+    for i in range(20):
+        code.append('def test_a{}(): assert True\n'.format(str(i).zfill(2)))
+    return ''.join(code)
+
+
+@pytest.mark.parametrize('disabled', [True, False])
+def test_pytest_mark_random_order_disabled(testdir, twenty_tests, get_test_calls, disabled):
+    testdir.makepyfile(
+        'import pytest\n' +
+        'pytest.mark.random_order_disabled = {}\n'.format(disabled) +
+        twenty_tests
+    )
+
+    result = testdir.runpytest('--random-order-bucket=module', '-v')
+    result.assert_outcomes(passed=20)
+    names = [c.name for c in get_test_calls(testdir.runpytest())]
+    sorted_names = sorted(list(names))
+
+    if disabled:
+        assert names == sorted_names
+    else:
+        assert names != sorted_names

tests/test_shuffle.py

@@ -12,6 +12,11 @@ def oddity_key(x):
     return x % 2
 
 
+def disable_if_gt_1000(x):
+    # if disable returns a truthy value, it must also be usable as a key
+    return (x > 1000), (x // 1000)
+
+
 @pytest.mark.parametrize('key', [
     None,
     lambda x: None,
@@ -95,3 +100,38 @@ def test_shuffles_buckets():
     assert items[0] == items[1]
     assert items[2] == items[3]
     assert items[-1] == items[-2]
+
+
+def test_shuffle_respects_single_disabled_group_in_each_of_two_buckets():
+    items = [
+        11, 13, 9995, 9997, 19, 21, 23, 25, 27, 29,  # bucket 1 -- odd numbers
+        12, 14, 9996, 9998, 20, 22, 24, 26, 28, 30,  # bucket 2 -- even numbers
+    ]
+    items_copy = list(items)
+
+    _shuffle_items(items, key=oddity_key, disable=disable_if_gt_1000)
+
+    assert items != items_copy
+    assert items.index(9995) + 1 == items.index(9997)
+    assert items.index(9996) + 1 == items.index(9998)
+
+
+def test_shuffle_respects_two_distinct_disabled_groups_in_one_bucket():
+    # all items are in one oddity bucket, but the two groups
+    # of large numbers are separate because they are disabled
+    # from two different units.
+    # This is simulating two disabled modules within same package.
+    # The two modules shouldn't be mixed up in one bucket.
+    items = [
+        11, 13, 8885, 8887, 8889, 21, 23, 9995, 9997, 9999,
+    ]
+    items_copy = list(items)
+
+    for i in range(5):
+        _shuffle_items(items, key=oddity_key, disable=disable_if_gt_1000)
+        if items != items_copy:
+            assert items[items.index(8885):items.index(8885) + 3] == [8885, 8887, 8889]
+            assert items[items.index(9995):items.index(9995) + 3] == [9995, 9997, 9999]
+            return
+
+    assert False