Move fuzzing tests into separate files

These tests will only run with the --run-slow option.

Replicates graphql/graphql-js@8a0346f

Author: Cito

.coveragerc

@@ -2,8 +2,10 @@
 branch = True
 source = src
 omit =
+    */conftest.py
     */cached_property.py
     */is_collection.py
+    */test_*_fuzz.py
 
 [report]
 exclude_lines =

poetry.lock

@@ -37,6 +37,7 @@ pytest-asyncio = ">=0.14,<1"
 pytest-benchmark = "^3.2"
 pytest-cov = "^2.10"
 pytest-describe = "^1.0"
+pytest-timeout = "^1.4"
 black = "19.10b0"
 flake8 = "^3.8"
 mypy = "0.782"

pyproject.toml

@@ -7,4 +7,6 @@ test = pytest
 [tool:pytest]
 # Only run benchmarks as tests.
 # To actually run the benchmarks, use --benchmark-enable on the command line.
+# To run the slow tests (fuzzing), add --run-slow on the command line.
 addopts = --benchmark-disable
+timeout = 100

setup.cfg

@@ -0,0 +1,22 @@
+# pytest configuration
+
+import pytest  # type: ignore
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-slow", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption("--run-slow"):
+        # without --run-slow option: skip all slow tests
+        skip_slow = pytest.mark.skip(reason="need --run-slow option to run")
+        for item in items:
+            if "slow" in item.keywords:
+                item.add_marker(skip_slow)

tests/conftest.py

@@ -1,15 +1,9 @@
-from typing import Optional
-
-from graphql.error import GraphQLSyntaxError
-from graphql.language import Source, Lexer, TokenKind
 from graphql.language.block_string import (
     dedent_block_string_value,
     print_block_string,
     get_block_string_indentation,
 )
 
-from ..utils import dedent, gen_fuzz_strings
-
 
 def join_lines(*args):
     return "\n".join(args)
@@ -138,43 +132,3 @@ def correctly_prints_string_with_a_first_line_indentation():
         assert print_block_string(s) == join_lines(
             '"""', "    first  ", "  line     ", "indentation", "     string", '"""'
         )
-
-    def correctly_print_random_strings():
-        def lex_value(s: str) -> Optional[str]:
-            lexer = Lexer(Source(s))
-            value = lexer.advance().value
-            assert lexer.advance().kind == TokenKind.EOF, "Expected EOF"
-            return value
-
-        # Testing with length >5 is taking exponentially more time. However it is
-        # highly recommended to test with increased limit if you make any change.
-        for fuzz_str in gen_fuzz_strings(allowed_chars='\n\t "a\\', max_length=5):
-            test_str = f'"""{fuzz_str}"""'
-
-            try:
-                test_value = lex_value(test_str)
-            except (AssertionError, GraphQLSyntaxError):
-                continue  # skip invalid values
-            assert isinstance(test_value, str)
-
-            printed_value = lex_value(print_block_string(test_value))
-
-            assert test_value == printed_value, dedent(
-                f"""
-                Expected lex_value(print_block_string({test_value!r})
-                  to equal {test_value!r}
-                  but got {printed_value!r}
-                """
-            )
-
-            printed_multiline_string = lex_value(
-                print_block_string(test_value, " ", True)
-            )
-
-            assert test_value == printed_multiline_string, dedent(
-                f"""
-                Expected lex_value(print_block_string({test_value!r}, ' ', True)
-                  to equal {test_value!r}
-                  but got {printed_multiline_string!r}
-                """
-            )

tests/language/test_block_string.py

@@ -0,0 +1,54 @@
+from typing import Optional
+
+from pytest import mark  # type: ignore
+
+from graphql.error import GraphQLSyntaxError
+from graphql.language import Source, Lexer, TokenKind
+from graphql.language.block_string import print_block_string
+
+from ..utils import dedent, gen_fuzz_strings
+
+
+def lex_value(s: str) -> Optional[str]:
+    lexer = Lexer(Source(s))
+    value = lexer.advance().value
+    assert lexer.advance().kind == TokenKind.EOF, "Expected EOF"
+    return value
+
+
+def describe_print_block_string():
+    @mark.slow
+    @mark.timeout(20)
+    def correctly_print_random_strings():
+        # Testing with length >7 is taking exponentially more time. However it is
+        # highly recommended to test with increased limit if you make any change.
+        for fuzz_str in gen_fuzz_strings(allowed_chars='\n\t "a\\', max_length=7):
+            test_str = f'"""{fuzz_str}"""'
+
+            try:
+                test_value = lex_value(test_str)
+            except (AssertionError, GraphQLSyntaxError):
+                continue  # skip invalid values
+            assert isinstance(test_value, str)
+
+            printed_value = lex_value(print_block_string(test_value))
+
+            assert test_value == printed_value, dedent(
+                f"""
+                Expected lex_value(print_block_string({test_value!r})
+                  to equal {test_value!r}
+                  but got {printed_value!r}
+                """
+            )
+
+            printed_multiline_string = lex_value(
+                print_block_string(test_value, " ", True)
+            )
+
+            assert test_value == printed_multiline_string, dedent(
+                f"""
+                Expected lex_value(print_block_string({test_value!r}, ' ', True)
+                  to equal {test_value!r}
+                  but got {printed_multiline_string!r}
+                """
+            )

tests/language/test_block_string_fuzz.py

@@ -8,7 +8,7 @@
 from graphql.utilities import strip_ignored_characters
 
 from ..fixtures import kitchen_sink_query, kitchen_sink_sdl  # noqa: F401
-from ..utils import dedent, gen_fuzz_strings
+from ..utils import dedent
 
 ignored_tokens = [
     # UnicodeBOM
@@ -374,27 +374,6 @@ def expect_stripped_string(block_str: str):
         expect_stripped_string('"""\n a\n b"""').to_equal('"""a\nb"""')
         expect_stripped_string('"""\na\n b\nc"""').to_equal('"""a\n b\nc"""')
 
-    def strips_ignored_characters_inside_random_block_strings():
-        # Testing with length >5 is taking exponentially more time. However it is
-        # highly recommended to test with increased limit if you make any change.
-        for fuzz_str in gen_fuzz_strings(allowed_chars='\n\t "a\\', max_length=5):
-            test_str = f'"""{fuzz_str}"""'
-
-            try:
-                test_value = lex_value(test_str)
-            except (AssertionError, GraphQLSyntaxError):
-                continue  # skip invalid values
-
-            stripped_value = lex_value(strip_ignored_characters(test_str))
-
-            assert test_value == stripped_value, dedent(
-                f"""
-                Expected lexValue(stripIgnoredCharacters({test_str!r})
-                  to equal {test_value!r}
-                  but got {stripped_value!r}
-                """
-            )
-
     # noinspection PyShadowingNames
     def strips_kitchen_sink_query_but_maintains_the_exact_same_ast(
         kitchen_sink_query,  # noqa: F811

tests/utilities/test_strip_ignored_characters.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+from pytest import mark  # type: ignore
+
+from graphql.error import GraphQLSyntaxError
+from graphql.language import Lexer, Source, TokenKind
+from graphql.utilities import strip_ignored_characters
+
+from ..utils import dedent, gen_fuzz_strings
+
+
+def lex_value(s: str) -> Optional[str]:
+    lexer = Lexer(Source(s))
+    value = lexer.advance().value
+    assert lexer.advance().kind == TokenKind.EOF, "Expected EOF"
+    return value
+
+
+def describe_strip_ignored_characters():
+    @mark.slow
+    @mark.timeout(20)
+    def strips_ignored_characters_inside_random_block_strings():
+        # Testing with length >7 is taking exponentially more time. However it is
+        # highly recommended to test with increased limit if you make any change.
+        for fuzz_str in gen_fuzz_strings(allowed_chars='\n\t "a\\', max_length=7):
+            test_str = f'"""{fuzz_str}"""'
+
+            try:
+                test_value = lex_value(test_str)
+            except (AssertionError, GraphQLSyntaxError):
+                continue  # skip invalid values
+
+            stripped_value = lex_value(strip_ignored_characters(test_str))
+
+            assert test_value == stripped_value, dedent(
+                f"""
+                Expected lexValue(stripIgnoredCharacters({test_str!r})
+                  to equal {test_value!r}
+                  but got {stripped_value!r}
+                """
+            )

tests/utilities/test_strip_ignored_characters_fuzz.py

@@ -44,6 +44,7 @@ deps =
     pytest-benchmark>=3.2,<4
     pytest-cov>=2.10,<3
     pytest-describe>=1,<2
+    pytest-timeout>=1.4,<2
 commands =
     poetry install
     poetry run pytest tests {posargs: --cov-report=term-missing --cov=graphql --cov=tests --cov-fail-under=100}