⚡️ Speed up function _test_type_from_string by 128% in PR #949 (feat/behavior-test-as-tool)

[codeflash-ai]

⚡️ This pull request contains optimizations for PR #949

If you approve this dependent PR, these changes will be merged into the original PR branch feat/behavior-test-as-tool.

This PR will be automatically closed if the original PR is merged.

---

📄 128% (1.28x) speedup for _test_type_from_string in codeflash/verification/llm_tools.py

⏱️ Runtime : 3.07 milliseconds → 1.34 milliseconds (best of 128 runs)

📝 Explanation and details

The optimization moves the mapping dictionary from inside the function to the module level as a constant _MAPPING, eliminating the need to recreate it on every function call.

Key Performance Improvements:

• Dictionary construction elimination: The original code reconstructs a 5-element dictionary with enum lookups on every call (consuming ~75% of execution time according to line profiler). The optimized version accesses a pre-built dictionary, reducing function body to a single lookup operation.
• Memory allocation reduction: Eliminates repeated dictionary allocation and garbage collection overhead for each invocation.

Why This Optimization Matters:
The line profiler shows the original function spends 86.2% of its time (lines 1-6) just building the mapping dictionary, with only 23.8% on the actual lookup. The optimized version eliminates this overhead entirely, achieving a 128% speedup and reducing runtime from 3.07ms to 1.34ms.

Impact on Existing Workloads:
Based on the function reference, _test_type_from_string is called within run_behavioral_tests_tool during test file processing. This function processes lists of test files and converts string test types to enums for each file. With the optimization, batch processing of test files will see significant performance improvements, especially when processing many test files where this conversion happens repeatedly.

Test Case Performance:
The annotated tests show consistent 70-140% speedup across all scenarios, with the largest gains in batch processing tests (125-143% faster) where the function is called many times in succession. This confirms the optimization is particularly effective for high-frequency usage patterns common in test processing workflows.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 4074 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from __future__ import annotations

from enum import Enum

# imports
import pytest

from codeflash.verification.llm_tools import _test_type_from_string


# Simulate the TestType Enum as it would be in codeflash.models.models
class TestType(Enum):
    EXISTING_UNIT_TEST = "existing_unit_test"
    GENERATED_REGRESSION = "generated_regression"
    REPLAY_TEST = "replay_test"
    CONCOLIC_COVERAGE_TEST = "concolic_coverage_test"


# unit tests

# -------------------------
# BASIC TEST CASES
# -------------------------


@pytest.mark.parametrize(
    "input_str,expected_enum",
    [
        # Standard string values
        ("existing_unit_test", TestType.EXISTING_UNIT_TEST),
        ("generated_regression", TestType.GENERATED_REGRESSION),
        ("replay_test", TestType.REPLAY_TEST),
        ("concolic_test", TestType.CONCOLIC_COVERAGE_TEST),
        ("concolic_coverage_test", TestType.CONCOLIC_COVERAGE_TEST),
    ],
)
def test_basic_mappings(input_str, expected_enum):
    """Test basic, expected string to enum mappings."""
    codeflash_output = _test_type_from_string(input_str)
    result = codeflash_output  # 9.62μs -> 5.34μs (80.1% faster)


@pytest.mark.parametrize(
    "input_str,expected_enum",
    [
        # Case insensitivity
        ("Existing_Unit_Test", TestType.EXISTING_UNIT_TEST),
        ("GENERATED_REGRESSION", TestType.GENERATED_REGRESSION),
        ("RePlaY_TeSt", TestType.REPLAY_TEST),
        ("CONCOLIC_TEST", TestType.CONCOLIC_COVERAGE_TEST),
        ("cOnCoLiC_CoVeRaGe_TeSt", TestType.CONCOLIC_COVERAGE_TEST),
    ],
)
def test_case_insensitivity(input_str, expected_enum):
    """Test that input is case-insensitive."""
    codeflash_output = _test_type_from_string(input_str)
    result = codeflash_output  # 9.35μs -> 5.39μs (73.4% faster)


# -------------------------
# EDGE TEST CASES
# -------------------------


@pytest.mark.parametrize(
    "input_str",
    [
        "",  # Empty string
        " ",  # Space
        "unknown",  # Completely unknown
        "unit_test",  # Partial match
        "existing_unit_test_",  # Trailing underscore
        "existing unit test",  # Spaces instead of underscores
        "existing-unit-test",  # Dashes instead of underscores
        "generated regression",  # Spaces
        "concolictest",  # Missing underscore
        "replay",  # Partial valid word
        "concolic coverage test",  # Spaces
        "123",  # Numbers only
        "None",  # String None
        "null",  # String null
        "true",  # String true
        "false",  # String false
        "\n",  # Newline
        "\t",  # Tab
        "EXISTING_UNIT_TEST\n",  # Valid with newline
        "  generated_regression  ",  # Valid with spaces
    ],
)
def test_invalid_and_edge_strings_default(input_str):
    """Test that unknown or malformed strings default to EXISTING_UNIT_TEST."""
    codeflash_output = _test_type_from_string(input_str)
    result = codeflash_output  # 37.4μs -> 19.7μs (89.5% faster)


def test_leading_trailing_whitespace():
    """Test that leading/trailing whitespace causes fallback to default."""
    codeflash_output = _test_type_from_string("  existing_unit_test  ")  # 1.91μs -> 952ns (101% faster)
    codeflash_output = _test_type_from_string("\nreplay_test\t")  # 1.02μs -> 510ns (100% faster)


def test_non_string_input_raises():
    """Test that non-string input raises AttributeError (since .lower() is called)."""
    # These should raise AttributeError due to .lower() call
    for bad_input in [None, 123, 12.34, [], {}, set()]:
        with pytest.raises(AttributeError):
            _test_type_from_string(bad_input)


# -------------------------
# LARGE SCALE TEST CASES
# -------------------------


def test_large_batch_mixed_inputs():
    """Test performance and correctness with a large batch of mixed valid/invalid inputs."""
    valid_types = [
        "existing_unit_test",
        "generated_regression",
        "replay_test",
        "concolic_test",
        "concolic_coverage_test",
    ]
    # Create 500 valid and 500 invalid entries
    test_inputs = []
    expected_outputs = []
    for i in range(500):
        # Valid, with random casing
        v = valid_types[i % len(valid_types)]
        if i % 2 == 0:
            v = v.upper()
        test_inputs.append(v)
        expected_outputs.append(_test_type_from_string(v))  # 372μs -> 162μs (129% faster)

        # Invalid
        inv = f"invalid_type_{i}"
        test_inputs.append(inv)
        expected_outputs.append(TestType.EXISTING_UNIT_TEST)

    # Run all in a loop
    for inp, expected in zip(test_inputs, expected_outputs):
        codeflash_output = _test_type_from_string(inp)  # 370μs -> 163μs (127% faster)


def test_performance_large_scale(benchmark):
    """Benchmark the function with a large number of calls."""
    # 1000 calls, half valid, half invalid
    valid_types = [
        "existing_unit_test",
        "generated_regression",
        "replay_test",
        "concolic_test",
        "concolic_coverage_test",
    ]
    inputs = []
    for i in range(500):
        inputs.append(valid_types[i % len(valid_types)])
        inputs.append(f"invalid_{i}")

    def call_all():
        for s in inputs:
            _test_type_from_string(s)

    benchmark(call_all)


# -------------------------
# SPECIAL EDGE CASES
# -------------------------


def test_mapping_is_not_mutated():
    """Ensure that calling the function does not mutate the mapping."""
    before = _test_type_from_string.__code__.co_consts
    _test_type_from_string("existing_unit_test")  # 2.18μs -> 1.27μs (71.6% faster)
    after = _test_type_from_string.__code__.co_consts


def test_enum_identity():
    """Test that the returned enum is the exact same object as in TestType."""
    codeflash_output = _test_type_from_string("existing_unit_test")
    result = codeflash_output  # 2.06μs -> 1.14μs (80.7% faster)
    codeflash_output = _test_type_from_string("concolic_test")
    result2 = codeflash_output  # 941ns -> 481ns (95.6% faster)


# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

from enum import Enum  # used to define TestType Enum

# imports
import pytest  # used for our unit tests

from codeflash.verification.llm_tools import _test_type_from_string


# function to test
class TestType(Enum):
    EXISTING_UNIT_TEST = 1
    GENERATED_REGRESSION = 2
    REPLAY_TEST = 3
    CONCOLIC_COVERAGE_TEST = 4


# unit tests

# --- Basic Test Cases ---


def test_existing_unit_test_lowercase():
    # Basic: canonical string
    codeflash_output = _test_type_from_string("existing_unit_test")  # 2.07μs -> 1.13μs (83.1% faster)


def test_generated_regression_lowercase():
    # Basic: canonical string
    codeflash_output = _test_type_from_string("generated_regression")  # 1.95μs -> 1.09μs (78.9% faster)


def test_replay_test_lowercase():
    # Basic: canonical string
    codeflash_output = _test_type_from_string("replay_test")  # 1.96μs -> 1.09μs (79.7% faster)


def test_concolic_test_lowercase():
    # Basic: canonical string
    codeflash_output = _test_type_from_string("concolic_test")  # 1.91μs -> 1.07μs (78.5% faster)


def test_concolic_coverage_test_lowercase():
    # Basic: canonical string
    codeflash_output = _test_type_from_string("concolic_coverage_test")  # 1.93μs -> 1.07μs (80.3% faster)


# --- Case Insensitivity ---


@pytest.mark.parametrize(
    "input_str,expected",
    [
        ("EXISTING_UNIT_TEST", TestType.EXISTING_UNIT_TEST),
        ("Generated_Regression", TestType.GENERATED_REGRESSION),
        ("REPLAY_TEST", TestType.REPLAY_TEST),
        ("Concolic_Test", TestType.CONCOLIC_COVERAGE_TEST),
        ("CONCOLIC_COVERAGE_TEST", TestType.CONCOLIC_COVERAGE_TEST),
    ],
)
def test_case_insensitivity(input_str, expected):
    # Should be case-insensitive
    codeflash_output = _test_type_from_string(input_str)  # 9.79μs -> 5.42μs (80.6% faster)


# --- Edge Test Cases ---


def test_unrecognized_string_returns_default():
    # Edge: unrecognized string should return EXISTING_UNIT_TEST
    codeflash_output = _test_type_from_string("unknown_type")  # 1.94μs -> 1.07μs (81.4% faster)


def test_empty_string_returns_default():
    # Edge: empty string should return EXISTING_UNIT_TEST
    codeflash_output = _test_type_from_string("")  # 1.70μs -> 952ns (78.9% faster)


def test_whitespace_string_returns_default():
    # Edge: whitespace only string should return EXISTING_UNIT_TEST
    codeflash_output = _test_type_from_string("   ")  # 1.91μs -> 1.06μs (80.2% faster)


def test_partial_match_returns_default():
    # Edge: partial match should NOT work, must be full string
    codeflash_output = _test_type_from_string("existing")  # 1.93μs -> 1.06μs (82.1% faster)


def test_none_input_raises_attribute_error():
    # Edge: None input should raise AttributeError (since None has no .lower())
    with pytest.raises(AttributeError):
        _test_type_from_string(None)  # 3.90μs -> 2.62μs (48.5% faster)


def test_numeric_string_returns_default():
    # Edge: numeric string should return EXISTING_UNIT_TEST
    codeflash_output = _test_type_from_string("12345")  # 1.97μs -> 1.00μs (97.0% faster)


def test_special_characters_returns_default():
    # Edge: special characters should return EXISTING_UNIT_TEST
    codeflash_output = _test_type_from_string("@!#*")  # 1.85μs -> 1.07μs (72.9% faster)


def test_leading_trailing_spaces():
    # Edge: leading/trailing spaces should not match (no strip)
    codeflash_output = _test_type_from_string(" existing_unit_test ")  # 1.89μs -> 1.05μs (79.9% faster)


def test_mixed_case_and_spaces():
    # Edge: mixed case and spaces should not match
    codeflash_output = _test_type_from_string("  Concolic_Test  ")  # 1.93μs -> 1.05μs (83.8% faster)


# --- Large Scale Test Cases ---


def test_large_batch_known_types():
    # Large scale: test a list of valid types in random cases
    valid_types = [
        "existing_unit_test",
        "generated_regression",
        "replay_test",
        "concolic_test",
        "concolic_coverage_test",
    ]
    # Generate 200 random case versions of each valid type
    import random

    def random_case(s):
        return "".join(random.choice([c.lower(), c.upper()]) for c in s)

    all_inputs = [random_case(t) for t in valid_types for _ in range(200)]
    expected_outputs = []
    for s in all_inputs:
        s_lower = s.lower()
        if s_lower == "existing_unit_test":
            expected_outputs.append(TestType.EXISTING_UNIT_TEST)
        elif s_lower == "generated_regression":
            expected_outputs.append(TestType.GENERATED_REGRESSION)
        elif s_lower == "replay_test":
            expected_outputs.append(TestType.REPLAY_TEST)
        elif s_lower in ("concolic_test", "concolic_coverage_test"):
            expected_outputs.append(TestType.CONCOLIC_COVERAGE_TEST)
        else:
            expected_outputs.append(TestType.EXISTING_UNIT_TEST)
    for inp, exp in zip(all_inputs, expected_outputs):
        codeflash_output = _test_type_from_string(inp)  # 731μs -> 317μs (130% faster)


def test_large_batch_unknown_types():
    # Large scale: test a list of 500 unknown types, should all return EXISTING_UNIT_TEST
    unknown_types = [f"unknown_type_{i}" for i in range(500)]
    for s in unknown_types:
        codeflash_output = _test_type_from_string(s)  # 371μs -> 164μs (125% faster)


def test_large_batch_empty_and_whitespace():
    # Large scale: test a batch of empty/whitespace strings
    test_inputs = ["", " ", "   ", "\t", "\n"] * 100
    for s in test_inputs:
        codeflash_output = _test_type_from_string(s)  # 358μs -> 147μs (143% faster)


def test_large_batch_leading_trailing_spaces():
    # Large scale: test valid types with leading/trailing spaces
    valid_types = [
        "existing_unit_test",
        "generated_regression",
        "replay_test",
        "concolic_test",
        "concolic_coverage_test",
    ]
    test_inputs = [f"  {t}  " for t in valid_types for _ in range(100)]
    for s in test_inputs:
        codeflash_output = _test_type_from_string(s)  # 370μs -> 160μs (132% faster)


def test_large_batch_partial_matches():
    # Large scale: test partial matches, should all return EXISTING_UNIT_TEST
    partials = ["existing", "generated", "replay", "concolic", "coverage"] * 100
    for s in partials:
        codeflash_output = _test_type_from_string(s)  # 361μs -> 154μs (135% faster)


# --- Mutation Testing Guards ---


def test_mutation_guard_wrong_mapping():
    # If mapping is changed, this should fail
    codeflash_output = _test_type_from_string("generated_regression")  # 2.07μs -> 1.12μs (84.8% faster)
    codeflash_output = _test_type_from_string("replay_test")  # 1.01μs -> 601ns (68.4% faster)
    codeflash_output = _test_type_from_string("concolic_test")  # 822ns -> 431ns (90.7% faster)


def test_mutation_guard_default_behavior():
    # If default is changed from EXISTING_UNIT_TEST, this should fail
    codeflash_output = _test_type_from_string("not_a_type")  # 2.01μs -> 1.02μs (97.1% faster)


# --- Determinism ---


def test_determinism():
    # Calling multiple times with same input yields same result
    for _ in range(10):
        codeflash_output = _test_type_from_string("generated_regression")  # 8.80μs -> 4.12μs (114% faster)


# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-pr949-2025-12-03T04.19.03 and push.