More robust pretend missing

Author: LysandreJik

tests/test_dependencies.py

@@ -23,40 +23,54 @@
 # tests/utils/pretend_missing.py
 import functools
 import importlib
+import sys
+from unittest.mock import patch
 
 import pytest
-from packaging.requirements import Requirement
 
-import lighteval.utils.imports as imports
-from lighteval.utils.imports import DummyObject, Extra
+
+def purge_lighteval():
+    for name in list(sys.modules):
+        if name == "lighteval" or name.startswith("lighteval."):
+            sys.modules.pop(name, None)
 
 
 def pretend_missing(*names):
     """
-    Decorator: pretend that certain packages are missing
-    by patching mypkg.utils.is_package_available.
+    Decorator for individual tests: make is_package_available(...) return False
+    for given names. After the test, restore the initial dependencies.
     """
 
     def decorator(test_func):
         @functools.wraps(test_func)
         def wrapper(*args, **kwargs):
-            from unittest.mock import patch
+            # Ensure import-time guards run under the fake
+            purge_lighteval()
 
             def fake(requirement):
-                name = requirement
-                if isinstance(requirement, Requirement):
-                    name = requirement.name
-                elif isinstance(requirement, Extra):
-                    name = requirement.value
+                name = getattr(requirement, "name", getattr(requirement, "value", requirement))
 
+                # Probably a cleaner option here; returns False
                 return False if name in names else (importlib.util.find_spec(name) is not None)
 
-            with patch.object(imports, "is_package_available", side_effect=fake):
-                # In case lighteval caches results at import time, reload here
-                import lighteval
+            result = None
+            try:
+                # Patch dependencies to remove the one we want to remove
+                with patch("lighteval.utils.imports.is_package_available", side_effect=fake):
+                    importlib.invalidate_caches()
+                    import lighteval  # noqa
+
+                    importlib.reload(lighteval)
+                    result = test_func(*args, **kwargs)
+            finally:
+                # After the decorator, restore the initial dependencies
+                importlib.invalidate_caches()
+                purge_lighteval()
+                import lighteval  # noqa
 
                 importlib.reload(lighteval)
-                return test_func(*args, **kwargs)
+
+            return result
 
         return wrapper
 
@@ -94,7 +108,7 @@ def test_vllm_required_for_vllm_usage():
 
     with pytest.raises(
         ImportError,
-        match="Through the use of VLLMModel, you requested the use of `vllm<0.10.2,>=0.10.0` for this evaluation, but it is not available in your current environment. Please install it using pip.",
+        match="Through the use of VLLMModel, you requested the use of ",
     ):
         vllm(model_args="model_name=gpt2", tasks="lighteval|aime24|0", max_samples=0)
 
@@ -103,10 +117,12 @@ def test_vllm_required_for_vllm_usage():
 def test_placeholder_is_returned_when_missing_dependencies():
     from lighteval.metrics.utils.linguistic_tokenizers import SpaCyTokenizer
 
-    assert isinstance(SpaCyTokenizer, DummyObject)
+    assert SpaCyTokenizer.__name__ == "SpaCyTokenizer"
+    assert SpaCyTokenizer.__class__.__name__ == "DummyObject"
 
 
 def test_original_object_is_returned_when_dependencies_are_satisfied():
     from lighteval.metrics.utils.linguistic_tokenizers import SpaCyTokenizer
 
-    assert not isinstance(SpaCyTokenizer, DummyObject)
+    assert SpaCyTokenizer.__name__ == "SpaCyTokenizer"
+    assert SpaCyTokenizer.__class__.__name__ == "ABCMeta"