Should not return placeholder if everything is installed

Author: LysandreJik

src/lighteval/utils/imports.py

@@ -189,7 +189,7 @@ def __init__(self, *args, **kwargs):
             Placeholder.__name__ = _object.__name__
             Placeholder.__module__ = _object.__module__
 
-            return Placeholder
+            return _object if all(is_package_available(backend) for backend in applied_backends) else Placeholder
         else:
 
             @functools.wraps(_object)

tests/test_dependencies.py

@@ -28,7 +28,7 @@
 from packaging.requirements import Requirement
 
 import lighteval.utils.imports as imports
-from lighteval.utils.imports import Extra
+from lighteval.utils.imports import DummyObject, Extra
 
 
 def pretend_missing(*names):
@@ -48,14 +48,14 @@ def fake(requirement):
                     name = requirement.name
                 elif isinstance(requirement, Extra):
                     name = requirement.value
+
                 return False if name in names else (importlib.util.find_spec(name) is not None)
 
             with patch.object(imports, "is_package_available", side_effect=fake):
-                # If your module caches results at import time, reload here
+                # In case lighteval caches results at import time, reload here
                 import lighteval
 
                 importlib.reload(lighteval)
-
                 return test_func(*args, **kwargs)
 
         return wrapper
@@ -97,3 +97,16 @@ def test_vllm_required_for_vllm_usage():
         match="Through the use of VLLMModel, you requested the use of `vllm<0.10.2,>=0.10.0` for this evaluation, but it is not available in your current environment. Please install it using pip.",
     ):
         vllm(model_args="model_name=gpt2", tasks="lighteval|aime24|0", max_samples=0)
+
+
+@pretend_missing("spacy")
+def test_placeholder_is_returned_when_missing_dependencies():
+    from lighteval.metrics.utils.linguistic_tokenizers import SpaCyTokenizer
+
+    assert isinstance(SpaCyTokenizer, DummyObject)
+
+
+def test_original_object_is_returned_when_dependencies_are_satisfied():
+    from lighteval.metrics.utils.linguistic_tokenizers import SpaCyTokenizer
+
+    assert not isinstance(SpaCyTokenizer, DummyObject)