Load WorldSense dataset at runtime

Author: MattFisher

src/inspect_evals/worldsense/worldsense.py

@@ -52,6 +52,8 @@ def worldsense(problemnames: str | list[str] = []) -> Task:
     Returns:
         Task: A task object configured with a dataset filtered by problem names (if specified), a solver, a scoring pattern for evaluating task responses, and custom metrics.
     """
+    dataset = load_worldsense_dataset(sample_fields=record_to_sample, shuffle=True)
+
     # filter dataset if requested
     problemnames = problemnames if isinstance(problemnames, list) else [problemnames]
     if len(problemnames) > 0:
@@ -139,9 +141,6 @@ def load_worldsense_dataset(
     return dataset
 
 
-dataset = load_worldsense_dataset(sample_fields=record_to_sample, shuffle=True)
-
-
 @scorer(metrics=[accuracy(), stderr()])
 def pattern_with_metadata(
     pattern_str: str, ignore_case: bool = True, match_all: bool = False