Merge pull request rllm-org#103 from AarushSah/asah/fix-seed

jjallaire · web-flow · commit 1b368f111faa · 2024-11-18T20:59:19.000-08:00
fix: set seed for shuffling MMLU samples to allow reproducibility
diff --git a/src/inspect_evals/mmlu/mmlu.py b/src/inspect_evals/mmlu/mmlu.py
@@ -200,6 +200,7 @@ def get_mmlu_dataset(
         split=split,
         sample_fields=record_to_sample,
         shuffle=shuffle,
+        seed=42,
     )
 
     # filter dataset if requested

Original file line number	Diff line number	Diff line change
`@@ -200,6 +200,7 @@ def get_mmlu_dataset(`
`200`	`200`	`split=split,`
`201`	`201`	`sample_fields=record_to_sample,`
`202`	`202`	`shuffle=shuffle,`
	`203`	`+ seed=42,`
`203`	`204`	`)`
`204`	`205`
`205`	`206`	`# filter dataset if requested`