[MGSM] Enable auto_id, add language tag, consistent shuffling (take 2) (rllm-org#136)

* [MGSM] Enable auto_id, add language tag, consistent shuffling

Fixes rllm-org#92

* unique sample_id

Author: evanmiller-anthropic

src/inspect_evals/mgsm/mgsm.py

@@ -1,3 +1,4 @@
+from time import time
 from typing import List
 
 from inspect_ai import Task, task
@@ -68,6 +69,7 @@ def load_mgsm_dataset(
         languages = ALL_LANGUAGES
 
     samples = []
+    seed = int(time())  # use same shuffling seed for all languages
 
     for lang in languages:
         csv_filename = LANG_TO_FPATH[lang]
@@ -77,14 +79,18 @@ def load_mgsm_dataset(
             dialect="excel-tab",
             limit=limit_samples_per_lang,
             shuffle=shuffle,
+            seed=seed,
+            auto_id=True,
             delimiter="\t",
         )
 
         lang_samples = lang_dataset.samples  # type: ignore
+        cot_template = LANG_TO_INSTRUCTIONS[lang]
 
-        if use_cot:
-            cot_template = LANG_TO_INSTRUCTIONS[lang]
-            for sample in lang_samples:
+        for sample in lang_samples:
+            sample.metadata = {"language": lang, "question_id": sample.id}
+            sample.id = f"{lang}_{sample.id}"
+            if use_cot:
                 cot_prompt = cot_template.format(prompt=sample.input)
                 sample.input = cot_prompt
 
@@ -105,6 +111,7 @@ def mgsm(
         languages=languages,
         limit_samples_per_lang=limit_samples_per_lang,
         use_cot=use_cot,
+        shuffle=shuffle,
     )
 
     task = Task(