Skip to content

Commit 5b840db

Browse files
committed
Revert "[MGSM] Enable auto_id, add language tag, consistent shuffling (rllm-org#130)"
This reverts commit ff0bfc6.
1 parent 3fbe3f4 commit 5b840db

File tree

1 file changed

+3
-9
lines changed

1 file changed

+3
-9
lines changed

src/inspect_evals/mgsm/mgsm.py

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
from time import time
21
from typing import List
32

43
from inspect_ai import Task, task
@@ -69,7 +68,6 @@ def load_mgsm_dataset(
6968
languages = ALL_LANGUAGES
7069

7170
samples = []
72-
seed = int(time()) # use same shuffling seed for all languages
7371

7472
for lang in languages:
7573
csv_filename = LANG_TO_FPATH[lang]
@@ -79,17 +77,14 @@ def load_mgsm_dataset(
7977
dialect="excel-tab",
8078
limit=limit_samples_per_lang,
8179
shuffle=shuffle,
82-
seed=seed,
83-
auto_id=True,
8480
delimiter="\t",
8581
)
8682

8783
lang_samples = lang_dataset.samples # type: ignore
88-
cot_template = LANG_TO_INSTRUCTIONS[lang]
8984

90-
for sample in lang_samples:
91-
sample.metadata = {"language": lang}
92-
if use_cot:
85+
if use_cot:
86+
cot_template = LANG_TO_INSTRUCTIONS[lang]
87+
for sample in lang_samples:
9388
cot_prompt = cot_template.format(prompt=sample.input)
9489
sample.input = cot_prompt
9590

@@ -110,7 +105,6 @@ def mgsm(
110105
languages=languages,
111106
limit_samples_per_lang=limit_samples_per_lang,
112107
use_cot=use_cot,
113-
shuffle=shuffle,
114108
)
115109

116110
task = Task(

0 commit comments

Comments
 (0)