Skip to content

Commit d2551b9

Browse files
add level0 / 1 descriptions
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
1 parent 8bd788b commit d2551b9

File tree

1 file changed

+14
-0
lines changed

1 file changed

+14
-0
lines changed

tests/models/test_decoders.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,12 @@ def _run_validation_level_0(
662662
extra_kwargs,
663663
model,
664664
):
665+
"""Runs validation level 0 for comparing AIU / CPU models. validation level 0 entails
666+
comparing the extracted tokens for AIU / CPU.
667+
668+
NOTE: we try to load CPU validation info from disk to save time. Some of the computations
669+
here are also reused in level1 validation.
670+
"""
665671
cpu_validation_info = _get_device_validation_information(
666672
model_path=model_path,
667673
batch_size=batch_size,
@@ -729,6 +735,14 @@ def _run_validation_level_1(
729735
micro_model_path,
730736
validation_zero_info,
731737
):
738+
"""Runs validation level 1 for comparing AIU / CPU models. validation level 1 entails
739+
cycling over a variety of seeds & filtering by sequence length before computing level 1
740+
metrics, which currently consists of cross entropy loss per token & mean absolute diff
741+
per decode step.
742+
743+
Thresholds for what are allowed to be passing are determined based on the model ID and
744+
the micro model configuration.
745+
"""
732746
iters = 1024 // max_new_tokens
733747
ce_fail_responses_list = []
734748
diff_fail_responses_list = []

0 commit comments

Comments
 (0)