File tree Expand file tree Collapse file tree 1 file changed +14
-0
lines changed Expand file tree Collapse file tree 1 file changed +14
-0
lines changed Original file line number Diff line number Diff line change @@ -662,6 +662,12 @@ def _run_validation_level_0(
662662 extra_kwargs ,
663663 model ,
664664):
665+ """Runs validation level 0 for comparing AIU / CPU models. validation level 0 entails
666+ comparing the extracted tokens for AIU / CPU.
667+
668+ NOTE: we try to load CPU validation info from disk to save time. Some of the computations
669+ here are also reused in level1 validation.
670+ """
665671 cpu_validation_info = _get_device_validation_information (
666672 model_path = model_path ,
667673 batch_size = batch_size ,
@@ -729,6 +735,14 @@ def _run_validation_level_1(
729735 micro_model_path ,
730736 validation_zero_info ,
731737):
738+ """Runs validation level 1 for comparing AIU / CPU models. validation level 1 entails
739+ cycling over a variety of seeds & filtering by sequence length before computing level 1
740+ metrics, which currently consists of cross entropy loss per token & mean absolute diff
741+ per decode step.
742+
743+ Thresholds for what are allowed to be passing are determined based on the model ID and
744+ the micro model configuration.
745+ """
732746 iters = 1024 // max_new_tokens
733747 ce_fail_responses_list = []
734748 diff_fail_responses_list = []
You can’t perform that action at this time.
0 commit comments