Fix BERT measuring (#239)

Author: jan-grzybek-ampere

natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py

@@ -47,7 +47,7 @@ def run_single_pass(tf_runner, squad):
         tf_runner.set_input_tensor("input_mask:0", squad.get_attention_mask_array())
         tf_runner.set_input_tensor("segment_ids:0", squad.get_token_type_ids_array())
 
-        output = tf_runner.run(batch_size)
+        output = tf_runner.run(batch_size * seq_size)
 
         for i in range(batch_size):
             answer_start_id, answer_end_id = np.argmax(output["logits:0"][i], axis=0)
@@ -85,8 +85,8 @@ def run_pytorch_fp(model_path, batch_size, num_runs, timeout, squad_path, disabl
     from utils.pytorch import PyTorchRunner
 
     def run_single_pass(pytorch_runner, squad):
-
-        output = pytorch_runner.run(batch_size, **dict(squad.get_input_arrays()))
+        input_tensor = squad.get_input_arrays()
+        output = pytorch_runner.run(batch_size * input_tensor["input_ids"].size()[1], **dict(input_tensor))
 
         for i in range(batch_size):
             answer_start_id = output[0][i].argmax()
@@ -137,8 +137,9 @@ def run_pytorch_cuda(model_path, batch_size, num_runs, timeout, squad_path, disa
     from transformers import AutoTokenizer, BertConfig, BertForQuestionAnswering
 
     def run_single_pass(pytorch_runner, squad):
-
-        output = pytorch_runner.run(batch_size, **{k: v.cuda() for k, v in squad.get_input_arrays().items()})
+        input_tensor = squad.get_input_arrays()
+        output = pytorch_runner.run(batch_size * input_tensor["input_ids"].size()[1],
+                                    **{k: v.cuda() for k, v in input_tensor.items()})
 
         for i in range(batch_size):
             answer_start_id = output[0][i].argmax()
@@ -189,9 +190,11 @@ def main():
     download_squad_1_1_dataset()
 
     if args.framework == "tf":
+        if args.batch_size > 1:
+            print_goodbye_message_and_die("This model supports only BS=1")
+
         if args.model_path is None:
-            print_goodbye_message_and_die(
-                "a path to model is unspecified!")
+            print_goodbye_message_and_die("a path to model is unspecified!")
 
         if args.precision == "fp32":
             run_tf_fp32(**vars(args))

tests/test_pytorch_models.py

@@ -200,7 +200,7 @@ def test_bert_large_mlperf(self):
         def wrapper(**kwargs):
             kwargs["q"].put(run_pytorch_fp32(**kwargs)[0])
 
-        exact_match_ref, f1_ref = 0.792, 0.825
+        exact_match_ref, f1_ref = 0.750, 0.817
         acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path,
                                     "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False})
         self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95)

utils/nlp/squad.py

@@ -4,6 +4,7 @@
 import numpy as np
 import json
 import re
+import random
 import string
 from collections import Counter
 import utils.misc as utils
@@ -71,8 +72,12 @@ def __examples(self):
 
         :yield: str, str, list: context, questions, list of possible (correct) answers
         """
+        random.seed(44)
+        random.shuffle(self.__dataset)
         for section in self.__dataset:
+            random.shuffle(section["paragraphs"])
             for paragraph in section["paragraphs"]:
+                random.shuffle(paragraph["qas"])
                 for qas in paragraph["qas"]:
                     yield paragraph["context"], qas["question"], qas["answers"]