From 35aa0ec3210d25b03294094b192d94bc4d0db81a Mon Sep 17 00:00:00 2001 From: Sergii Dymchenko Date: Mon, 17 Jul 2023 14:54:31 -0700 Subject: [PATCH 1/2] Fix require_grad typo --- .../BERT_pytorch/bert_pytorch/model/embedding/position.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py b/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py index d55c224b5c..75ce9f8264 100644 --- a/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py +++ b/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py @@ -10,7 +10,7 @@ def __init__(self, d_model, max_len=512): # Compute the positional encodings once in log space. pe = torch.zeros(max_len, d_model).float() - pe.require_grad = False + pe.requires_grad = False position = torch.arange(0, max_len).float().unsqueeze(1) div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp() From c2b42c9a1ddf994cdf7acce58c4174bee466dfa6 Mon Sep 17 00:00:00 2001 From: Sergii Dymchenko Date: Wed, 19 Jul 2023 11:30:33 -0700 Subject: [PATCH 2/2] Update position.py --- .../models/BERT_pytorch/bert_pytorch/model/embedding/position.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py b/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py index 75ce9f8264..0f615b6def 100644 --- a/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py +++ b/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py @@ -10,6 +10,7 @@ def __init__(self, d_model, max_len=512): # Compute the positional encodings once in log space. pe = torch.zeros(max_len, d_model).float() + # Changed from upstream, see https://github.com/codertimo/BERT-pytorch/pull/104 pe.requires_grad = False position = torch.arange(0, max_len).float().unsqueeze(1)