[BERT/Paddle] Update LDDL

Author: Wong4j

PaddlePaddle/LanguageModeling/BERT/Dockerfile

@@ -1,15 +1,20 @@
-ARG FROM_IMAGE_NAME=nvcr.io/nvidia/paddlepaddle:22.08-py3
-
+ARG FROM_IMAGE_NAME=nvcr.io/nvidia/paddlepaddle:22.12-py3
 FROM ${FROM_IMAGE_NAME}
-
 RUN apt-get update && apt-get install -y pbzip2 pv bzip2 cabextract
 
 ENV BERT_PREP_WORKING_DIR /workspace/bert/data
-ADD requirements.txt /workspace/
+
 WORKDIR /workspace/
-RUN pip install --no-cache-dir -r requirements.txt
-RUN git clone https://github.com/attardi/wikiextractor.git && cd wikiextractor && git checkout 6408a430fc504a38b04d37ce5e7fc740191dee16 && cd ..
-RUN git clone https://github.com/soskek/bookcorpus.git
 
-ADD . /workspace/bert
 WORKDIR /workspace/bert
+RUN pip install --no-cache-dir \
+ tqdm boto3 requests six ipdb h5py nltk progressbar tokenizers>=0.7\
+ git+https://github.com/NVIDIA/dllogger wget
+
+RUN apt-get install -y iputils-ping
+
+COPY . .
+
+RUN apt-get install -y libjemalloc-dev
+RUN pip install git+https://github.com/NVIDIA/lddl.git
+RUN python -m nltk.downloader punkt

PaddlePaddle/LanguageModeling/BERT/README.md

@@ -13,36 +13,5 @@
 # limitations under the License.
 
 #Download
-to_download=${1:-"wiki_only"}
-
-#Download
-if [ "$to_download" = "wiki_books" ] ; then
-    python3 /workspace/bert/data/bertPrep.py --action download --dataset bookscorpus
-fi
-
-python3 /workspace/bert/data/bertPrep.py --action download --dataset wikicorpus_en
+download_wikipedia --outdir ${BERT_PREP_WORKING_DIR}/wikipedia/
 python3 /workspace/bert/data/bertPrep.py --action download --dataset squad
-
-# Properly format the text files
-if [ "$to_download" = "wiki_books" ] ; then
-    python3 /workspace/bert/data/bertPrep.py --action text_formatting --dataset bookscorpus
-fi
-python3 /workspace/bert/data/bertPrep.py --action text_formatting --dataset wikicorpus_en
-
-if [ "$to_download" = "wiki_books" ] ; then
-    DATASET="books_wiki_en_corpus"
-else
-    DATASET="wikicorpus_en"
-    # Shard the text files
-fi
-
-# Shard the text files
-python3 /workspace/bert/data/bertPrep.py --action sharding --dataset $DATASET
-
-# Create HDF5 files Phase 1
-python3 /workspace/bert/data/bertPrep.py --action create_hdf5_files --dataset $DATASET --max_seq_length 128 \
---max_predictions_per_seq 20 --vocab_file /workspace/bert/vocab/bert-large-uncased-vocab.txt --do_lower_case 1
-
-# Create HDF5 files Phase 2
-python3 /workspace/bert/data/bertPrep.py --action create_hdf5_files --dataset $DATASET --max_seq_length 512 \
---max_predictions_per_seq 80 --vocab_file /workspace/bert/vocab/bert-large-uncased-vocab.txt --do_lower_case 1

PaddlePaddle/LanguageModeling/BERT/data/create_datasets_from_start.sh

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import paddle
-import paddle.nn.functional as F
 
 
 class CrossEntropyLossForSQuAD(paddle.nn.Layer):
@@ -53,7 +52,7 @@ def __init__(self, vocab_size):
         self.vocab_size = vocab_size
 
     def forward(self, prediction_scores, seq_relationship_score,
-                masked_lm_labels, next_sentence_labels, masked_lm_scale):
+                masked_lm_labels, next_sentence_labels):
         """
         Args:
             prediction_scores(Tensor):
@@ -80,12 +79,11 @@ def forward(self, prediction_scores, seq_relationship_score,
             Its data type should be float32 and its shape is [1].
         """
         with paddle.static.amp.fp16_guard():
-            masked_lm_loss = F.cross_entropy(
-                prediction_scores,
-                masked_lm_labels,
-                reduction='none',
-                ignore_index=-1)
-            masked_lm_loss = masked_lm_loss / masked_lm_scale
-            next_sentence_loss = F.cross_entropy(
-                seq_relationship_score, next_sentence_labels, reduction='none')
-        return paddle.sum(masked_lm_loss) + paddle.mean(next_sentence_loss)
+            masked_lm_labels_flat = masked_lm_labels.reshape([-1])
+            mlm_labels = masked_lm_labels_flat[masked_lm_labels_flat != -1]
+            masked_lm_loss = self.loss_fn(prediction_scores, mlm_labels)
+            if next_sentence_labels.ndim == 1:
+                next_sentence_labels = next_sentence_labels.unsqueeze(axis=-1)
+            next_sentence_loss = self.loss_fn(seq_relationship_score,
+                                              next_sentence_labels)
+        return masked_lm_loss + next_sentence_loss

PaddlePaddle/LanguageModeling/BERT/loss.py

@@ -89,17 +89,15 @@ def __init__(self, bert_config):
         self.layer_norm = nn.LayerNorm(bert_config.hidden_size, epsilon=1e-12)
         self.dropout = nn.Dropout(bert_config.hidden_dropout_prob)
 
-    def forward(self, input_ids, token_type_ids=None, position_ids=None):
+    def forward(self, input_ids, token_type_ids=None):
         """
         Args:
             See class `BertModel`.
         """
-        if position_ids is None:
-            ones = paddle.ones_like(input_ids, dtype="int64")
-            seq_length = paddle.cumsum(ones, axis=-1)
-
-            position_ids = seq_length - ones
-            position_ids.stop_gradient = True
+        ones = paddle.ones_like(input_ids, dtype="int64")
+        seq_length = paddle.cumsum(ones, axis=-1)
+        position_ids = seq_length - ones
+        position_ids.stop_gradient = True
         if token_type_ids is None:
             token_type_ids = paddle.zeros_like(input_ids, dtype="int64")
 
@@ -174,18 +172,13 @@ def __init__(self, bert_config):
                 dropout=bert_config.hidden_dropout_prob,
                 activation=bert_config.hidden_act,
                 attn_dropout=bert_config.attention_probs_dropout_prob,
-                act_dropout=0,
-                enable_cudnn=False)
+                act_dropout=0)
             self.encoder = nn.TransformerEncoder(encoder_layer,
                                                  bert_config.num_hidden_layers)
 
         self.pooler = BertPooler(bert_config.hidden_size)
 
-    def forward(self,
-                input_ids,
-                token_type_ids=None,
-                position_ids=None,
-                attention_mask=None):
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None):
         """
         Args:
             input_ids(Tensor):
@@ -198,11 +191,6 @@ def forward(self,
                 to a `sentence A` and type 1 corresponds to a `sentence B` token.
                 (see BERT paper for more details). Its data type should be `int64`
                 Defaults: None, which means we don't add segment embeddings.
-            position_ids(Tensor, optional):
-                An optional Tensor of shape [batch_size, num_tokens] with the position
-                indices of each input sequence tokens in the position embeddings.
-                Selected in the range [0, max_position_embeddings - 1].
-                Its data type should be `int64`. Defaults: None.
             attention_mask(Tensor, optional):
                 An optional Tensor of shape [batch_size, sequence_length] with indices of
                 mask used in multi-head attention to avoid performing attention on to some
@@ -234,9 +222,7 @@ def forward(self,
                 attention_mask = attention_mask.unsqueeze(axis=[1, 2])
 
         embedding_output = self.embeddings(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            token_type_ids=token_type_ids)
+            input_ids=input_ids, token_type_ids=token_type_ids)
 
         if self.fuse:
             encoder_output = embedding_output
@@ -263,11 +249,7 @@ def __init__(self, bert_config):
         self.bert = BertModel(bert_config)
         self.classifier = nn.Linear(bert_config.hidden_size, 2)
 
-    def forward(self,
-                input_ids,
-                token_type_ids=None,
-                position_ids=None,
-                attention_mask=None):
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None):
         """
         Args:
             See class `BertModel`.
@@ -282,7 +264,6 @@ def forward(self,
         encoder_output, _ = self.bert(
             input_ids,
             token_type_ids=token_type_ids,
-            position_ids=position_ids,
             attention_mask=attention_mask)
 
         logits = self.classifier(encoder_output)
@@ -322,13 +303,7 @@ def __init__(self,
         self.decoder_bias = self.create_parameter(
             shape=[vocab_size], dtype=self.decoder_weight.dtype, is_bias=True)
 
-    def forward(self, hidden_states, masked_positions=None):
-        if masked_positions is not None:
-            hidden_states = paddle.reshape(hidden_states,
-                                           [-1, hidden_states.shape[-1]])
-            hidden_states = paddle.tensor.gather(hidden_states,
-                                                 masked_positions)
-        # gather masked tokens might be more quick
+    def forward(self, hidden_states):
         hidden_states = self.transform(hidden_states)
         hidden_states = self.activation(hidden_states)
         hidden_states = self.layer_norm(hidden_states)
@@ -362,7 +337,7 @@ def __init__(self,
                                                 activation, embedding_weights)
         self.seq_relationship = nn.Linear(hidden_size, 2)
 
-    def forward(self, encoder_output, pooled_output, masked_positions=None):
+    def forward(self, encoder_output, pooled_output, masked_lm_labels):
         """
         Args:
             sequence_output(Tensor):
@@ -384,7 +359,12 @@ def forward(self, encoder_output, pooled_output, masked_positions=None):
                 A Tensor of shape [batch_size, 2] with the scores of next sentence prediction.
                 Its data type should be float32.
         """
-        prediction_scores = self.predictions(encoder_output, masked_positions)
+
+        sequence_flattened = paddle.index_select(
+            encoder_output.reshape([-1, encoder_output.shape[-1]]),
+            paddle.nonzero(masked_lm_labels.reshape([-1]) != -1).squeeze(),
+            axis=0)
+        prediction_scores = self.predictions(sequence_flattened)
         seq_relationship_score = self.seq_relationship(pooled_output)
         return prediction_scores, seq_relationship_score
 
@@ -406,18 +386,13 @@ def __init__(self, bert_config):
             bert_config.hidden_act,
             embedding_weights=self.bert.embeddings.word_embeddings.weight)
 
-    def forward(self,
-                input_ids,
-                token_type_ids=None,
-                position_ids=None,
-                attention_mask=None,
-                masked_positions=None):
+    def forward(self, input_ids, token_type_ids, attention_mask,
+                masked_lm_labels):
         """
 
         Args:
             input_ids(Tensor): See class `BertModel`.
             token_type_ids(Tensor, optional): See class `BertModel`.
-            position_ids(Tensor, optional): See class `BertModel`.
             attention_mask(Tensor, optional): See class `BertModel`.
             masked_positions(Tensor, optional): See class `BertPretrainingHeads`.
 
@@ -434,9 +409,8 @@ def forward(self,
             outputs = self.bert(
                 input_ids,
                 token_type_ids=token_type_ids,
-                position_ids=position_ids,
                 attention_mask=attention_mask)
             sequence_output, pooled_output = outputs[:2]
             prediction_scores, seq_relationship_score = self.cls(
-                sequence_output, pooled_output, masked_positions)
+                sequence_output, pooled_output, masked_lm_labels)
             return prediction_scores, seq_relationship_score