修改读取测试数据的位置

Author: WillQvQ

tests/core/test_dataset.py

@@ -228,7 +228,7 @@ def test_apply2(self):
         def split_sent(ins):
             return ins['raw_sentence'].split()
         csv_loader = CSVLoader(headers=['raw_sentence', 'label'], sep='\t')
-        data_bundle = csv_loader.load('test/data_for_tests/tutorial_sample_dataset.csv')
+        data_bundle = csv_loader.load('tests/data_for_tests/tutorial_sample_dataset.csv')
         dataset = data_bundle.datasets['train']
         dataset.drop(lambda x: len(x['raw_sentence'].split()) == 0, inplace=True)
         dataset.apply(split_sent, new_field_name='words', is_input=True)

tests/core/test_utils.py

@@ -120,8 +120,8 @@ class TestCache(unittest.TestCase):
     def test_cache_save(self):
         try:
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
-                                             'test/data_for_tests/cws_train')
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
+                                             'tests/data_for_tests/cws_train')
             end_time = time.time()
             pre_time = end_time - start_time
             with open('test/demo1.pkl', 'rb') as f:
@@ -130,8 +130,8 @@ def test_cache_save(self):
             for i in range(embed.shape[0]):
                 self.assertListEqual(embed[i].tolist(), _embed[i].tolist())
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
-                                             'test/data_for_tests/cws_train')
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
+                                             'tests/data_for_tests/cws_train')
             end_time = time.time()
             read_time = end_time - start_time
             print("Read using {:.3f}, while prepare using:{:.3f}".format(read_time, pre_time))
@@ -142,7 +142,7 @@ def test_cache_save(self):
     def test_cache_save_overwrite_path(self):
         try:
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', 'test/data_for_tests/cws_train',
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', 'tests/data_for_tests/cws_train',
                                              _cache_fp='test/demo_overwrite.pkl')
             end_time = time.time()
             pre_time = end_time - start_time
@@ -152,8 +152,8 @@ def test_cache_save_overwrite_path(self):
             for i in range(embed.shape[0]):
                 self.assertListEqual(embed[i].tolist(), _embed[i].tolist())
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
-                                             'test/data_for_tests/cws_train',
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
+                                             'tests/data_for_tests/cws_train',
                                              _cache_fp='test/demo_overwrite.pkl')
             end_time = time.time()
             read_time = end_time - start_time
@@ -165,8 +165,8 @@ def test_cache_save_overwrite_path(self):
     def test_cache_refresh(self):
         try:
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
-                                             'test/data_for_tests/cws_train',
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
+                                             'tests/data_for_tests/cws_train',
                                              _refresh=True)
             end_time = time.time()
             pre_time = end_time - start_time
@@ -176,8 +176,8 @@ def test_cache_refresh(self):
             for i in range(embed.shape[0]):
                 self.assertListEqual(embed[i].tolist(), _embed[i].tolist())
             start_time = time.time()
-            embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
-                                             'test/data_for_tests/cws_train',
+            embed, vocab, d = process_data_1('tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt',
+                                             'tests/data_for_tests/cws_train',
                                              _refresh=True)
             end_time = time.time()
             read_time = end_time - start_time

tests/embeddings/test_bert_embedding.py

@@ -32,22 +32,22 @@ def test_word_drop(self):
 class TestBertEmbedding(unittest.TestCase):
     def test_bert_embedding_1(self):
         vocab = Vocabulary().add_word_lst("this is a test . [SEP] NotInBERT".split())
-        embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1)
+        embed = BertEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.1)
         requires_grad = embed.requires_grad
         embed.requires_grad = not requires_grad
         embed.train()
         words = torch.LongTensor([[2, 3, 4, 0]])
         result = embed(words)
         self.assertEqual(result.size(), (1, 4, 16))
 
-        embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1)
+        embed = BertEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.1)
         embed.eval()
         words = torch.LongTensor([[2, 3, 4, 0]])
         result = embed(words)
         self.assertEqual(result.size(), (1, 4, 16))
 
         # 自动截断而不报错
-        embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1,
+        embed = BertEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.1,
                               auto_truncate=True)
 
         words = torch.LongTensor([[2, 3, 4, 1]*10,
@@ -60,7 +60,7 @@ def test_save_load(self):
         try:
             os.makedirs(bert_save_test, exist_ok=True)
             vocab = Vocabulary().add_word_lst("this is a test . [SEP] NotInBERT".split())
-            embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1,
+            embed = BertEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.1,
                                   auto_truncate=True)
 
             embed.save(bert_save_test)
@@ -76,15 +76,15 @@ def test_save_load(self):
 
 class TestBertWordPieceEncoder(unittest.TestCase):
     def test_bert_word_piece_encoder(self):
-        embed = BertWordPieceEncoder(model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1)
+        embed = BertWordPieceEncoder(model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.1)
         ds = DataSet({'words': ["this is a test . [SEP]".split()]})
         embed.index_datasets(ds, field_name='words')
         self.assertTrue(ds.has_field('word_pieces'))
         result = embed(torch.LongTensor([[1,2,3,4]]))
 
     def test_bert_embed_eq_bert_piece_encoder(self):
         ds = DataSet({'words': ["this is a texta model vocab".split(), 'this is'.split()]})
-        encoder = BertWordPieceEncoder(model_dir_or_name='test/data_for_tests/embedding/small_bert')
+        encoder = BertWordPieceEncoder(model_dir_or_name='tests/data_for_tests/embedding/small_bert')
         encoder.eval()
         encoder.index_datasets(ds, field_name='words')
         word_pieces = torch.LongTensor(ds['word_pieces'].get([0, 1]))
@@ -95,7 +95,7 @@ def test_bert_embed_eq_bert_piece_encoder(self):
         vocab.index_dataset(ds, field_name='words', new_field_name='words')
         ds.set_input('words')
         words = torch.LongTensor(ds['words'].get([0, 1]))
-        embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert',
+        embed = BertEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_bert',
                               pool_method='first', include_cls_sep=True, pooled_cls=False, min_freq=1)
         embed.eval()
         words_res = embed(words)
@@ -109,7 +109,7 @@ def test_save_load(self):
         bert_save_test = 'bert_save_test'
         try:
             os.makedirs(bert_save_test, exist_ok=True)
-            embed = BertWordPieceEncoder(model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.0,
+            embed = BertWordPieceEncoder(model_dir_or_name='tests/data_for_tests/embedding/small_bert', word_dropout=0.0,
                                          layers='-2')
             ds = DataSet({'words': ["this is a test . [SEP]".split()]})
             embed.index_datasets(ds, field_name='words')

tests/embeddings/test_elmo_embedding.py

@@ -21,7 +21,7 @@ def test_download_small(self):
 class TestRunElmo(unittest.TestCase):
     def test_elmo_embedding(self):
         vocab = Vocabulary().add_word_lst("This is a test .".split())
-        elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_elmo', layers='0,1')
+        elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_elmo', layers='0,1')
         words = torch.LongTensor([[0, 1, 2]])
         hidden = elmo_embed(words)
         print(hidden.size())
@@ -30,7 +30,7 @@ def test_elmo_embedding(self):
     def test_elmo_embedding_layer_assertion(self):
         vocab = Vocabulary().add_word_lst("This is a test .".split())
         try:
-            elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_elmo',
+            elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='tests/data_for_tests/embedding/small_elmo',
                                        layers='0,1,2')
         except AssertionError as e:
             print(e)

tests/embeddings/test_gpt2_embedding.py

@@ -21,7 +21,7 @@ def test_download(self):
             print(embed(words).size())
 
     def test_gpt2_embedding(self):
-        weight_path = 'test/data_for_tests/embedding/small_gpt2'
+        weight_path = 'tests/data_for_tests/embedding/small_gpt2'
         vocab = Vocabulary().add_word_lst("this is a texta sentence".split())
         embed = GPT2Embedding(vocab, model_dir_or_name=weight_path, word_dropout=0.1)
         requires_grad = embed.requires_grad
@@ -49,7 +49,7 @@ def test_gpt2_embedding(self):
     def test_gpt2_ebembedding_2(self):
         # 测试only_use_pretrain_vocab与truncate_embed是否正常工作
         Embedding = GPT2Embedding
-        weight_path = 'test/data_for_tests/embedding/small_gpt2'
+        weight_path = 'tests/data_for_tests/embedding/small_gpt2'
         vocab = Vocabulary().add_word_lst("this is a texta and".split())
         embed1 = Embedding(vocab, model_dir_or_name=weight_path,layers=list(range(3)),
                               only_use_pretrain_bpe=True, truncate_embed=True, min_freq=1)
@@ -89,13 +89,13 @@ def test_gpt2_ebembedding_2(self):
     def test_gpt2_tokenizer(self):
         from fastNLP.modules.tokenizer import GPT2Tokenizer
 
-        tokenizer = GPT2Tokenizer.from_pretrained('test/data_for_tests/embedding/small_gpt2')
+        tokenizer = GPT2Tokenizer.from_pretrained('tests/data_for_tests/embedding/small_gpt2')
         print(tokenizer.encode("this is a texta a sentence"))
         print(tokenizer.encode('this is'))
 
     def test_gpt2_embed_eq_gpt2_piece_encoder(self):
         # 主要检查一下embedding的结果与wordpieceencoder的结果是否一致
-        weight_path = 'test/data_for_tests/embedding/small_gpt2'
+        weight_path = 'tests/data_for_tests/embedding/small_gpt2'
         ds = DataSet({'words': ["this is a texta a sentence".split(), 'this is'.split()]})
         encoder = GPT2WordPieceEncoder(model_dir_or_name=weight_path)
         encoder.eval()
@@ -187,7 +187,7 @@ def test_generate_small_gpt2(self):
 
         print(used_pairs)
         import json
-        with open('test/data_for_tests/embedding/small_gpt2/vocab.json', 'w') as f:
+        with open('tests/data_for_tests/embedding/small_gpt2/vocab.json', 'w') as f:
             new_used_vocab = {}
             for idx, key in enumerate(used_vocab.keys()):
                 new_used_vocab[key] = len(new_used_vocab)
@@ -201,12 +201,12 @@ def test_generate_small_gpt2(self):
 
             json.dump(new_used_vocab, f)
 
-        with open('test/data_for_tests/embedding/small_gpt2/merges.txt', 'w') as f:
+        with open('tests/data_for_tests/embedding/small_gpt2/merges.txt', 'w') as f:
             f.write('#version: small\n')
             for k,v in sorted(sorted(used_pairs.items(), key=lambda kv:kv[1])):
                 f.write('{} {}\n'.format(k[0], k[1]))
 
-        new_tokenizer = GPT2Tokenizer.from_pretrained('test/data_for_tests/embedding/small_gpt2')
+        new_tokenizer = GPT2Tokenizer.from_pretrained('tests/data_for_tests/embedding/small_gpt2')
         new_all_tokens = []
         for sent in [sent1, sent2, sent3]:
             tokens = new_tokenizer.tokenize(sent, add_prefix_space=True)
@@ -227,21 +227,21 @@ def test_generate_small_gpt2(self):
                       "n_positions": 20,
                       "vocab_size": len(new_used_vocab)
                     }
-        with open('test/data_for_tests/embedding/small_gpt2/config.json', 'w') as f:
+        with open('tests/data_for_tests/embedding/small_gpt2/config.json', 'w') as f:
             json.dump(config, f)
 
         # 生成更小的merges.txt与vocab.json, 方法是通过记录tokenizer中的值实现
         from fastNLP.modules.encoder.gpt2 import GPT2LMHeadModel, GPT2Config
 
-        config = GPT2Config.from_pretrained('test/data_for_tests/embedding/small_gpt2')
+        config = GPT2Config.from_pretrained('tests/data_for_tests/embedding/small_gpt2')
 
         model = GPT2LMHeadModel(config)
-        torch.save(model.state_dict(), 'test/data_for_tests/embedding/small_gpt2/small_pytorch_model.bin')
+        torch.save(model.state_dict(), 'tests/data_for_tests/embedding/small_gpt2/small_pytorch_model.bin')
         print(model(torch.LongTensor([[0,1,2,3]])))
 
     def test_gpt2_word_piece_encoder(self):
         # 主要检查可以运行
-        weight_path = 'test/data_for_tests/embedding/small_gpt2'
+        weight_path = 'tests/data_for_tests/embedding/small_gpt2'
         ds = DataSet({'words': ["this is a test sentence".split()]})
         embed = GPT2WordPieceEncoder(model_dir_or_name=weight_path, word_dropout=0.1)
         embed.index_datasets(ds, field_name='words')
@@ -256,7 +256,7 @@ def test_gpt2_word_piece_encoder(self):
 
     @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis")
     def test_generate(self):
-        # weight_path = 'test/data_for_tests/embedding/small_gpt2'
+        # weight_path = 'tests/data_for_tests/embedding/small_gpt2'
         weight_path = 'en'
 
         encoder = GPT2WordPieceEncoder(model_dir_or_name=weight_path, language_model=True)