Merge branch 'revert-b73e835c' into 'main'

Revert "Merge branch 'feat/bert_large' into 'main'" See merge request deep-learning-nlp/token-tricksters!17
token-tricksters · Jul 27, 2023 · 9893c66 · 9893c66
2 parents 608f5ac + 6737696
commit 9893c66
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 8 deletions.
diff --git a/config.py b/config.py
@@ -186,7 +186,7 @@ class BertConfig(PretrainedConfig):
   def __init__(
     self,
     vocab_size=30522,
-    hidden_size=1024,
+    hidden_size=768,
     num_hidden_layers=12,
     num_attention_heads=12,
     intermediate_size=3072,

diff --git a/multitask_classifier.py b/multitask_classifier.py
@@ -32,6 +32,7 @@ def seed_everything(seed=11711):
     torch.backends.cudnn.deterministic = True
 
 
+BERT_HIDDEN_SIZE = 768
 N_SENTIMENT_CLASSES = 5
 
 
@@ -48,12 +49,13 @@ def __init__(self, config):
         super(MultitaskBERT, self).__init__()
         # You will want to add layers here to perform the downstream tasks.
         # Pretrain mode does not require updating bert paramters.
-        self.bert = BertModel.from_pretrained(config.model)
+        self.bert = BertModel.from_pretrained('bert-base-uncased')
         for param in self.bert.parameters():
             if config.option == 'pretrain':
                 param.requires_grad = False
             elif config.option == 'finetune':
                 param.requires_grad = True
+
         self.linear_layer = nn.Linear(config.hidden_size, N_SENTIMENT_CLASSES)
 
         self.paraphrase_linear = nn.Linear(config.hidden_size, config.hidden_size)
@@ -197,13 +199,10 @@ def train_multitask(args):
     sts_dev_dataloader = DataLoader(sts_dev_data, shuffle=False, batch_size=args.batch_size,
                                     collate_fn=sts_dev_data.collate_fn)
 
-    print(f"Model: {args.model}")
-    hidden_size = {'bert-base-uncased': 768, 'bert-large-uncased': 1024}
     # Init model
     config = {'hidden_dropout_prob': args.hidden_dropout_prob,
               'num_labels': num_labels,
-              'model': args.model,
-              'hidden_size': hidden_size[args.model],
+              'hidden_size': 768,
               'data_dir': '.',
               'option': args.option,
               'local_files_only': args.local_files_only}
@@ -375,8 +374,6 @@ def get_args():
     parser.add_argument("--option", type=str,
                         help='pretrain: the BERT parameters are frozen; finetune: BERT parameters are updated',
                         choices=('pretrain', 'finetune'), default="pretrain")
-    parser.add_argument("--model", type=str,
-                        choices=('bert-base-uncased', 'bert-large-uncased'), default="bert-base-uncased")
     parser.add_argument("--use_gpu", action='store_true')
 
     parser.add_argument("--sst_dev_out", type=str, default="predictions/sst-dev-output.csv")