ChEB-AI · sfluegel05 · Mar 18, 2026 · Feb 18, 2026 · Feb 19, 2026 · Mar 18, 2026
diff --git a/configs/model/OPT_experiments/electra_LR.yml b/configs/model/OPT_experiments/electra_LR.yml
diff --git a/configs/model/OPT_experiments/electra_tox_expl.yml b/configs/model/OPT_experiments/electra_tox_expl.yml
diff --git a/configs/model/electra-for-pretraining.yml b/configs/model/electra-for-pretraining.yml
@@ -4,16 +4,16 @@ init_args:
     class_path: chebai.loss.pretraining.ElectraPreLoss
   out_dim: null
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3
   config:
     generator:
-      vocab_size: 1400
+      vocab_size: 4400
       max_position_embeddings: 1800
       num_attention_heads: 8
       num_hidden_layers: 6
       type_vocab_size: 1
     discriminator:
-      vocab_size: 1400
+      vocab_size: 4400
       max_position_embeddings: 1800
       num_attention_heads: 8
       num_hidden_layers: 6

diff --git a/configs/model/electra.yml b/configs/model/electra.yml
@@ -2,7 +2,7 @@ class_path: chebai.models.Electra
 init_args:
   model_type: classification
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3
   config:
     vocab_size: 4400
     max_position_embeddings: 1800

diff --git a/configs/model/electra300.yml b/configs/model/electra300.yml
@@ -1,9 +1,10 @@
 class_path: chebai.models.Electra
 init_args:
+  model_type: classification
   optimizer_kwargs:
     lr: 1e-3
   config:
-    vocab_size: 1400
+    vocab_size: 4400
     max_position_embeddings: 301
     num_attention_heads: 8
     num_hidden_layers: 6

diff --git a/configs/model/electra_pretraining.yml b/configs/model/electra_pretraining.yml
diff --git a/configs/model/electra_tox.yml b/configs/model/electra_tox.yml
@@ -2,7 +2,7 @@ class_path: chebai.models.Electra
 init_args:
   model_type: classification
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3 # we recommend  1e-4 for OPT finetuning, however, 1e-3 is the default
   config:
     vocab_size: 1400
     max_position_embeddings: 1800