MediaMetz/autoresearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VE WD 0.002 to 0.003

Browse Source
This commit is contained in:
autoresearch
2026-03-08 13:54:07 +00:00
parent 73c77cad75
commit 637f82f215
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
train.py
+1 -1
View File
@@ -249,7 +249,7 @@ class GPT(nn.Module):
param_groups = [
dict(kind='adamw', params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.01),
dict(kind='adamw', params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.001),
dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.002),
dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale, betas=adam_betas, eps=1e-10, weight_decay=0.003),
dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=adam_betas, eps=1e-10, weight_decay=0.0),
dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0),
]