FINAL_LR_FRAC 0.0 to 0.05 (small LR floor)

2026-03-08 04:36:38 +00:00
parent 0640555a1f
commit 772dada6cc
1 changed files with 1 additions and 1 deletions
@@ -443,7 +443,7 @@ WEIGHT_DECAY = 0.2      # cautious weight decay for Muon
 ADAM_BETAS = (0.8, 0.95) # Adam beta1, beta2
 WARMUP_RATIO = 0.0      # fraction of time budget for LR warmup
 WARMDOWN_RATIO = 0.7    # fraction of time budget for LR warmdown
-FINAL_LR_FRAC = 0.0     # final LR as fraction of initial
+FINAL_LR_FRAC = 0.05    # final LR as fraction of initial

 # Model size
 DEPTH = 9               # number of transformer layers