revert interrupted softcap experiment (restore softcap=15)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-08 16:27:48 +00:00
parent fedfef398b
commit 6c087cb5e2
1 changed files with 1 additions and 1 deletions
@@ -278,7 +278,7 @@ class GPT(nn.Module):
            x = block(x, ve, cos_sin, self.window_sizes[i])
        x = norm(x)

-        softcap = 17
+        softcap = 15
        logits = self.lm_head(x)
        logits = logits.float()
        logits = softcap * torch.tanh(logits / softcap)