a better default rms_norm_eps

LostRuins · LostRuins · commit 0c26799e7775 · 2023-07-25T22:51:01.000+08:00
diff --git a/llama.cpp b/llama.cpp
@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(
     auto & hparams = model.hparams;
 
     // TODO: read from file
-    hparams.f_rms_norm_eps = rms_norm_eps;
+    hparams.f_rms_norm_eps = 5e-6;
 
     {
         switch (hparams.n_layer) {

Original file line number	Diff line number	Diff line change
`@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(`
`1030`	`1030`	`auto & hparams = model.hparams;`
`1031`	`1031`
`1032`	`1032`	`// TODO: read from file`
`1033`		`- hparams.f_rms_norm_eps = rms_norm_eps;`
	`1033`	`+ hparams.f_rms_norm_eps = 5e-6;`
`1034`	`1034`
`1035`	`1035`	`{`
`1036`	`1036`	`switch (hparams.n_layer) {`