tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-07-02 05:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add LLAMA_DEFAULT_RMS_EPS so we can change the default (#2384)

Browse Source 
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
This commit is contained in:
Kawrakow
2023-07-25 18:35:53 +03:00
committed by GitHub
parent 07aaa0f63f
commit eb542d3932
5 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llama.cpp
View File

@ -186,7 +186,7 @@ struct llama_hparams {
// LLaMAv2
// TODO: load from model data hparams
float f_ffn_mult = 1.0f;
float f_rms_norm_eps = 1e-6f;
float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS;
float rope_freq_base = 10000.0f;
float rope_freq_scale = 1.0f;
@ -870,7 +870,7 @@ struct llama_context_params llama_context_default_params() {
/*.n_ctx =*/ 512,
/*.n_batch =*/ 512,
/*.n_gqa =*/ 1,
/*.rms_norm_eps =*/ 1e-6f,
/*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS,
/*.gpu_layers =*/ 0,
/*.main_gpu =*/ 0,
/*.tensor_split =*/ nullptr,