tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-07-21 18:28:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

mtmd : Use RMS norm for InternVL 3 38B and 78B mmproj (#13459)

Browse Source
This commit is contained in:
City
2025-05-12 00:39:06 +02:00
committed by GitHub
parent 9a390c4829
commit c104023994
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tools/mtmd/clip.cpp
View File

@ -879,9 +879,15 @@ struct clip_graph {
// add CLS token // add CLS token
inp = ggml_concat(ctx0, inp, model.class_embedding, 1); inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
// The larger models use a different ViT, which uses RMS norm instead of layer norm
// ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188
norm_type norm_t = (hparams.n_embd == 3200 && hparams.n_layer == 45)
? NORM_TYPE_RMS // 6B ViT (Used by InternVL 2.5/3 - 26B, 38B, 78B)
: NORM_TYPE_NORMAL; // 300M ViT (Used by all smaller InternVL models)
ggml_tensor * cur = build_vit( ggml_tensor * cur = build_vit(
inp, n_pos, inp, n_pos,
NORM_TYPE_NORMAL, norm_t,
hparams.ffn_op, hparams.ffn_op,
model.position_embeddings, model.position_embeddings,
nullptr); nullptr);