tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-29 04:35:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add RMS norm and use it (#187)

Browse Source 
* add ggml_rms_norm

* update op num
This commit is contained in:
hoangmit
2023-03-15 18:41:38 -04:00
committed by GitHub
parent 27944c4206
commit 6eac39ba95
3 changed files with 134 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
main.cpp
View File

@ -588,7 +588,7 @@ bool llama_eval(
// norm
{
cur = ggml_norm(ctx0, inpL);
cur = ggml_rms_norm(ctx0, inpL);
// cur = attention_norm*cur
cur = ggml_mul(ctx0,
@ -678,7 +678,7 @@ bool llama_eval(
{
// norm
{
cur = ggml_norm(ctx0, inpFF);
cur = ggml_rms_norm(ctx0, inpFF);
// cur = ffn_norm*cur
cur = ggml_mul(ctx0,
@ -713,7 +713,7 @@ bool llama_eval(
// norm
{
inpL = ggml_norm(ctx0, inpL);
inpL = ggml_rms_norm(ctx0, inpL);
// inpL = norm*inpL
inpL = ggml_mul(ctx0,