tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-07-26 03:03:25 -04:00
Code Issues Packages Projects Releases Wiki Activity

metal : add GELU implementation (#1770)

Browse Source 
Co-authored-by: Adam Treat <adam@nomic.ai>
This commit is contained in:
AT
2023-06-09 04:00:51 -04:00
committed by GitHub
parent 245fc3c37d
commit 92f44ff7f7
2 changed files with 27 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ggml-metal.metal
View File

@ -81,6 +81,17 @@ kernel void kernel_relu(
dst[tpig] = max(0.0f, src0[tpig]);
}
constant float GELU_COEF_A = 0.044715f;
constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
kernel void kernel_gelu(
device const float * src0,
device float * dst,
uint tpig[[thread_position_in_grid]]) {
float x = src0[tpig];
dst[tpig] = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
}
kernel void kernel_soft_max(
device const float * src0,
device float * dst,