tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-08-29 03:28:52 -04:00
Code Issues Packages Projects Releases Wiki Activity

CUDA: stream-k decomposition for MMQ (#8018)

Browse Source 
* CUDA: stream-k decomposition for MMQ

* fix undefined memory reads for small matrices
This commit is contained in:
Johannes Gäßler
2024-06-20 14:39:21 +02:00
committed by GitHub
parent 2075a66a96
commit d50f8897a7
4 changed files with 292 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ggml-cuda.cu
View File

@@ -635,7 +635,7 @@ static int64_t get_row_rounding(const std::array<float, GGML_CUDA_MAX_DEVICES> &
}
const int cc = ggml_cuda_info().devices[id].cc;
row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc, get_mmq_x_max_host(cc)));
row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc));
}
return row_rounding;
}