tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 12:05:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit from fork

Browse Source 
* vocab : prevent integer overflow during load

* Add static cast and GGML_ABORT

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Guy Goldenberg
2025-06-13 19:20:25 +03:00
committed by GitHub
parent 80709b70a2
commit 3cfbbdb44e
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llama-vocab.cpp
View File

@ -19,6 +19,7 @@
#include <set> #include <set>
#include <unordered_map> #include <unordered_map>
#include <cctype> #include <cctype>
#include <cinttypes>
// //
// helpers // helpers
@ -2572,6 +2573,10 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
// copy piece chars to output text buffer // copy piece chars to output text buffer
// skip up to 'lstrip' leading spaces before copying // skip up to 'lstrip' leading spaces before copying
auto _try_copy = [=] (const char * token, size_t size) -> int32_t { auto _try_copy = [=] (const char * token, size_t size) -> int32_t {
if (size >= static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
GGML_ABORT("invalid token size: %zu exceeds int32_t limit", size);
}
for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) { for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) {
token++; token++;
size--; size--;