tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-30 12:55:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : keep track of all EOG tokens in the vocab (#9609)

Browse Source 
ggml-ci
This commit is contained in:
Georgi Gerganov
2024-09-24 10:16:06 +03:00
committed by GitHub
parent cea1486ecf
commit 31ac5834fe
3 changed files with 61 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llama-vocab.cpp
View File

@ -1570,11 +1570,7 @@ llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, lla
}
bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
return token != -1 && (
token == llama_token_eos_impl(vocab) ||
token == llama_token_eot_impl(vocab) ||
token == llama_token_eom_impl(vocab)
);
return token != -1 && vocab.special_eog_ids.count(token) > 0;
}
bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token) {