tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-29 04:35:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

examplse : de-shadow

Browse Source 
ggml-ci
This commit is contained in:
Georgi Gerganov
2025-01-12 14:25:32 +02:00
parent 82caffa74e
commit 9a735ae6d8
16 changed files with 152 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
examples/server/utils.hpp
View File

@ -129,15 +129,15 @@ static llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_
if (p.is_string()) {
auto s = p.template get<std::string>();
llama_tokens p;
llama_tokens ids;
if (first) {
p = common_tokenize(vocab, s, add_special, parse_special);
ids = common_tokenize(vocab, s, add_special, parse_special);
first = false;
} else {
p = common_tokenize(vocab, s, false, parse_special);
ids = common_tokenize(vocab, s, false, parse_special);
}
prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());
prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.end());
} else {
if (first) {
first = false;