tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-07-02 05:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : rename jina tokenizers to v2 (#7249)

Browse Source 
* refactor: rename jina tokenizers to v2

* refactor: keep refactoring non-breaking
This commit is contained in:
Joan Fontanals
2024-05-13 10:35:14 +02:00
committed by GitHub
parent b1f8af1886
commit 9aa672490c
3 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llama.cpp
View File

@ -4424,7 +4424,9 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de") {
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
} else if (
tokenizer_pre == "refact") {