diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 521a6ec5e..5d5cafbea 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2221,14 +2221,12 @@ void llama_vocab::impl::tokenizer_st_partition(std::forward_list length - if (match + text.length() > raw_text_base_offset + raw_text_base_length) break; - #ifdef PRETOKENIZERDEBUG LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); #endif