tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-07-27 19:53:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

revert vocab_size() change [no ci]

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2025-05-26 08:40:46 +02:00
committed by GitHub
parent f2d876ad5a
commit b17e9811f4
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
convert_hf_to_gguf.py
View File

@ -3668,7 +3668,7 @@ class BertModel(TextModel):
toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size
if isinstance(tokenizer, SentencePieceProcessor):
for token_id in range(vocab_size):
for token_id in range(tokenizer.vocab_size()):
piece = tokenizer.IdToPiece(token_id)
text = piece.encode("utf-8")
score = tokenizer.GetScore(token_id)