From b17e9811f47adc32682042a238219ea3b37d7e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 26 May 2025 08:40:46 +0200 Subject: [PATCH] revert vocab_size() change [no ci] --- convert_hf_to_gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 0f2c41ecc..753c88e7c 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -3668,7 +3668,7 @@ class BertModel(TextModel): toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size if isinstance(tokenizer, SentencePieceProcessor): - for token_id in range(vocab_size): + for token_id in range(tokenizer.vocab_size()): piece = tokenizer.IdToPiece(token_id) text = piece.encode("utf-8") score = tokenizer.GetScore(token_id)