tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 03:55:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

swift : fix prompt tokenization logic (#4321)

Browse Source
This commit is contained in:
Miwa / Ensan
2023-12-04 22:43:45 +09:00
committed by GitHub
parent 4fa44e84ad
commit 5c9f90cba1
2 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
View File

@ -147,9 +147,10 @@ actor LlamaContext {
}
private func tokenize(text: String, add_bos: Bool) -> [llama_token] {
let n_tokens = text.count + (add_bos ? 1 : 0)
let utf8Count = text.utf8.count
let n_tokens = utf8Count + (add_bos ? 1 : 0)
let tokens = UnsafeMutablePointer<llama_token>.allocate(capacity: n_tokens)
let tokenCount = llama_tokenize(model, text, Int32(text.count), tokens, Int32(n_tokens), add_bos, false)
let tokenCount = llama_tokenize(model, text, Int32(utf8Count), tokens, Int32(n_tokens), add_bos, false)
var swiftTokens: [llama_token] = []
for i in 0..<tokenCount {