tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 03:55:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : require first token to be BOS (#1303)

Browse Source 
* llama : require first token to be BOS

* scripts : add ppl-run-all.sh

* perplexity : add BOS for each chunk

* readme : update perplexity values after BOS fix

* perplexity : add clarifying comments
This commit is contained in:
Georgi Gerganov
2023-05-08 17:41:54 +03:00
committed by GitHub
parent 95078cc554
commit f9a6364912
7 changed files with 116 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/main/main.cpp
View File

@ -313,7 +313,8 @@ int main(int argc, char ** argv) {
if (n_past + (int) embd.size() > n_ctx) {
const int n_left = n_past - params.n_keep;
n_past = params.n_keep;
// always keep the first token - BOS
n_past = std::max(1, params.n_keep);
// insert n_left/2 tokens at the start of embd from last_n_tokens
embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size());
@ -331,7 +332,6 @@ int main(int argc, char ** argv) {
}
// try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
// REVIEW
if (n_session_consumed < (int) session_tokens.size()) {
size_t i = 0;
for ( ; i < embd.size(); i++) {