tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-08-28 02:58:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

server : enable continuous batching by default (#6231)

Browse Source
This commit is contained in:
Georgi Gerganov
2024-03-22 13:08:28 +02:00
committed by GitHub
parent b3e94f26ba
commit 68e210b354
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/common.h
View File

@@ -139,7 +139,7 @@ struct gpt_params {
bool interactive_first = false; // wait for user input immediately
bool multiline_input = false; // reverse the usage of `\`
bool simple_io = false; // improves compatibility with subprocesses and limited consoles
bool cont_batching = false; // insert new sequences for decoding on-the-fly
bool cont_batching = true; // insert new sequences for decoding on-the-fly
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
bool ignore_eos = false; // ignore generated EOS tokens