tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-28 04:15:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into xsn/private_batch_api

Browse Source
This commit is contained in:
Xuan Son Nguyen
2025-03-18 15:45:22 +01:00
parent eab5606d7b 8551c44d84
commit dc4bb64290
76 changed files with 3990 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
examples/server/server.cpp
View File

@ -1868,6 +1868,10 @@ struct server_context {
params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers;
params_dft.n_parallel = 1;
// force F16 KV cache for the draft model for extra performance
params_dft.cache_type_k = GGML_TYPE_F16;
params_dft.cache_type_v = GGML_TYPE_F16;
llama_init_dft = common_init_from_params(params_dft);
model_dft = llama_init_dft.model.get();
@ -1888,10 +1892,6 @@ struct server_context {
cparams_dft = common_context_params_to_llama(params_dft);
cparams_dft.n_batch = n_ctx_dft;
// force F16 KV cache for the draft model for extra performance
cparams_dft.type_k = GGML_TYPE_F16;
cparams_dft.type_v = GGML_TYPE_F16;
// the context is not needed - we will create one for each slot
llama_init_dft.context.reset();
}