speculative: add --n-gpu-layers-draft option (#3063)

2025-06-27 03:55:20 +00:00 · 2023-09-13 08:50:46 +02:00
parent b52b29ab9d
commit 84e723653c
3 changed files with 15 additions and 0 deletions
--- a/examples/speculative/speculative.cpp
+++ b/examples/speculative/speculative.cpp
@ -42,6 +42,7 @@ int main(int argc, char ** argv) {

    // load the draft model
    params.model = params.model_draft;
+    params.n_gpu_layers = params.n_gpu_layers_draft;
    std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params);

    // tokenize the prompt