diff --git a/common/common.h b/common/common.h index 4bcc4a2c4..c223685f2 100644 --- a/common/common.h +++ b/common/common.h @@ -607,7 +607,7 @@ struct common_batch { n_outputs++; } } - void add_text(llama_token token, llama_pos pos, std::vector seq_ids, bool logits) { + void add_text_multi_seq(llama_token token, llama_pos pos, std::vector seq_ids, bool logits) { llama_batch_ext_add_text(batch.get(), token, pos, seq_ids.data(), seq_ids.size(), logits); tokens.push_back({token, seq_ids[0], logits}); if (logits) { diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index 40aa0876f..b4b6e63c7 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -20,7 +20,8 @@ static bool eval_tokens(struct llama_context * ctx_llama, std::vector n_batch) { n_eval = n_batch; } - if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval))) { + llama_batch_ext_ptr batch(llama_batch_ext_init_from_text(&tokens[i], n_eval, *n_past, 0)); + if (llama_decode_ext(ctx_llama, batch.get())) { LOG_ERR("%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, N, n_batch, *n_past); return false; } diff --git a/examples/llava/minicpmv-cli.cpp b/examples/llava/minicpmv-cli.cpp index 12f536cf5..adc3a615f 100644 --- a/examples/llava/minicpmv-cli.cpp +++ b/examples/llava/minicpmv-cli.cpp @@ -101,7 +101,8 @@ static bool eval_tokens(struct llama_context * ctx_llama, std::vector n_batch) { n_eval = n_batch; } - if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval))) { + llama_batch_ext_ptr batch(llama_batch_ext_init_from_text(&tokens[i], n_eval, *n_past, 0)); + if (llama_decode_ext(ctx_llama, batch.get())) { LOG_ERR("%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, N, n_batch, *n_past); return false; } diff --git a/examples/llava/qwen2vl-cli.cpp b/examples/llava/qwen2vl-cli.cpp index d65e88f9d..1f4242580 100644 --- a/examples/llava/qwen2vl-cli.cpp +++ b/examples/llava/qwen2vl-cli.cpp @@ -96,16 +96,24 @@ static bool eval_tokens(struct llama_context * ctx_llama, std::vector n_batch) { n_eval = n_batch; } - auto batch = llama_batch_get_one(&tokens[i], n_eval); - // TODO: add mrope pos ids somewhere else - pos.resize(batch.n_tokens * 4); - std::fill(pos.begin(), pos.end(), 0); - for (int j = 0; j < batch.n_tokens * 3; j ++) { - pos[j] = *st_pos_id + (j % batch.n_tokens); - } - batch.pos = pos.data(); - if (llama_decode(ctx_llama, batch)) { + // TODO: add mrope pos ids somewhere else + int n_tokens = n_eval; + pos.resize(n_tokens * 4); + std::fill(pos.begin(), pos.end(), 0); + for (int j = 0; j < n_tokens * 3; j ++) { + pos[j] = *st_pos_id + (j % n_tokens); + } + + llama_batch_ext_ptr batch(llama_batch_ext_init(n_eval, 1)); + for (int j = 0; j < n_eval; j++) { + llama_token token = tokens[i + j]; + llama_seq_id seq_id = 0; + llama_batch_ext_add_text(batch.get(), token, pos[j], &seq_id, 1, false); + } + llama_batch_ext_set_output_last(batch.get()); + + if (llama_decode_ext(ctx_llama, batch.get())) { LOG_ERR("%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, N, n_batch, *n_past); return false; } diff --git a/examples/lookahead/lookahead.cpp b/examples/lookahead/lookahead.cpp index 1c2c3ec46..1e8de9673 100644 --- a/examples/lookahead/lookahead.cpp +++ b/examples/lookahead/lookahead.cpp @@ -92,8 +92,10 @@ int main(int argc, char ** argv) { const auto t_enc_start = ggml_time_us(); // eval the prompt - llama_decode(ctx, llama_batch_get_one( inp.data(), n_input - 1)); - llama_decode(ctx, llama_batch_get_one(&inp.back(), 1)); + llama_batch_ext_ptr batch0(llama_batch_ext_init_from_text( inp.data(), n_input - 1, 0, 0)); + llama_batch_ext_ptr batch1(llama_batch_ext_init_from_text(&inp.back(), 1, 0, 0)); + llama_decode_ext(ctx, batch0.get()); + llama_decode_ext(ctx, batch1.get()); for (int s = 1; s < W + G + 1; ++s) { llama_kv_self_seq_cp(ctx, 0, s, -1, -1); diff --git a/examples/main/main.cpp b/examples/main/main.cpp index fd7410a64..2f735c420 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -548,7 +548,8 @@ int main(int argc, char ** argv) { int enc_input_size = embd_inp.size(); llama_token * enc_input_buf = embd_inp.data(); - if (llama_encode(ctx, llama_batch_get_one(enc_input_buf, enc_input_size))) { + llama_batch_ext_ptr batch(llama_batch_ext_init_from_text(enc_input_buf, enc_input_size, 0, 0)); + if (llama_decode_ext(ctx, batch.get())) { LOG_ERR("%s : failed to eval\n", __func__); return 1; } @@ -668,7 +669,8 @@ int main(int argc, char ** argv) { LOG_DBG("eval: %s\n", string_from(ctx, embd).c_str()); - if (llama_decode(ctx, llama_batch_get_one(&embd[i], n_eval))) { + llama_batch_ext_ptr batch(llama_batch_ext_init_from_text(&embd[i], n_eval, 0, 0)); + if (llama_decode_ext(ctx, batch.get())) { LOG_ERR("%s : failed to eval\n", __func__); return 1; } diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index d24fddbf4..956c115d4 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -565,7 +565,6 @@ static results_perplexity perplexity(llama_context * ctx, const common_params & } for (int k = 0; k < batch_size; ++k) { - const int idx = seq*n_ctx + k; const llama_pos pos = j*n_batch + k; bool output = pos >= first; batch.add_text(tokens[seq_start + k], pos, seq, output); @@ -876,7 +875,7 @@ static void hellaswag_score(llama_context * ctx, const common_params & params) { } for (size_t i = 0; i < hs_cur.common_prefix; ++i) { - batch.add_text(hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false); + batch.add_text_multi_seq(hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false); } llama_batch_ext_set_output_last(batch.get()); n_logits += 1; @@ -886,7 +885,7 @@ static void hellaswag_score(llama_context * ctx, const common_params & params) { // TODO: don't evaluate the last token of each sequence for (size_t i = hs_cur.common_prefix; i < seq_tokens_size; ++i) { const bool needs_logits = i < seq_tokens_size - 1; - batch.add_text(hs_cur.seq_tokens[s][i], i, { s0 + s }, needs_logits); + batch.add_text_multi_seq(hs_cur.seq_tokens[s][i], i, { s0 + s }, needs_logits); n_logits += needs_logits; } } @@ -1155,7 +1154,7 @@ static void winogrande_score(llama_context * ctx, const common_params & params) } for (size_t i = 0; i < data[i1].common_prefix; ++i) { - batch.add_text(data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false); + batch.add_text_multi_seq(data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false); } llama_batch_ext_set_output_last(batch.get()); n_logits += 1; @@ -1163,7 +1162,7 @@ static void winogrande_score(llama_context * ctx, const common_params & params) for (int s = 0; s < 2; ++s) { // TODO: end before the last token, no need to predict past the end of the sequences for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size(); ++i) { - batch.add_text(data[i1].seq_tokens[s][i], i, { s0 + s }, true); + batch.add_text_multi_seq(data[i1].seq_tokens[s][i], i, { s0 + s }, true); n_logits += 1; } } @@ -1523,7 +1522,7 @@ static void multiple_choice_score(llama_context * ctx, const common_params & par for (size_t i = 0; i < cur_task.common_prefix; ++i) { //llama_batch_add(batch, cur_task.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false); - batch.add_text(cur_task.seq_tokens[0][i], i, batch_indeces, false); + batch.add_text_multi_seq(cur_task.seq_tokens[0][i], i, batch_indeces, false); } llama_batch_ext_set_output_last(batch.get()); // we need logits for the last token of the common prefix n_logits += 1; @@ -1533,7 +1532,7 @@ static void multiple_choice_score(llama_context * ctx, const common_params & par // TODO: don't evaluate the last token of each sequence for (size_t i = cur_task.common_prefix; i < seq_tokens_size; ++i) { const bool needs_logits = i < seq_tokens_size - 1; - batch.add_text(cur_task.seq_tokens[s][i], i, { s0 + s }, needs_logits); + batch.add_text_multi_seq(cur_task.seq_tokens[s][i], i, { s0 + s }, needs_logits); n_logits += needs_logits; } } @@ -1760,7 +1759,7 @@ static void kl_divergence(llama_context * ctx, const common_params & params) { batch.clear(); for (int i = 0; i < batch_size; i++) { - batch.add_text(tokens[batch_start + i], j*n_batch + i, {0}, true); + batch.add_text_multi_seq(tokens[batch_start + i], j*n_batch + i, {0}, true); } if (llama_decode_ext(ctx, batch.get())) { diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index e61e863ce..2f4a85abd 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -113,7 +113,8 @@ int main(int argc, char ** argv) { struct common_sampler * smpl = common_sampler_init(model_tgt, params.sampling); // eval the prompt - llama_decode(ctx_tgt, llama_batch_get_one(inp.data(), inp.size() - 1)); + llama_batch_ext_ptr batch(llama_batch_ext_init_from_text(inp.data(), inp.size() - 1, 0, 0)); + llama_decode_ext(ctx_tgt, batch.get()); // note: keep the last token separate! llama_token id_last = inp.back(); diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 1f55db7b6..2d44dc82c 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -45,7 +45,7 @@ int main(int argc, char ** argv) { } common_init(); -#ifdef 0 +#if 0 if (params.speculative.model.empty()) { LOG_ERR("%s: --model-draft is required\n", __func__); return 1; @@ -166,9 +166,12 @@ int main(int argc, char ** argv) { const auto t_enc_start = ggml_time_us(); // eval the prompt with both models - llama_decode(ctx_tgt, llama_batch_get_one( inp.data(), n_input - 1)); - llama_decode(ctx_tgt, llama_batch_get_one(&inp.back(), 1)); - llama_decode(ctx_dft, llama_batch_get_one( inp.data(), n_input)); + llama_batch_ext_ptr batch0(llama_batch_ext_init_from_text( inp.data(), n_input - 1, 0, 0)); + llama_batch_ext_ptr batch1(llama_batch_ext_init_from_text(&inp.back(), 1, 0, 0)); + llama_batch_ext_ptr batch2(llama_batch_ext_init_from_text( inp.data(), n_input , 0, 0)); + llama_decode_ext(ctx_tgt, batch0); + llama_decode_ext(ctx_tgt, batch1); + llama_decode_ext(ctx_dft, batch2); const auto t_enc_end = ggml_time_us();