tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-08-15 04:33:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into cisc/jina-embeddings-v3

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2025-07-20 20:16:05 +02:00
committed by GitHub
parent e092776f9b 2be60cbc27
commit 80094033fa
154 changed files with 41058 additions and 3977 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
tools/server/server.cpp
View File

@@ -127,7 +127,6 @@ struct slot_params {
std::vector<std::string> response_fields;
bool timings_per_token = false;
bool post_sampling_probs = false;
bool ignore_eos = false;
struct common_params_sampling sampling;
struct common_params_speculative speculative;
@@ -441,7 +440,6 @@ struct server_task {
{
params.sampling.logit_bias.clear();
params.ignore_eos = json_value(data, "ignore_eos", false);
const auto & logit_bias = data.find("logit_bias");
if (logit_bias != data.end() && logit_bias->is_array()) {
@@ -472,6 +470,13 @@ struct server_task {
}
}
}
params.sampling.ignore_eos = json_value(data, "ignore_eos", params_base.sampling.ignore_eos);
if (params.sampling.ignore_eos) {
params.sampling.logit_bias.insert(
params.sampling.logit_bias.end(),
defaults.sampling.logit_bias_eog.begin(), defaults.sampling.logit_bias_eog.end());
}
}
{
@@ -1898,7 +1903,6 @@ struct server_context {
bool clean_kv_cache = true;
bool add_bos_token = true;
bool has_eos_token = false;
int32_t n_ctx; // total context for all clients / slots
@@ -1957,7 +1961,6 @@ struct server_context {
n_ctx = llama_n_ctx(ctx);
add_bos_token = llama_vocab_get_add_bos(vocab);
has_eos_token = llama_vocab_eos(vocab) != LLAMA_TOKEN_NULL;
if (!params_base.speculative.model.path.empty() || !params_base.speculative.model.hf_repo.empty()) {
SRV_INF("loading draft model '%s'\n", params_base.speculative.model.path.c_str());
@@ -2217,10 +2220,6 @@ struct server_context {
slot.params.n_predict = slot.n_predict;
}
if (slot.params.ignore_eos && has_eos_token) {
slot.params.sampling.logit_bias.push_back({llama_vocab_eos(vocab), -INFINITY});
}
{
if (slot.smpl != nullptr) {
common_sampler_free(slot.smpl);
@@ -2581,12 +2580,14 @@ struct server_context {
continue;
}
const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);
if (embd == NULL) {
const float * embd = nullptr;
if (llama_pooling_type(slot.ctx) == LLAMA_POOLING_TYPE_NONE) {
embd = llama_get_embeddings_ith(ctx, i);
} else {
embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);
}
if (embd == NULL) {
if (embd == nullptr) {
SLT_ERR(slot, "failed to get embeddings, token = %d, seq_id = %d\n", batch.token[i], batch.seq_id[i][0]);
res->embedding.push_back(std::vector<float>(n_embd, 0.0f));
@@ -2594,12 +2595,12 @@ struct server_context {
}
// normalize only when there is pooling
// TODO: configurable
if (llama_pooling_type(slot.ctx) != LLAMA_POOLING_TYPE_NONE) {
common_embd_normalize(embd, embd_res.data(), n_embd, 2);
res->embedding.push_back(embd_res);
break;
} else {
res->embedding.push_back({ embd, embd + n_embd });
res->embedding.emplace_back(embd, embd + n_embd);
}
}
@@ -4808,14 +4809,14 @@ int main(int argc, char ** argv) {
// register static assets routes
if (!params.public_path.empty()) {
// Set the base directory for serving static files
bool is_found = svr->set_mount_point("/", params.public_path);
bool is_found = svr->set_mount_point(params.api_prefix + "/", params.public_path);
if (!is_found) {
LOG_ERR("%s: static assets path not found: %s\n", __func__, params.public_path.c_str());
return 1;
}
} else {
// using embedded static index.html
svr->Get("/", [](const httplib::Request & req, httplib::Response & res) {
svr->Get(params.api_prefix + "/", [](const httplib::Request & req, httplib::Response & res) {
if (req.get_header_value("Accept-Encoding").find("gzip") == std::string::npos) {
res.set_content("Error: gzip is not supported by this browser", "text/plain");
} else {
@@ -4831,37 +4832,37 @@ int main(int argc, char ** argv) {
}
// register API routes
svr->Get ("/health", handle_health); // public endpoint (no API key check)
svr->Get ("/metrics", handle_metrics);
svr->Get ("/props", handle_props);
svr->Post("/props", handle_props_change);
svr->Post("/api/show", handle_api_show);
svr->Get ("/models", handle_models); // public endpoint (no API key check)
svr->Get ("/v1/models", handle_models); // public endpoint (no API key check)
svr->Get ("/api/tags", handle_models); // ollama specific endpoint. public endpoint (no API key check)
svr->Post("/completion", handle_completions); // legacy
svr->Post("/completions", handle_completions);
svr->Post("/v1/completions", handle_completions_oai);
svr->Post("/chat/completions", handle_chat_completions);
svr->Post("/v1/chat/completions", handle_chat_completions);
svr->Post("/api/chat", handle_chat_completions); // ollama specific endpoint
svr->Post("/infill", handle_infill);
svr->Post("/embedding", handle_embeddings); // legacy
svr->Post("/embeddings", handle_embeddings);
svr->Post("/v1/embeddings", handle_embeddings_oai);
svr->Post("/rerank", handle_rerank);
svr->Post("/reranking", handle_rerank);
svr->Post("/v1/rerank", handle_rerank);
svr->Post("/v1/reranking", handle_rerank);
svr->Post("/tokenize", handle_tokenize);
svr->Post("/detokenize", handle_detokenize);
svr->Post("/apply-template", handle_apply_template);
svr->Get (params.api_prefix + "/health", handle_health); // public endpoint (no API key check)
svr->Get (params.api_prefix + "/metrics", handle_metrics);
svr->Get (params.api_prefix + "/props", handle_props);
svr->Post(params.api_prefix + "/props", handle_props_change);
svr->Post(params.api_prefix + "/api/show", handle_api_show);
svr->Get (params.api_prefix + "/models", handle_models); // public endpoint (no API key check)
svr->Get (params.api_prefix + "/v1/models", handle_models); // public endpoint (no API key check)
svr->Get (params.api_prefix + "/api/tags", handle_models); // ollama specific endpoint. public endpoint (no API key check)
svr->Post(params.api_prefix + "/completion", handle_completions); // legacy
svr->Post(params.api_prefix + "/completions", handle_completions);
svr->Post(params.api_prefix + "/v1/completions", handle_completions_oai);
svr->Post(params.api_prefix + "/chat/completions", handle_chat_completions);
svr->Post(params.api_prefix + "/v1/chat/completions", handle_chat_completions);
svr->Post(params.api_prefix + "/api/chat", handle_chat_completions); // ollama specific endpoint
svr->Post(params.api_prefix + "/infill", handle_infill);
svr->Post(params.api_prefix + "/embedding", handle_embeddings); // legacy
svr->Post(params.api_prefix + "/embeddings", handle_embeddings);
svr->Post(params.api_prefix + "/v1/embeddings", handle_embeddings_oai);
svr->Post(params.api_prefix + "/rerank", handle_rerank);
svr->Post(params.api_prefix + "/reranking", handle_rerank);
svr->Post(params.api_prefix + "/v1/rerank", handle_rerank);
svr->Post(params.api_prefix + "/v1/reranking", handle_rerank);
svr->Post(params.api_prefix + "/tokenize", handle_tokenize);
svr->Post(params.api_prefix + "/detokenize", handle_detokenize);
svr->Post(params.api_prefix + "/apply-template", handle_apply_template);
// LoRA adapters hotswap
svr->Get ("/lora-adapters", handle_lora_adapters_list);
svr->Post("/lora-adapters", handle_lora_adapters_apply);
svr->Get (params.api_prefix + "/lora-adapters", handle_lora_adapters_list);
svr->Post(params.api_prefix + "/lora-adapters", handle_lora_adapters_apply);
// Save & load slots
svr->Get ("/slots", handle_slots);
svr->Post("/slots/:id_slot", handle_slots_action);
svr->Get (params.api_prefix + "/slots", handle_slots);
svr->Post(params.api_prefix + "/slots/:id_slot", handle_slots_action);
//
// Start the server