tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-28 20:25:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

server : passthrough the /models endpoint during loading

Browse Source
This commit is contained in:
Georgi Gerganov
2025-05-14 14:15:42 +03:00
parent 09d13d94fb
commit 6190e1c1c9
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tools/server/server.cpp
View File

@ -3705,6 +3705,9 @@ int main(int argc, char ** argv) {
if (req.path == "/" || tmp.back() == "html") { if (req.path == "/" || tmp.back() == "html") {
res.set_content(reinterpret_cast<const char*>(loading_html), loading_html_len, "text/html; charset=utf-8"); res.set_content(reinterpret_cast<const char*>(loading_html), loading_html_len, "text/html; charset=utf-8");
res.status = 503; res.status = 503;
} else if (req.path == "/models" || req.path == "/v1/models") {
// allow the models endpoint to be accessed during loading
return true;
} else { } else {
res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE)); res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE));
} }
@ -4363,7 +4366,13 @@ int main(int argc, char ** argv) {
res_ok(res, {{ "prompt", std::move(data.at("prompt")) }}); res_ok(res, {{ "prompt", std::move(data.at("prompt")) }});
}; };
const auto handle_models = [&params, &ctx_server, &res_ok](const httplib::Request &, httplib::Response & res) { const auto handle_models = [&params, &ctx_server, &state, &res_ok](const httplib::Request &, httplib::Response & res) {
server_state current_state = state.load();
std::string model_meta;
if (current_state == SERVER_STATE_READY) {
model_meta = ctx_server.model_meta();
}
json models = { json models = {
{"object", "list"}, {"object", "list"},
{"data", { {"data", {
@ -4372,7 +4381,7 @@ int main(int argc, char ** argv) {
{"object", "model"}, {"object", "model"},
{"created", std::time(0)}, {"created", std::time(0)},
{"owned_by", "llamacpp"}, {"owned_by", "llamacpp"},
{"meta", ctx_server.model_meta()} {"meta", model_meta},
}, },
}} }}
}; };