tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 12:05:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: update deepseek reasoning format (pass reasoning_content as diffs) (#13933)

Browse Source 
* server: update deepseek reasoning format (now in reasoning_content diffs), add legacy option for compat
* update unit/test_tool_call.py::test_thoughts
This commit is contained in:
Olivier Chafik
2025-06-02 10:15:44 -07:00
committed by GitHub
parent bfd322796c
commit c9bbc77931
8 changed files with 30 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/arg.cpp
View File

@ -2869,6 +2869,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
"(default: deepseek)", "(default: deepseek)",
[](common_params & params, const std::string & value) { [](common_params & params, const std::string & value) {
/**/ if (value == "deepseek") { params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; } /**/ if (value == "deepseek") { params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; }
else if (value == "deepseek-legacy") { params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY; }
else if (value == "none") { params.reasoning_format = COMMON_REASONING_FORMAT_NONE; } else if (value == "none") { params.reasoning_format = COMMON_REASONING_FORMAT_NONE; }
else { throw std::invalid_argument("invalid value"); } else { throw std::invalid_argument("invalid value"); }
} }

15
common/chat.cpp
View File

@ -82,10 +82,10 @@ json common_chat_msg::to_json_oaicompat() const
std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const common_chat_msg & previous_msg, const common_chat_msg & new_msg) { std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const common_chat_msg & previous_msg, const common_chat_msg & new_msg) {
std::vector<common_chat_msg_diff> diffs; std::vector<common_chat_msg_diff> diffs;
// if (previous_msg.reasoning_content != current.reasoning_content) { if (previous_msg.reasoning_content != new_msg.reasoning_content) {
// auto & diff = diffs.emplace_back(); auto & diff = diffs.emplace_back();
// diff.reasoning_content_delta = string_diff(previous_msg.reasoning_content, current.reasoning_content); diff.reasoning_content_delta = string_diff(previous_msg.reasoning_content, new_msg.reasoning_content);
// } }
if (previous_msg.content != new_msg.content) { if (previous_msg.content != new_msg.content) {
auto & diff = diffs.emplace_back(); auto & diff = diffs.emplace_back();
diff.content_delta = string_diff(previous_msg.content, new_msg.content); diff.content_delta = string_diff(previous_msg.content, new_msg.content);
@ -385,9 +385,9 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
template <> json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) { template <> json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
json delta = json::object(); json delta = json::object();
// if (!diff.reasoning_content_delta.empty()) { if (!diff.reasoning_content_delta.empty()) {
// delta["reasoning_content"] = msg.reasoning_content; delta["reasoning_content"] = diff.reasoning_content_delta;
// } }
if (!diff.content_delta.empty()) { if (!diff.content_delta.empty()) {
delta["content"] = diff.content_delta; delta["content"] = diff.content_delta;
} }
@ -598,6 +598,7 @@ const char * common_reasoning_format_name(common_reasoning_format format) {
switch (format) { switch (format) {
case COMMON_REASONING_FORMAT_NONE: return "none"; case COMMON_REASONING_FORMAT_NONE: return "none";
case COMMON_REASONING_FORMAT_DEEPSEEK: return "deepseek"; case COMMON_REASONING_FORMAT_DEEPSEEK: return "deepseek";
case COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY: return "deepseek-legacy";
default: default:
throw std::runtime_error("Unknown reasoning format"); throw std::runtime_error("Unknown reasoning format");
} }

2
common/chat.h
View File

@ -70,7 +70,7 @@ struct common_chat_msg {
}; };
struct common_chat_msg_diff { struct common_chat_msg_diff {
// std::string reasoning_content_delta; std::string reasoning_content_delta;
std::string content_delta; std::string content_delta;
size_t tool_call_index = std::string::npos; size_t tool_call_index = std::string::npos;
common_chat_tool_call tool_call_delta; common_chat_tool_call tool_call_delta;

3
common/common.h
View File

@ -215,7 +215,8 @@ struct common_params_vocoder {
enum common_reasoning_format { enum common_reasoning_format {
COMMON_REASONING_FORMAT_NONE, COMMON_REASONING_FORMAT_NONE,
COMMON_REASONING_FORMAT_DEEPSEEK, // Extract thinking tag contents and return as `message.reasoning_content` COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY, // Extract thinking tag contents and return as `message.reasoning_content`, or leave inline in <think> tags in stream mode
COMMON_REASONING_FORMAT_DEEPSEEK, // Extract thinking tag contents and return as `message.reasoning_content`, including in streaming deltas.
}; };
struct common_params { struct common_params {

2
tests/test-chat.cpp
View File

@ -19,8 +19,8 @@
using json = nlohmann::ordered_json; using json = nlohmann::ordered_json;
static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) { static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) {
// os << "reasoning_content_delta: " << diff.reasoning_content_delta << '\n';
os << "{ content_delta: " << diff.content_delta << "; "; os << "{ content_delta: " << diff.content_delta << "; ";
os << "reasoning_content_delta: " << diff.reasoning_content_delta << "; ";
if (diff.tool_call_index != std::string::npos) { if (diff.tool_call_index != std::string::npos) {
os << "tool_call_index: " << diff.tool_call_index << "; "; os << "tool_call_index: " << diff.tool_call_index << "; ";
os << "tool_call_delta.name: " << diff.tool_call_delta.name << "; "; os << "tool_call_delta.name: " << diff.tool_call_delta.name << "; ";

2
tools/server/server.cpp
View File

@ -360,7 +360,7 @@ struct server_task {
params.oaicompat_chat_syntax.format = defaults.oaicompat_chat_syntax.format; params.oaicompat_chat_syntax.format = defaults.oaicompat_chat_syntax.format;
} }
params.oaicompat_chat_syntax.reasoning_format = params_base.reasoning_format; params.oaicompat_chat_syntax.reasoning_format = params_base.reasoning_format;
params.oaicompat_chat_syntax.reasoning_in_content = params.stream; params.oaicompat_chat_syntax.reasoning_in_content = params.stream && (params_base.reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false); params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false);
params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false); params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false);
} }

13
tools/server/tests/unit/test_tool_call.py
View File

@ -499,13 +499,12 @@ def do_test_calc_result(server: ServerProcess, result_override: str | None, n_pr
@pytest.mark.slow @pytest.mark.slow
@pytest.mark.parametrize("n_predict,reasoning_format,stream,expect_reasoning_content,expect_content,hf_repo,template_override", [ @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
(128, 'deepseek', CompletionMode.NORMAL, None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None), @pytest.mark.parametrize("n_predict,reasoning_format,expect_reasoning_content,expect_content,hf_repo,template_override", [
(128, None, CompletionMode.NORMAL, None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None), (128, 'deepseek', None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
(1024, 'deepseek', CompletionMode.NORMAL, "I need to calculate the sum of 102 and 7[\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None), (128, None, None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
(1024, 'deepseek', CompletionMode.STREAMED, None, "^<think>I need to calculate [\\s\\S]*?</think>To find the sum of [\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None), (1024, 'deepseek', "I need to calculate the sum of 102 and 7[\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
(1024, 'deepseek', CompletionMode.NORMAL, "First, I [\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)), (1024, 'deepseek', "First, I [\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),
(1024, 'deepseek', CompletionMode.STREAMED, None, "^<think>First, I [\\s\\S]*?</think>To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),
# (1024, 'none', CompletionMode.NORMAL, None, "^(<think>\\s*)?I need[\\s\\S]*?</think>\\s*To find[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None), # (1024, 'none', CompletionMode.NORMAL, None, "^(<think>\\s*)?I need[\\s\\S]*?</think>\\s*To find[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
# (128, 'deepseek', None, "^Okay, let me figure out the sum of 102 and 7[\\s\\S]*", "bartowski/Qwen_QwQ-32B-GGUF:Q4_K_M", None), # (128, 'deepseek', None, "^Okay, let me figure out the sum of 102 and 7[\\s\\S]*", "bartowski/Qwen_QwQ-32B-GGUF:Q4_K_M", None),
]) ])

11
tools/server/tests/utils.py
View File

@ -308,10 +308,12 @@ class ServerProcess:
stream = data.get('stream', False) stream = data.get('stream', False)
if stream: if stream:
content: list[str] = [] content: list[str] = []
reasoning_content: list[str] = []
tool_calls: list[dict] = [] tool_calls: list[dict] = []
finish_reason: Optional[str] = None finish_reason: Optional[str] = None
content_parts = 0 content_parts = 0
reasoning_content_parts = 0
tool_call_parts = 0 tool_call_parts = 0
arguments_parts = 0 arguments_parts = 0
@ -322,6 +324,10 @@ class ServerProcess:
assert len(choice['delta']['content']) > 0, f'Expected non empty content delta!' assert len(choice['delta']['content']) > 0, f'Expected non empty content delta!'
content.append(choice['delta']['content']) content.append(choice['delta']['content'])
content_parts += 1 content_parts += 1
if choice['delta'].get('reasoning_content') is not None:
assert len(choice['delta']['reasoning_content']) > 0, f'Expected non empty reasoning_content delta!'
reasoning_content.append(choice['delta']['reasoning_content'])
reasoning_content_parts += 1
if choice['delta'].get('finish_reason') is not None: if choice['delta'].get('finish_reason') is not None:
finish_reason = choice['delta']['finish_reason'] finish_reason = choice['delta']['finish_reason']
for tc in choice['delta'].get('tool_calls', []): for tc in choice['delta'].get('tool_calls', []):
@ -349,8 +355,10 @@ class ServerProcess:
tool_call['function']['name'] = tool_call['function'].get('name', '') + fct['name'] tool_call['function']['name'] = tool_call['function'].get('name', '') + fct['name']
if fct.get('arguments') is not None: if fct.get('arguments') is not None:
tool_call['function']['arguments'] += fct['arguments'] tool_call['function']['arguments'] += fct['arguments']
arguments_parts += 1
tool_call_parts += 1
print(f'Streamed response had {content_parts} content parts, {tool_call_parts} tool call parts incl. {arguments_parts} arguments parts') print(f'Streamed response had {content_parts} content parts, {reasoning_content_parts} reasoning_content parts, {tool_call_parts} tool call parts incl. {arguments_parts} arguments parts')
result = dict( result = dict(
choices=[ choices=[
dict( dict(
@ -359,6 +367,7 @@ class ServerProcess:
message=dict( message=dict(
role='assistant', role='assistant',
content=''.join(content) if content else None, content=''.join(content) if content else None,
reasoning_content=''.join(reasoning_content) if reasoning_content else None,
tool_calls=tool_calls if tool_calls else None, tool_calls=tool_calls if tool_calls else None,
), ),
) )