tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 03:55:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

arg : add --no-mmproj-offload (#13093)

Browse Source 
* arg : add --no-mmproj-offload

* Update common/arg.cpp
This commit is contained in:
Xuan-Son Nguyen
2025-04-24 14:04:14 +02:00
committed by GitHub
parent 80982e815e
commit 7c727fbe39
3 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
common/arg.cpp
View File

@ -2140,6 +2140,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.no_mmproj = true; params.no_mmproj = true;
} }
).set_examples(mmproj_examples)); ).set_examples(mmproj_examples));
add_opt(common_arg(
{"--no-mmproj-offload"},
"do not offload multimodal projector to GPU",
[](common_params & params) {
params.mmproj_use_gpu = false;
}
).set_examples(mmproj_examples));
add_opt(common_arg( add_opt(common_arg(
{"--image"}, "FILE", {"--image"}, "FILE",
"path to an image file. use with multimodal models. Specify multiple times for batching", "path to an image file. use with multimodal models. Specify multiple times for batching",

1
common/common.h
View File

@ -342,6 +342,7 @@ struct common_params {
// multimodal models (see examples/llava) // multimodal models (see examples/llava)
struct common_params_model mmproj; struct common_params_model mmproj;
bool mmproj_use_gpu = true; // use GPU for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) std::vector<std::string> image; // path to image file(s)

7
examples/llava/mtmd-cli.cpp
View File

@ -40,7 +40,8 @@ static void show_additional_info(int /*argc*/, char ** argv) {
"Usage: %s [options] -m <model> --mmproj <mmproj> --image <image> -p <prompt>\n\n" "Usage: %s [options] -m <model> --mmproj <mmproj> --image <image> -p <prompt>\n\n"
" -m and --mmproj are required\n" " -m and --mmproj are required\n"
" -hf user/repo can replace both -m and --mmproj in most cases\n" " -hf user/repo can replace both -m and --mmproj in most cases\n"
" --image and -p are optional, if NOT provided, the CLI will run in chat mode\n", " --image and -p are optional, if NOT provided, the CLI will run in chat mode\n"
" to disable using GPU for mmproj model, add --no-mmproj-offload\n",
argv[0] argv[0]
); );
} }
@ -112,10 +113,10 @@ struct mtmd_cli_context {
void init_vision_context(common_params & params) { void init_vision_context(common_params & params) {
const char * clip_path = params.mmproj.path.c_str(); const char * clip_path = params.mmproj.path.c_str();
ctx_vision.reset(mtmd_init_from_file(clip_path, model, mtmd_context_params{ ctx_vision.reset(mtmd_init_from_file(clip_path, model, mtmd_context_params{
/* use_gpu */ true, /* use_gpu */ params.mmproj_use_gpu,
/* timings */ true, /* timings */ true,
/* n_threads */ params.cpuparams.n_threads, /* n_threads */ params.cpuparams.n_threads,
/* verbosity */ GGML_LOG_LEVEL_INFO, /* verbosity */ params.verbosity > 0 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_INFO,
})); }));
if (!ctx_vision.get()) { if (!ctx_vision.get()) {
LOG_ERR("Failed to load vision model from %s\n", clip_path); LOG_ERR("Failed to load vision model from %s\n", clip_path);