tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-30 20:58:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

bert : add tests + fix quantization (#5475)

Browse Source 
* llama : do not quantize pos embd and token type tensors

* ci : add BERT tests

ggml-ci

* ci : do not do BERT tests on low-perf nodes

ggml-ci
This commit is contained in:
Georgi Gerganov
2024-02-13 13:01:29 +02:00
committed by GitHub
parent 99b8b43d7b
commit 49cc1f7d67
2 changed files with 51 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
llama.cpp
View File

@ -10444,7 +10444,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
quantize &= !params->only_copy;
// do not quantize expert gating tensors
quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;
quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight");
// do not quantize positional embeddings and token types (BERT)
quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight");
quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
enum ggml_type new_type;
void * new_data;