mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2025-08-17 21:51:27 -04:00
wpm : portable unicode tolower (#6305)
Also use C locale for ispunct/isspace, and split unicode-data.cpp from unicode.cpp.
This commit is contained in:
@@ -24,3 +24,5 @@ int unicode_cpt_type(const std::string & utf8);
|
||||
std::string unicode_byte_to_utf8(uint8_t byte);
|
||||
uint8_t unicode_utf8_to_byte(const std::string & utf8);
|
||||
|
||||
// simple tolower that only implements one-to-one mapping, not one-to-many
|
||||
char32_t unicode_tolower(char32_t cp);
|
||||
|
Reference in New Issue
Block a user