tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-06-27 03:55:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding UTF-8 support to llama.cpp (#12111)

Browse Source 
For emojis, non-alpha characters, etc.

Signed-off-by: Eric Curtin <ecurtin@redhat.com>
This commit is contained in:
Eric Curtin
2025-03-03 12:44:56 +00:00
committed by GitHub
parent 7b69003af7
commit c950a1f692
2 changed files with 917 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1081
examples/run/linenoise.cpp/linenoise.cpp
View File

File diff suppressed because it is too large Load Diff

15
examples/run/linenoise.cpp/linenoise.h
View File

@ -63,10 +63,10 @@ struct linenoiseState {
const char * prompt; /* Prompt to display. */ const char * prompt; /* Prompt to display. */
size_t plen; /* Prompt length. */ size_t plen; /* Prompt length. */
size_t pos; /* Current cursor position. */ size_t pos; /* Current cursor position. */
size_t oldpos; /* Previous refresh cursor position. */ size_t oldcolpos; /* Previous refresh cursor column position. */
size_t len; /* Current edited line length. */ size_t len; /* Current edited line length. */
size_t cols; /* Number of columns in terminal. */ size_t cols; /* Number of columns in terminal. */
size_t oldrows; /* Rows used by last refrehsed line (multiline mode) */ size_t oldrows; /* Rows used by last refreshed line (multiline mode) */
int history_index; /* The history index we are currently editing. */ int history_index; /* The history index we are currently editing. */
}; };
@ -89,7 +89,8 @@ struct linenoiseCompletions {
}; };
/* Non blocking API. */ /* Non blocking API. */
int linenoiseEditStart(struct linenoiseState *l, int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt); int linenoiseEditStart(struct linenoiseState * l, int stdin_fd, int stdout_fd, char * buf, size_t buflen,
const char * prompt);
const char * linenoiseEditFeed(struct linenoiseState * l); const char * linenoiseEditFeed(struct linenoiseState * l);
void linenoiseEditStop(struct linenoiseState * l); void linenoiseEditStop(struct linenoiseState * l);
void linenoiseHide(struct linenoiseState * l); void linenoiseHide(struct linenoiseState * l);
@ -121,6 +122,14 @@ void linenoisePrintKeyCodes(void);
void linenoiseMaskModeEnable(void); void linenoiseMaskModeEnable(void);
void linenoiseMaskModeDisable(void); void linenoiseMaskModeDisable(void);
/* Encoding functions. */
typedef size_t(linenoisePrevCharLen)(const char * buf, size_t buf_len, size_t pos, size_t * col_len);
typedef size_t(linenoiseNextCharLen)(const char * buf, size_t buf_len, size_t pos, size_t * col_len);
typedef size_t(linenoiseReadCode)(int fd, char * buf, size_t buf_len, int * c);
void linenoiseSetEncodingFunctions(linenoisePrevCharLen * prevCharLenFunc, linenoiseNextCharLen * nextCharLenFunc,
linenoiseReadCode * readCodeFunc);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif