tqcq/llama.cpp
0
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-08-18 14:18:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

download in batches

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2025-05-31 22:35:26 +02:00
committed by GitHub
parent de8ec1348b
commit f9a27178e5
1 changed files with 42 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
tests/test-tokenizers-remote.cpp
View File

@@ -4,7 +4,8 @@
#include <string>
#include <fstream>
#include <vector>
#include <json.hpp>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
@@ -109,7 +110,24 @@ int main(void) {
}
}
if (common_download_file_multiple(files, {}, false)) {
if (!files.empty()) {
bool downloaded = false;
const size_t batch_size = 6;
size_t batches = (files.size() + batch_size - 1) / batch_size;
for (size_t i = 0; i < batches; i++) {
size_t batch_pos = (i * batch_size);
size_t batch_step = batch_pos + batch_size;
auto batch_begin = files.begin() + batch_pos;
auto batch_end = batch_step >= files.size() ? files.end() : files.begin() + batch_step;
std::vector<std::pair<std::string, std::string>> batch(batch_begin, batch_end);
if (!(downloaded = common_download_file_multiple(batch, {}, false))) {
break;
}
}
if (downloaded) {
std::string dir_sep(1, DIRECTORY_SEPARATOR);
for (auto const & item : files) {
@@ -136,6 +154,7 @@ int main(void) {
} else {
printf("test-tokenizers-remote: failed to download files, unable to perform tests...\n");
}
}
} else {
printf("test-tokenizers-remote: failed to retrieve repository info, unable to perform tests...\n");
}