Commit 27b20ba8b for llama.cpp
commit 27b20ba8b1b85fc800b0a04caa8f1003ca64a2a3
Author: Adrien Gallouët <angt@huggingface.co>
Date: Fri Sep 25 11:40:41 2026 +0200
common : extract shared unicode path/string helpers (#29415)
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
diff --git a/common/common.cpp b/common/common.cpp
index a316b2ae1..364688ef4 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -900,7 +900,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
#ifdef _WIN32
-static std::wstring utf8_to_wstring(const std::string & str) {
+std::wstring utf8_to_wstring(const std::string & str) {
if (str.empty()) {
return std::wstring();
}
@@ -916,8 +916,31 @@ static std::wstring utf8_to_wstring(const std::string & str) {
return wstr;
}
+
+std::string wstring_to_utf8(const std::wstring & str) {
+ if (str.empty()) {
+ return std::string();
+ }
+
+ int size = WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0, NULL, NULL);
+
+ if (size <= 0) {
+ return std::string();
+ }
+
+ std::string utf8(size, 0);
+ WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), &utf8[0], size, NULL, NULL);
+
+ return utf8;
+}
#endif
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path) {
+ const auto value = path.u8string();
+ return std::string(value.begin(), value.end());
+}
+
// returns true if successful, false otherwise
bool fs_create_directory_with_parents(const std::string & path) {
#ifdef _WIN32
diff --git a/common/common.h b/common/common.h
index 39f855c4f..9194d3dcb 100644
--- a/common/common.h
+++ b/common/common.h
@@ -16,6 +16,7 @@
#include <vector>
#include <map>
#include <algorithm>
+#include <filesystem>
#include <fstream>
#if defined(_WIN32) && !defined(_WIN32_WINNT)
@@ -880,6 +881,18 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
bool glob_match(const std::string & pattern, const std::string & str);
+//
+// Unicode utils
+//
+
+#ifdef _WIN32
+std::wstring utf8_to_wstring(const std::string & str);
+std::string wstring_to_utf8(const std::wstring & str);
+#endif
+
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path);
+
//
// Environment utils
//
diff --git a/common/console.cpp b/common/console.cpp
index 36f645f33..9020eb513 100644
--- a/common/console.cpp
+++ b/common/console.cpp
@@ -1,4 +1,5 @@
#include "console.h"
+#include "common.h"
#include "log.h"
#include <vector>
#include <iostream>
@@ -1053,9 +1054,7 @@ namespace console {
return false;
}
- int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);
- line.resize(size_needed);
- WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);
+ line = wstring_to_utf8(wline);
#else
if (!std::getline(std::cin, line)) {
// Input stream is bad or EOF received
diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp
index 12d4fcc80..65c9722f3 100644
--- a/common/hf-cache.cpp
+++ b/common/hf-cache.cpp
@@ -63,12 +63,7 @@ static fs::path get_cache_directory() {
}
std::string get_cache_path() {
-#if defined(__cpp_lib_char8_t)
- const std::u8string u8str = get_cache_directory().u8string();
- return std::string(reinterpret_cast<const char *>(u8str.data()), u8str.size());
-#else
- return get_cache_directory().u8string();
-#endif
+ return fs_path_to_utf8(get_cache_directory());
}
static std::string folder_name_to_repo(const std::string & folder) {
diff --git a/tools/server/server-mcp.cpp b/tools/server/server-mcp.cpp
index 93db6164d..88b2c9239 100644
--- a/tools/server/server-mcp.cpp
+++ b/tools/server/server-mcp.cpp
@@ -1,5 +1,6 @@
#include "server-mcp.h"
+#include "common.h"
#include "subproc.h"
#include <atomic>
@@ -349,43 +350,21 @@ struct server_mcp_stdio::process_handle {
#if defined(_WIN32)
// config strings are UTF-8 (from JSON) and subprocess.h converts them with CP_UTF8, so inputs must be UTF-8, not the active code page
-static std::wstring windows_utf8_to_wide(const std::string & s) {
- if (s.empty()) {
- return std::wstring();
- }
- int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), NULL, 0);
- if (n <= 0) {
- return std::wstring();
- }
- std::wstring w((size_t) n, L'\0');
- MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), &w[0], n);
- return w;
-}
-
-static std::string windows_wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) {
- int n = WideCharToMultiByte(CP_UTF8, 0, s, len, NULL, 0, NULL, NULL);
- if (n <= 0) {
- return std::string();
- }
- std::string out((size_t) n, '\0');
- WideCharToMultiByte(CP_UTF8, 0, s, len, &out[0], n, NULL, NULL);
- if (len == -1 && !out.empty() && out.back() == '\0') {
- out.pop_back(); // drop the terminator WideCharToMultiByte counts for -1
- }
- return out;
+static std::string wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) {
+ return wstring_to_utf8(len == -1 ? std::wstring(s) : std::wstring(s, s + len));
}
#endif
static std::string mcp_resolve_command(const std::string & command) {
#if defined(_WIN32)
// For Windows: make sure we handle ".exe" correctly, as well as UTF-8
- std::wstring wcmd = windows_utf8_to_wide(command);
+ std::wstring wcmd = utf8_to_wstring(command);
wchar_t buf[MAX_PATH * 4];
const DWORD cap = (DWORD) (sizeof(buf) / sizeof(buf[0]));
auto search = [&](const wchar_t * ext) -> std::string {
DWORD n = SearchPathW(NULL, wcmd.c_str(), ext, cap, buf, NULL);
- return (n > 0 && n < cap) ? windows_wide_to_utf8(buf, (int) n) : std::string();
+ return (n > 0 && n < cap) ? wide_to_utf8(buf, (int) n) : std::string();
};
std::string found = search(NULL); // exact path / already-extensioned / .exe on PATH
@@ -429,7 +408,7 @@ static std::vector<std::string> mcp_parent_env() {
LPWCH block = GetEnvironmentStringsW();
if (block) {
for (LPWCH e = block; *e; e += wcslen(e) + 1) {
- env.emplace_back(windows_wide_to_utf8(e, -1));
+ env.emplace_back(wide_to_utf8(e, -1));
}
FreeEnvironmentStringsW(block);
}
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index 841221d8c..a8996b00c 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -483,25 +483,6 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
}
}
-#ifdef _WIN32
-static std::string wide_to_utf8(const wchar_t * ws) {
- if (!ws || !*ws) {
- return {};
- }
-
- const int len = static_cast<int>(std::wcslen(ws));
- const int bytes = WideCharToMultiByte(CP_UTF8, 0, ws, len, nullptr, 0, nullptr, nullptr);
- if (bytes == 0) {
- return {};
- }
-
- std::string utf8(bytes, '\0');
- WideCharToMultiByte(CP_UTF8, 0, ws, len, utf8.data(), bytes, nullptr, nullptr);
-
- return utf8;
-}
-#endif
-
static std::vector<std::string> get_environment() {
std::vector<std::string> env;
@@ -511,7 +492,7 @@ static std::vector<std::string> get_environment() {
return env;
}
for (LPWCH e = env_block; *e; e += wcslen(e) + 1) {
- env.emplace_back(wide_to_utf8(e));
+ env.emplace_back(wstring_to_utf8(e));
}
FreeEnvironmentStringsW(env_block);
#else
@@ -592,7 +573,7 @@ server_models::server_models(
// set binary path
try {
- bin_path = get_server_exec_path().string();
+ bin_path = fs_path_to_utf8(get_server_exec_path());
} catch (const std::exception & e) {
bin_path = argv[0];
LOG_WRN("failed to get server executable path: %s\n", e.what());