diff --git a/common/common.cpp b/common/common.cpp index a316b2ae17..364688ef46 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -900,7 +900,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) { #ifdef _WIN32 -static std::wstring utf8_to_wstring(const std::string & str) { +std::wstring utf8_to_wstring(const std::string & str) { if (str.empty()) { return std::wstring(); } @@ -916,8 +916,31 @@ static std::wstring utf8_to_wstring(const std::string & str) { return wstr; } + +std::string wstring_to_utf8(const std::wstring & str) { + if (str.empty()) { + return std::string(); + } + + int size = WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0, NULL, NULL); + + if (size <= 0) { + return std::string(); + } + + std::string utf8(size, 0); + WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), &utf8[0], size, NULL, NULL); + + return utf8; +} #endif +// returns the path as a UTF-8 string, preserving its separators +std::string fs_path_to_utf8(const std::filesystem::path & path) { + const auto value = path.u8string(); + return std::string(value.begin(), value.end()); +} + // returns true if successful, false otherwise bool fs_create_directory_with_parents(const std::string & path) { #ifdef _WIN32 diff --git a/common/common.h b/common/common.h index 39f855c4f2..9194d3dcb6 100644 --- a/common/common.h +++ b/common/common.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #if defined(_WIN32) && !defined(_WIN32_WINNT) @@ -880,6 +881,18 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat bool glob_match(const std::string & pattern, const std::string & str); +// +// Unicode utils +// + +#ifdef _WIN32 +std::wstring utf8_to_wstring(const std::string & str); +std::string wstring_to_utf8(const std::wstring & str); +#endif + +// returns the path as a UTF-8 string, preserving its separators +std::string fs_path_to_utf8(const std::filesystem::path & path); + // // Environment utils // diff --git a/common/console.cpp b/common/console.cpp index 36f645f332..9020eb5131 100644 --- a/common/console.cpp +++ b/common/console.cpp @@ -1,4 +1,5 @@ #include "console.h" +#include "common.h" #include "log.h" #include #include @@ -1053,9 +1054,7 @@ namespace console { return false; } - int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL); - line.resize(size_needed); - WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL); + line = wstring_to_utf8(wline); #else if (!std::getline(std::cin, line)) { // Input stream is bad or EOF received diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index 12d4fcc804..65c9722f33 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -63,12 +63,7 @@ static fs::path get_cache_directory() { } std::string get_cache_path() { -#if defined(__cpp_lib_char8_t) - const std::u8string u8str = get_cache_directory().u8string(); - return std::string(reinterpret_cast(u8str.data()), u8str.size()); -#else - return get_cache_directory().u8string(); -#endif + return fs_path_to_utf8(get_cache_directory()); } static std::string folder_name_to_repo(const std::string & folder) { diff --git a/tools/server/server-mcp.cpp b/tools/server/server-mcp.cpp index 93db6164d3..88b2c92399 100644 --- a/tools/server/server-mcp.cpp +++ b/tools/server/server-mcp.cpp @@ -1,5 +1,6 @@ #include "server-mcp.h" +#include "common.h" #include "subproc.h" #include @@ -349,43 +350,21 @@ struct server_mcp_stdio::process_handle { #if defined(_WIN32) // config strings are UTF-8 (from JSON) and subprocess.h converts them with CP_UTF8, so inputs must be UTF-8, not the active code page -static std::wstring windows_utf8_to_wide(const std::string & s) { - if (s.empty()) { - return std::wstring(); - } - int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), NULL, 0); - if (n <= 0) { - return std::wstring(); - } - std::wstring w((size_t) n, L'\0'); - MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), &w[0], n); - return w; -} - -static std::string windows_wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) { - int n = WideCharToMultiByte(CP_UTF8, 0, s, len, NULL, 0, NULL, NULL); - if (n <= 0) { - return std::string(); - } - std::string out((size_t) n, '\0'); - WideCharToMultiByte(CP_UTF8, 0, s, len, &out[0], n, NULL, NULL); - if (len == -1 && !out.empty() && out.back() == '\0') { - out.pop_back(); // drop the terminator WideCharToMultiByte counts for -1 - } - return out; +static std::string wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) { + return wstring_to_utf8(len == -1 ? std::wstring(s) : std::wstring(s, s + len)); } #endif static std::string mcp_resolve_command(const std::string & command) { #if defined(_WIN32) // For Windows: make sure we handle ".exe" correctly, as well as UTF-8 - std::wstring wcmd = windows_utf8_to_wide(command); + std::wstring wcmd = utf8_to_wstring(command); wchar_t buf[MAX_PATH * 4]; const DWORD cap = (DWORD) (sizeof(buf) / sizeof(buf[0])); auto search = [&](const wchar_t * ext) -> std::string { DWORD n = SearchPathW(NULL, wcmd.c_str(), ext, cap, buf, NULL); - return (n > 0 && n < cap) ? windows_wide_to_utf8(buf, (int) n) : std::string(); + return (n > 0 && n < cap) ? wide_to_utf8(buf, (int) n) : std::string(); }; std::string found = search(NULL); // exact path / already-extensioned / .exe on PATH @@ -429,7 +408,7 @@ static std::vector mcp_parent_env() { LPWCH block = GetEnvironmentStringsW(); if (block) { for (LPWCH e = block; *e; e += wcslen(e) + 1) { - env.emplace_back(windows_wide_to_utf8(e, -1)); + env.emplace_back(wide_to_utf8(e, -1)); } FreeEnvironmentStringsW(block); } diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 841221d8c7..a8996b00c9 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -483,25 +483,6 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) { } } -#ifdef _WIN32 -static std::string wide_to_utf8(const wchar_t * ws) { - if (!ws || !*ws) { - return {}; - } - - const int len = static_cast(std::wcslen(ws)); - const int bytes = WideCharToMultiByte(CP_UTF8, 0, ws, len, nullptr, 0, nullptr, nullptr); - if (bytes == 0) { - return {}; - } - - std::string utf8(bytes, '\0'); - WideCharToMultiByte(CP_UTF8, 0, ws, len, utf8.data(), bytes, nullptr, nullptr); - - return utf8; -} -#endif - static std::vector get_environment() { std::vector env; @@ -511,7 +492,7 @@ static std::vector get_environment() { return env; } for (LPWCH e = env_block; *e; e += wcslen(e) + 1) { - env.emplace_back(wide_to_utf8(e)); + env.emplace_back(wstring_to_utf8(e)); } FreeEnvironmentStringsW(env_block); #else @@ -592,7 +573,7 @@ server_models::server_models( // set binary path try { - bin_path = get_server_exec_path().string(); + bin_path = fs_path_to_utf8(get_server_exec_path()); } catch (const std::exception & e) { bin_path = argv[0]; LOG_WRN("failed to get server executable path: %s\n", e.what());