diff --git a/src/convert.cpp b/src/convert.cpp index 7fb0825f5..b8a62121d 100644 --- a/src/convert.cpp +++ b/src/convert.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,7 @@ static bool preallocate_output_file(const std::string& output_path, uint64_t fil return true; } - std::fstream file(output_path, std::ios::binary | std::ios::in | std::ios::out); + std::fstream file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out); if (!file.is_open()) { if (error != nullptr) { *error = "failed to open output file '" + output_path + "' for preallocation"; @@ -251,7 +252,7 @@ static bool stream_tensor_data(ModelLoader& model_loader, for (int worker = 0; worker < n_threads; worker++) { workers.emplace_back([&]() { - std::fstream output_file(output_path, std::ios::binary | std::ios::in | std::ios::out); + std::fstream output_file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out); if (!output_file.is_open()) { fail("failed to open output file '" + output_path + "' for tensor writing"); return; diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp index 1747678d4..5639c7ed9 100644 --- a/src/core/ggml_tensor_utils.cpp +++ b/src/core/ggml_tensor_utils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "core/ggml_extend_backend.h" @@ -100,7 +101,7 @@ void ggml_ext_tensor_diff( } ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { LOG_ERROR("failed to open '%s'", file_path.c_str()); return nullptr; diff --git a/src/core/tensor_ggml.hpp b/src/core/tensor_ggml.hpp index 66ea22206..5082ce207 100644 --- a/src/core/tensor_ggml.hpp +++ b/src/core/tensor_ggml.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -105,7 +106,7 @@ namespace sd { template inline Tensor load_tensor_from_file_as_tensor(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { throw std::runtime_error("failed to open tensor file: " + file_path); } diff --git a/src/core/util.cpp b/src/core/util.cpp index 5292c38ba..c1d463e4d 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -123,12 +124,12 @@ int round_up_to(int value, int base) { #include bool file_exists(const std::string& filename) { - DWORD attributes = GetFileAttributesA(filename.c_str()); + DWORD attributes = GetFileAttributesW(std::filesystem::u8path(filename).c_str()); return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY)); } bool is_directory(const std::string& path) { - DWORD attributes = GetFileAttributesA(path.c_str()); + DWORD attributes = GetFileAttributesW(std::filesystem::u8path(path).c_str()); return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY)); } @@ -152,8 +153,8 @@ std::unique_ptr MmapWrapper::create(const std::string& filename, bo void* mapped_data = nullptr; size_t file_size = 0; - HANDLE file_handle = CreateFileA( - filename.c_str(), + HANDLE file_handle = CreateFileW( + std::filesystem::u8path(filename).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index 3e5744878..8120131ea 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) { } bool is_gguf_file(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { return false; } @@ -178,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path, } LOG_INFO("trying to save tensors to %s", file_path.c_str()); - FILE* file = fopen(file_path.c_str(), "wb+"); + FILE* file = ggml_fopen(file_path.c_str(), "wb+"); if (file == nullptr) { set_error(error, "failed to open output file '" + file_path + "'"); close(); diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h index b55d49f11..f02fe3c6f 100644 --- a/src/model_io/gguf_reader_ext.h +++ b/src/model_io/gguf_reader_ext.h @@ -2,6 +2,7 @@ #define __SD_MODEL_IO_GGUF_READER_EXT_H__ #include +#include #include #include #include @@ -184,7 +185,7 @@ class GGUFReader { public: bool load(const std::string& file_path) { - std::ifstream fin(file_path, std::ios::binary | std::ios::ate); + std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate); if (!fin) { LOG_ERROR("failed to open '%s'", file_path.c_str()); return false; diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index 3e6974237..cf424af92 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) { } static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) { - fs::path shard_fs_path(shard_path); + fs::path shard_fs_path = fs::u8path(shard_path); if (shard_fs_path.is_absolute()) { - return shard_fs_path.lexically_normal().string(); + return shard_fs_path.lexically_normal().u8string(); } - return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string(); + return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string(); } bool is_safetensors_file(const std::string& file_path) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { return false; } @@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path, std::string* error, std::map* metadata, std::set* tensor_names) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; @@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path, std::string* error) { shard_paths.clear(); - std::ifstream file(file_path); + std::ifstream file(fs::u8path(file_path)); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; @@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path, const std::string header_str = header.dump(); - std::ofstream file(file_path, std::ios::binary); + std::ofstream file(fs::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "' for writing"); return false; @@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path, data_start_ = ST_HEADER_SIZE_LEN + header_str.size(); LOG_INFO("trying to save tensors to %s", file_path.c_str()); - std::ofstream file(file_path, std::ios::binary | std::ios::trunc); + std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "' for writing"); return false; diff --git a/src/model_io/torch_legacy_io.cpp b/src/model_io/torch_legacy_io.cpp index 3170fe8d4..163e6649e 100644 --- a/src/model_io/torch_legacy_io.cpp +++ b/src/model_io/torch_legacy_io.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const bool read_torch_legacy_file(const std::string& file_path, std::vector& tensor_storages, std::string* error) { - std::ifstream file(file_path, std::ios::binary); + std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { set_error(error, "failed to open '" + file_path + "'"); return false; diff --git a/src/model_loader.cpp b/src/model_loader.cpp index c24351fb3..d881c7e58 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -333,7 +334,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path, } } - std::ifstream index_file(file_path); + std::ifstream index_file(std::filesystem::u8path(file_path)); const auto index = nlohmann::json::parse(index_file); for (const auto& entry : index.at("weight_map").items()) { const auto expected = (std::filesystem::u8path(file_path).parent_path() / @@ -1150,7 +1151,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, return; } } else if (!mmapped) { - file.open(file_path, std::ios::binary); + file.open(std::filesystem::u8path(file_path), std::ios::binary); if (!file.is_open()) { LOG_ERROR("failed to open '%s'", file_path.c_str()); failed = true; diff --git a/src/runtime/imatrix.cpp b/src/runtime/imatrix.cpp index a6f3bca95..616059d52 100644 --- a/src/runtime/imatrix.cpp +++ b/src/runtime/imatrix.cpp @@ -10,6 +10,7 @@ #include #include #include +#include static IMatrixCollector imatrix_collector; @@ -227,7 +228,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const { LOG_WARN("storing only %zu out of %zu entries\n", to_store.size(), stats_.size()); } - std::ofstream out(fname, std::ios::binary); + std::ofstream out(std::filesystem::u8path(fname), std::ios::binary); out.write((const char*)&n_entries, sizeof(n_entries)); for (const auto& name : to_store) { const auto& stat = stats_.at(name); @@ -251,7 +252,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const { } bool IMatrixCollector::load_imatrix(const char* fname) { - std::ifstream in(fname, std::ios::binary); + std::ifstream in(std::filesystem::u8path(fname), std::ios::binary); if (!in) { LOG_ERROR("failed to open %s\n", fname); return false; diff --git a/src/tokenizers/hf_tokenizer.cpp b/src/tokenizers/hf_tokenizer.cpp index f8daf6d2a..6e6401819 100644 --- a/src/tokenizers/hf_tokenizer.cpp +++ b/src/tokenizers/hf_tokenizer.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -495,7 +496,7 @@ struct HFTokenizer::Impl { HFTokenizer::HFTokenizer(const std::string& path) : impl_(new Impl) { - std::ifstream stream(path, std::ios::binary); + std::ifstream stream(std::filesystem::u8path(path), std::ios::binary); tokenizer_require(stream.good(), "cannot open " + path); TokenizerJSON config; stream >> config;