Commit 9d340ec34 for llama.cpp
commit 9d340ec341e079d2c28afd50871afce358cdbf48
Author: Julius Tischbein <jtischbein@nvidia.com>
Date: Mon Nov 9 19:22:08 2026 +0100
Windows unbuffered model load (#26014)
* Windows unbuffered io
* Fix read len calculation
* Add test and model load timing
* chunked fallback read + removing filepointer semantic divergence between linux and windows
* tests : remove unused mmap test arguments
Assisted-by: Codex
* remove timing, set file pointer after chunk read, test refinement
* Update tests/test-llama-mmap.cpp
Co-authored-by: Oliver Simons <osimons@nvidia.com>
* Update src/llama-mmap.cpp
Co-authored-by: Oliver Simons <osimons@nvidia.com>
* test refinement
Assisted-by: Codex
* remove duplicate constexpr
Assisted-by: Codex
* adding label behavior to tests
Assisted-by: Codex
---------
Co-authored-by: Oliver Simons <osimons@nvidia.com>
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
index 4d1cd31e9..9688a7ed4 100644
--- a/src/llama-mmap.cpp
+++ b/src/llama-mmap.cpp
@@ -30,6 +30,7 @@
#ifndef NOMINMAX
#define NOMINMAX
#endif
+ #include <malloc.h>
#include <windows.h>
#ifndef PATH_MAX
#define PATH_MAX MAX_PATH
@@ -68,7 +69,10 @@ static std::string llama_format_win_err(DWORD err) {
struct llama_file::impl {
#if defined(_WIN32)
- HANDLE fp_win32;
+ // Keep the buffered handle as the logical cursor; the direct handle only issues aligned I/O.
+ HANDLE fp_win32 = INVALID_HANDLE_VALUE;
+ HANDLE fp_win32_direct = INVALID_HANDLE_VALUE;
+
std::string GetErrorMessageWin32(DWORD error_code) const {
std::string ret;
LPSTR lpMsgBuf = NULL;
@@ -81,13 +85,93 @@ struct llama_file::impl {
LocalFree(lpMsgBuf);
}
+ while (!ret.empty() && (ret.back() == '\r' || ret.back() == '\n')) {
+ ret.pop_back();
+ }
+
return ret;
}
- impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
- fp = ggml_fopen(fname, mode);
+ impl(const char * fname, const char * mode, const bool use_direct_io = false) : fname(fname) {
+ init_buffered_io(mode);
+ if (use_direct_io && std::strcmp(mode, "rb") == 0) {
+ DWORD error = ERROR_SUCCESS;
+ if (!init_direct_io(error)) {
+ LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n",
+ fname, GetErrorMessageWin32(error).c_str());
+ }
+ }
+ }
+
+ bool init_direct_io(DWORD& error) {
+ error = ERROR_SUCCESS;
+ alignment = 1;
+
+ fp_win32_direct = ReOpenFile(
+ fp_win32,
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ FILE_FLAG_NO_BUFFERING);
+
+ if (fp_win32_direct == INVALID_HANDLE_VALUE) {
+ error = GetLastError();
+ return false;
+ }
+
+ const bool ok = [&]() -> bool {
+#if defined(NTDDI_VERSION) && defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8)
+ FILE_STORAGE_INFO storage_info{};
+ if (!GetFileInformationByHandleEx(
+ fp_win32_direct,
+ FileStorageInfo,
+ &storage_info,
+ static_cast<DWORD>(sizeof(storage_info)))) {
+ error = GetLastError();
+ return false;
+ }
+
+ alignment = std::max<size_t>({
+ static_cast<size_t>(storage_info.LogicalBytesPerSector),
+ static_cast<size_t>(storage_info.PhysicalBytesPerSectorForAtomicity),
+ static_cast<size_t>(storage_info.PhysicalBytesPerSectorForPerformance),
+ static_cast<size_t>(storage_info.FileSystemEffectivePhysicalBytesPerSectorForAtomicity)
+ });
+#else
+ error = ERROR_NOT_SUPPORTED;
+ return false;
+#endif
+ // Required for bit-mask rounding and aligned allocation schemes.
+ if (alignment <= 1 || (alignment & (alignment - 1)) != 0) {
+ error = ERROR_NOT_SUPPORTED;
+ return false;
+ }
+
+ // ReOpenFile creates a distinct handle; explicitly start direct I/O at offset 0.
+ LARGE_INTEGER zero{};
+ if (!SetFilePointerEx(fp_win32_direct, zero, nullptr, FILE_BEGIN)) {
+ error = GetLastError();
+ return false;
+ }
+
+ return true;
+ }();
+
+ if (!ok) {
+ CloseHandle(fp_win32_direct);
+ fp_win32_direct = INVALID_HANDLE_VALUE;
+ alignment = 1;
+ return false;
+ }
+
+ LLAMA_LOG_INFO("%s: file '%s' opened with unbuffered I/O (alignment: %zu bytes)\n",
+ __func__, fname.c_str(), alignment);
+ return true;
+ }
+
+ void init_buffered_io(const char * mode) {
+ fp = ggml_fopen(fname.c_str(), mode);
if (fp == NULL) {
- throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+ throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));
}
fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
seek(0, SEEK_END);
@@ -95,7 +179,7 @@ struct llama_file::impl {
seek(0, SEEK_SET);
}
- impl(FILE * file) : owns_fp(false) {
+ impl(FILE * file) : fname("(file*)"), owns_fp(false) {
fp = file;
fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
seek(0, SEEK_END);
@@ -108,7 +192,7 @@ struct llama_file::impl {
li.QuadPart = 0;
BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
if (!ret) {
- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ throw std::runtime_error(format("tell error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
return li.QuadPart;
@@ -123,24 +207,159 @@ struct llama_file::impl {
li.QuadPart = offset;
BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
if (!ret) {
- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ throw std::runtime_error(format("seek error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
}
- void read_raw(void * ptr, size_t len) {
+ void read_raw_unsafe(void * ptr, size_t len) {
+ if (len == 0) {
+ return;
+ }
+
+ const size_t read_offset = tell();
+ if (read_offset > size) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+
+ if (!has_direct_io()) {
+ const size_t read_size = std::min(len, size - read_offset);
+ size_t bytes_read = 0;
+ while (bytes_read < read_size) {
+ const DWORD to_read = (DWORD) std::min<size_t>(read_size - bytes_read, MAXDWORD);
+ DWORD chunk_read = 0;
+ BOOL result = ReadFile(fp_win32, reinterpret_cast<char *>(ptr) + bytes_read,
+ to_read, &chunk_read, NULL);
+ if (!result) {
+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
+ }
+ if (chunk_read < to_read) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+
+ bytes_read += chunk_read;
+ }
+ return;
+ }
+
+ DWORD error = ERROR_SUCCESS;
+ if ((read_offset & (alignment - 1)) != 0 ||
+ (len & (alignment - 1)) != 0 ||
+ (reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)) != 0) {
+ error = ERROR_INVALID_PARAMETER;
+ } else {
+ LARGE_INTEGER li;
+ li.QuadPart = read_offset;
+ if (!SetFilePointerEx(fp_win32_direct, li, NULL, FILE_BEGIN)) {
+ error = GetLastError();
+ }
+ }
+
size_t bytes_read = 0;
- while (bytes_read < len) {
- size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
+ const size_t max_read_size = (size_t) MAXDWORD & ~(alignment - 1);
+ while (error == ERROR_SUCCESS && bytes_read < len) {
+ const DWORD to_read = (DWORD) std::min(len - bytes_read, max_read_size);
DWORD chunk_read = 0;
- BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
+ BOOL result = ReadFile(fp_win32_direct, reinterpret_cast<char *>(ptr) + bytes_read,
+ to_read, &chunk_read, NULL);
if (!result) {
- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
- }
- if (chunk_read < chunk_size || chunk_read == 0) {
- throw std::runtime_error("unexpectedly reached end of file");
+ error = GetLastError();
+ if (error == ERROR_HANDLE_EOF && bytes_read == size - read_offset) {
+ error = ERROR_SUCCESS;
+ break;
+ }
+ break;
}
bytes_read += chunk_read;
+ if (chunk_read < to_read) {
+ if (bytes_read == size - read_offset) {
+ break;
+ }
+ error = ERROR_HANDLE_EOF;
+ }
+ }
+
+ if (error == ERROR_INVALID_PARAMETER || error == ERROR_INVALID_USER_BUFFER || error == ERROR_NOACCESS) {
+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",
+ __func__, fname.c_str(), GetErrorMessageWin32(error).c_str());
+ CloseHandle(fp_win32_direct);
+ fp_win32_direct = INVALID_HANDLE_VALUE;
+ alignment = 1;
+ seek(read_offset, SEEK_SET);
+ read_raw_unsafe(ptr, len);
+ return;
+ }
+ if (error != ERROR_SUCCESS) {
+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(error).c_str()));
+ }
+
+ seek(len > size - read_offset ? size : read_offset + len, SEEK_SET);
+ }
+
+ void read_aligned_chunk(void * dest, size_t len) {
+ if (len == 0) {
+ return;
+ }
+
+ const size_t offset = tell();
+ if (offset > size || len > size - offset) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+
+ const size_t aligned_offset = offset & ~(alignment - 1);
+ const size_t offset_from_alignment = offset - aligned_offset;
+ const size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1);
+ const size_t buffer_size = std::min<size_t>(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE);
+
+ struct aligned_buffer_deleter {
+ void operator()(void * p) const { _aligned_free(p); }
+ };
+ std::unique_ptr<void, aligned_buffer_deleter> buffer(_aligned_malloc(buffer_size, alignment));
+ if (!buffer) {
+ LLAMA_LOG_WARN("%s: Falling back to buffered I/O due to %s\n",
+ __func__, GetErrorMessageWin32(ERROR_NOT_ENOUGH_MEMORY).c_str());
+ CloseHandle(fp_win32_direct);
+ fp_win32_direct = INVALID_HANDLE_VALUE;
+ alignment = 1;
+ seek(offset, SEEK_SET);
+ read_raw_unsafe(dest, len);
+ return;
+ }
+
+ seek(aligned_offset, SEEK_SET);
+
+ size_t skip = offset_from_alignment;
+ size_t copied = 0;
+ for (size_t done = 0; done < bytes_to_read; ) {
+ const size_t n = std::min(buffer_size, bytes_to_read - done);
+ read_raw_unsafe(buffer.get(), n);
+
+ const size_t chunk_skip = std::min(skip, n);
+ const size_t count = std::min(n - chunk_skip, len - copied);
+ memcpy(reinterpret_cast<char *>(dest) + copied, reinterpret_cast<char *>(buffer.get()) + chunk_skip, count);
+
+ copied += count;
+ skip -= chunk_skip;
+ done += n;
+ }
+
+ seek(offset + len, SEEK_SET);
+ }
+
+ void read_raw(void * ptr, size_t len) {
+ if (len == 0) {
+ return;
+ }
+
+ const size_t offset = tell();
+ if (offset > size || len > size - offset) {
+ throw std::runtime_error("unexpectedly reached end of file");
+ }
+
+ if (has_direct_io()) {
+ read_aligned_chunk(ptr, len);
+ } else {
+ read_raw_unsafe(ptr, len);
}
}
@@ -153,7 +372,7 @@ struct llama_file::impl {
void write_raw(const void * ptr, size_t len) const {
size_t bytes_written = 0;
while (bytes_written < len) {
- size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
+ size_t chunk_size = std::min<size_t>(len - bytes_written, LLAMA_DIRECT_IO_BUFFER_SIZE);
DWORD chunk_written = 0;
BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
if (!result) {
@@ -172,23 +391,29 @@ struct llama_file::impl {
}
bool has_direct_io() const {
- return true;
+ return fp_win32_direct != INVALID_HANDLE_VALUE && alignment > 1;
}
~impl() {
+ if (fp_win32_direct != INVALID_HANDLE_VALUE) {
+ CloseHandle(fp_win32_direct);
+ }
if (fp && owns_fp) {
std::fclose(fp);
}
}
+ std::string fname;
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
if (init_fd()) {
+ LLAMA_LOG_INFO("%s: file '%s' opened with direct I/O (alignment: %zu bytes)\n",
+ __func__, fname, alignment);
return;
}
- LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",
+ LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n",
fname, strerror(errno));
}
#endif
@@ -268,8 +493,8 @@ struct llama_file::impl {
}
errno = 0;
if (fd == -1) {
- const size_t curr_off = tell();
- const size_t to_read = std::min(len, size - curr_off);
+ const size_t read_offset = tell();
+ const size_t to_read = std::min(len, size - read_offset);
std::size_t ret = std::fread(ptr, to_read, 1, fp);
if (ferror(fp)) {
@@ -286,24 +511,23 @@ struct llama_file::impl {
if (ret == -1) {
if (errno == EINTR) {
- continue; // Interrupted by signal, retry
+ continue;
}
- // Fallback to std::fread in case the DMA controller cannot access the buffer
if (errno == EFAULT || errno == EINVAL) {
- LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));
- auto curr_off = tell();
+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",
+ __func__, fname.c_str(), strerror(errno));
+ const size_t read_offset = tell();
close(fd);
fd = -1;
alignment = 1;
init_fp("rb");
- seek(curr_off, SEEK_SET);
+ seek(read_offset, SEEK_SET);
read_raw_unsafe(ptr, len);
return;
}
throw std::runtime_error(format("read error: %s", strerror(errno)));
}
if (ret == 0) {
- // EOF: allow if this read was only pulling alignment padding past file end
off_t pos = lseek(fd, 0, SEEK_CUR);
if (pos != -1 && (size_t) pos == size) {
std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);
@@ -317,11 +541,11 @@ struct llama_file::impl {
}
}
- void read_aligned_chunk(void * dest, size_t size) {
+ void read_aligned_chunk(void * dest, size_t len) {
size_t offset = tell();
off_t aligned_offset = offset & ~(alignment - 1);
off_t offset_from_alignment = offset - aligned_offset;
- size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);
+ size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1);
// stage through a bounded buffer, so that a large tensor is not held in memory twice while it loads
const size_t buffer_size = std::min<size_t>(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE);
@@ -329,7 +553,15 @@ struct llama_file::impl {
void * raw_buffer = nullptr;
int ret = posix_memalign(&raw_buffer, alignment, buffer_size);
if (ret != 0) {
- throw std::runtime_error(format("posix_memalign failed with error %d", ret));
+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",
+ __func__, fname.c_str(), strerror(ret));
+ close(fd);
+ fd = -1;
+ alignment = 1;
+ init_fp("rb");
+ seek(offset, SEEK_SET);
+ read_raw_unsafe(dest, len);
+ return;
}
struct aligned_buffer_deleter {
@@ -345,13 +577,15 @@ struct llama_file::impl {
const size_t n = std::min(buffer_size, bytes_to_read - done);
read_raw_unsafe(buffer.get(), n);
- const size_t count = std::min(n - skip, size - copied);
+ const size_t count = std::min(n - skip, len - copied);
memcpy(reinterpret_cast<char *>(dest) + copied, reinterpret_cast<char *>(buffer.get()) + skip, count);
copied += count;
skip = 0;
done += n;
}
+
+ seek(offset + len, SEEK_SET);
}
void read_raw(void * ptr, size_t len) {
@@ -390,7 +624,7 @@ struct llama_file::impl {
~impl() {
if (fd != -1) {
close(fd);
- } else if (owns_fp) {
+ } else if (fp && owns_fp) {
std::fclose(fp);
}
}
@@ -439,11 +673,7 @@ int llama_file::file_id() const {
void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }
void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
-#ifdef _WIN32
-void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
-#else
void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }
-#endif
uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 4224a15ae..4ba52d3cf 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -357,6 +357,13 @@ llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
llama_build_and_test(test-backend-tensor.cpp LINK ggml)
+if (BUILD_SHARED_LIBS)
+ llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior)
+ target_compile_definitions(test-llama-mmap PRIVATE LLAMA_BUILD)
+else()
+ llama_build_and_test(test-llama-mmap.cpp LABEL behavior)
+endif()
+
llama_build_and_test(test-tiled-mulmat.cpp LABEL "main behavior backend" LINK ggml)
llama_build_and_test(test-model-load-cancel.cpp LABEL "model behavior")
llama_build_and_test(test-autorelease.cpp LABEL "model behavior")
diff --git a/tests/test-llama-mmap.cpp b/tests/test-llama-mmap.cpp
new file mode 100644
index 000000000..06f0be188
--- /dev/null
+++ b/tests/test-llama-mmap.cpp
@@ -0,0 +1,232 @@
+#include "testing.h"
+
+#include "../src/llama-mmap.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
+
+#if defined(_WIN32)
+#include <malloc.h>
+#endif
+
+static constexpr size_t test_file_size = LLAMA_DIRECT_IO_BUFFER_SIZE + 3*1024*1024 + 1;
+static constexpr size_t test_io_size = 1024*1024;
+
+static uint8_t test_byte(size_t offset) {
+ // Mix high offset bits into each byte to detect repeated chunks.
+ uint64_t value = offset;
+ value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9ULL;
+ value = (value ^ (value >> 27)) * 0x94d049bb133111ebULL;
+ return static_cast<uint8_t>(value ^ (value >> 31));
+}
+
+static bool matches(const void * data, size_t offset, size_t size) {
+ const auto * bytes = static_cast<const uint8_t *>(data);
+ for (size_t i = 0; i < size; ++i) {
+ if (bytes[i] != test_byte(offset + i)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static void check_test_pattern(testing & t) {
+ uint8_t buffer[256];
+ for (size_t i = 0; i < sizeof(buffer); ++i) {
+ buffer[i] = test_byte(i);
+ }
+ t.assert_true("reject data repeated after 512 KiB", !matches(buffer, 512 * 1024, sizeof(buffer)));
+ t.assert_true("reject data repeated after a direct I/O chunk", !matches(buffer, LLAMA_DIRECT_IO_BUFFER_SIZE, sizeof(buffer)));
+}
+
+struct temporary_file {
+ std::filesystem::path path;
+
+ temporary_file() {
+ const auto now = std::chrono::steady_clock::now().time_since_epoch().count();
+ path = std::filesystem::temp_directory_path() / ("test-llama-mmap-" + std::to_string(now) + ".bin");
+ }
+
+ ~temporary_file() {
+ std::error_code error;
+ std::filesystem::remove(path, error);
+ }
+};
+
+struct aligned_buffer {
+ void * data = nullptr;
+
+ aligned_buffer(size_t alignment, size_t size) {
+ alignment = std::max(alignment, alignof(std::max_align_t));
+#if defined(_WIN32)
+ data = _aligned_malloc(size, alignment);
+#else
+ if (posix_memalign(&data, alignment, size) != 0) {
+ data = nullptr;
+ }
+#endif
+ }
+
+ ~aligned_buffer() {
+#if defined(_WIN32)
+ _aligned_free(data);
+#else
+ free(data);
+#endif
+ }
+
+ aligned_buffer(const aligned_buffer &) = delete;
+ aligned_buffer & operator=(const aligned_buffer &) = delete;
+};
+
+static void write_test_file(const std::filesystem::path & path) {
+ std::ofstream file(path, std::ios::binary);
+ if (!file) {
+ throw std::runtime_error("failed to create test file");
+ }
+
+ std::vector<uint8_t> buffer(test_io_size);
+ for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) {
+ const size_t size = std::min(buffer.size(), test_file_size - offset);
+ for (size_t i = 0; i < size; ++i) {
+ buffer[i] = test_byte(offset + i);
+ }
+ file.write(reinterpret_cast<const char *>(buffer.data()), size);
+ if (!file) {
+ throw std::runtime_error("failed to write test file");
+ }
+ }
+}
+
+static void check_file(testing & t, const std::filesystem::path & path, bool use_direct_io) {
+ llama_file file(path.string().c_str(), "rb", use_direct_io);
+ if (use_direct_io && !file.has_direct_io()) {
+ t.skip("direct I/O is unavailable for the test file");
+ return;
+ }
+ t.assert_equal("requested I/O mode", use_direct_io, file.has_direct_io());
+ t.assert_equal(test_file_size, file.size());
+ t.assert_true("read alignment is nonzero", file.read_alignment() != 0);
+
+ const size_t alignment = file.read_alignment();
+ const std::vector<std::pair<size_t, size_t>> ranges = {
+ { 0, 1 },
+ { 1, 31 },
+ { alignment - 1, alignment + 3 },
+ { LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27 },
+ { test_file_size - 31, 31 },
+ { 1, LLAMA_DIRECT_IO_BUFFER_SIZE + 31 },
+ { 1, test_file_size - 1 },
+ {0, test_file_size}
+ };
+ for (const auto & range : ranges) {
+ std::vector<uint8_t> buffer(range.second);
+ file.seek(range.first, SEEK_SET);
+ file.read_raw(buffer.data(), buffer.size());
+ t.assert_equal("I/O mode after read", use_direct_io, file.has_direct_io());
+ t.assert_true("read expected bytes", matches(buffer.data(), range.first, buffer.size()));
+ t.assert_equal(range.first + range.second, file.tell());
+ }
+
+ std::vector<uint8_t> buffer(test_io_size - 1);
+ file.seek(0, SEEK_SET);
+ for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) {
+ const size_t size = std::min(buffer.size(), test_file_size - offset);
+ file.read_raw(buffer.data(), size);
+ t.assert_equal("I/O mode after sequential read", use_direct_io, file.has_direct_io());
+ t.assert_true("sequential read expected bytes", matches(buffer.data(), offset, size));
+ t.assert_equal(offset + size, file.tell());
+ }
+
+ const size_t unsafe_size = ((LLAMA_DIRECT_IO_BUFFER_SIZE + 3 * alignment - 1) / alignment) * alignment;
+ aligned_buffer buffer_aligned(alignment, unsafe_size);
+ t.assert_true("allocate aligned buffer", buffer_aligned.data != nullptr);
+ if (buffer_aligned.data != nullptr) {
+ const size_t offset = 7 * alignment;
+ file.seek(offset, SEEK_SET);
+ file.read_raw_unsafe(buffer_aligned.data, unsafe_size);
+ t.assert_equal("I/O mode after unsafe read", use_direct_io, file.has_direct_io());
+ t.assert_true("unsafe read expected bytes", matches(buffer_aligned.data, offset, unsafe_size));
+ t.assert_equal(offset + unsafe_size, file.tell());
+
+ const size_t final_offset = (test_file_size - 1) & ~(alignment - 1);
+ file.seek(final_offset, SEEK_SET);
+ file.read_raw_unsafe(buffer_aligned.data, alignment);
+ t.assert_equal("I/O mode after unsafe EOF read", use_direct_io, file.has_direct_io());
+ t.assert_true("unsafe EOF read expected bytes", matches(buffer_aligned.data, final_offset, test_file_size - final_offset));
+ t.assert_equal(test_file_size, file.tell());
+ }
+}
+
+static void check_direct_io_fallback(testing & t, const std::filesystem::path & path) {
+ llama_file file(path.string().c_str(), "rb", true);
+ if (!file.has_direct_io()) {
+ t.skip("direct I/O is unavailable for the test file");
+ return;
+ }
+ t.assert_true("direct I/O is enabled before fallback", file.has_direct_io());
+
+ const size_t alignment = file.read_alignment();
+ aligned_buffer buffer(alignment, alignment + 1);
+ if (!t.assert_true("allocate aligned buffer", buffer.data != nullptr)) {
+ return;
+ }
+ auto * unaligned = static_cast<uint8_t *>(buffer.data) + 1;
+ const size_t offset = 7 * alignment;
+ file.seek(offset, SEEK_SET);
+ file.read_raw_unsafe(unaligned, alignment);
+
+#ifdef _WIN32
+ t.assert_true("unaligned read disables direct I/O", !file.has_direct_io());
+ t.assert_equal(size_t(1), file.read_alignment());
+#endif
+ t.assert_true("fallback read expected bytes", matches(unaligned, offset, alignment));
+ t.assert_equal(offset + alignment, file.tell());
+
+ file.read_raw(unaligned, alignment);
+#ifdef _WIN32
+ t.assert_true("subsequent read stays buffered", !file.has_direct_io());
+#endif
+ t.assert_true("subsequent read expected bytes", matches(unaligned, offset + alignment, alignment));
+ t.assert_equal(offset + 2 * alignment, file.tell());
+}
+
+static void check_mmap(testing & t, const std::filesystem::path & path) {
+ if (!llama_mmap::SUPPORTED) {
+ return;
+ }
+
+ llama_file file(path.string().c_str(), "rb");
+ llama_mmap mapping(&file, 0, false);
+ t.assert_equal(test_file_size, mapping.size());
+ const auto * data = static_cast<const uint8_t *>(mapping.addr());
+ t.assert_true("mapping has an address", data != nullptr);
+ if (data != nullptr) {
+ t.assert_true("first byte", matches(data, 0, 1));
+ t.assert_true("chunk boundary", matches(data + LLAMA_DIRECT_IO_BUFFER_SIZE - 11, LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27));
+ t.assert_true("last bytes", matches(data + test_file_size - 31, test_file_size - 31, 31));
+ }
+}
+
+int main() {
+ testing t;
+ temporary_file test_file;
+ t.test("test_pattern", [&] (testing & t) { check_test_pattern(t); });
+ t.test("write_test_file", [&] (testing &) { write_test_file(test_file.path); });
+
+ t.test("llama_file.buffered", [&] (testing & t) { check_file(t, test_file.path, false); });
+ t.test("llama_file.direct", [&] (testing & t) { check_file(t, test_file.path, true); });
+ t.test("llama_file.direct_fallback", [&] (testing & t) { check_direct_io_fallback(t, test_file.path); });
+ t.test("llama_mmap", [&] (testing & t) { check_mmap(t, test_file.path); });
+ return t.summary();
+}