Commit 6122b566b for llama.cpp
commit 6122b566b0a93aac37d1c25f107333c39dd96b8a
Author: figkcoci-debug <figkcoci@gmail.com>
Date: Tue Nov 10 12:40:35 2026 +0500
ggml-rpc : fail gracefully on unreachable endpoints and failed handshakes (#30415)
* ggml-rpc: fail gracefully on unreachable endpoints and failed handshakes
rpc_dispatcher::start() aborted the whole process (GGML_ABORT -> SIGABRT) when
the endpoint could not be parsed, the transport failed to initialise, the socket
could not connect, or the HELLO handshake failed. A typo in --rpc or a
temporarily down node killed the host application, and the reason went to a log
stream the host may never see.
Make the connect-time failure reportable instead of fatal: start() returns bool
(LOG_ERROR + false; the dispatcher thread only starts after a successful
handshake), negotiate_hello() returns false on a half-open socket,
get_dispatcher() returns nullptr and does not cache failed attempts, and every
consumer treats nullptr as "endpoint did not answer" with the conservative
behaviour of its domain (allocation -> nullptr, alloc-size -> lower bound,
device memory -> 0/0, events -> no-op, device count -> 0, and add_server()
already maps 0 devices to nullptr).
Mid-stream failures keep the existing RPC_STATUS_ASSERT by design (fail fast
against silent corruption) and are intentionally out of scope.
Field motivation: on Android (llama.rn + phone RPC cluster) an unreachable
worker crashed the app; the crash buffer showed
abort <- ggml_abort <- rpc_dispatcher::start <- ggml_backend_rpc_add_server.
* tests : add a regression test for unreachable rpc endpoints
Calls ggml_backend_rpc_add_server with a socket that nothing listens on and with a
malformed endpoint, and expects nullptr from both. Without the fix in this branch
the first call aborts the process, so the test cannot even fail cleanly.
* ggml-rpc remove low-value comments
* tests : drop the internal device-count check after the rpc registry refactor
---------
Co-authored-by: figkcoci-debug <figkcoci-debug@users.noreply.github.com>
diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp
index ffa9118f7..e4a84cb0c 100644
--- a/ggml/src/ggml-rpc/ggml-rpc.cpp
+++ b/ggml/src/ggml-rpc/ggml-rpc.cpp
@@ -444,7 +444,10 @@ static bool negotiate_hello(const std::shared_ptr<socket_t> & sock, rpc_dispatch
sock->get_caps(request.conn_caps);
bool status = send_rpc_cmd(sock, RPC_CMD_HELLO, &request, sizeof(request), &response, sizeof(response));
- RPC_STATUS_ASSERT(status);
+ if (!status) {
+ LOG_ERROR("RPC hello exchange with the server failed\n");
+ return false;
+ }
if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) {
LOG_ERROR("RPC server version mismatch: %d.%d.%d\n",
@@ -526,7 +529,7 @@ public:
void busy_spin_release();
void graph_compute(uint32_t device, const ggml_cgraph * cgraph);
- void start(const std::string & endpoint);
+ bool start(const std::string & endpoint);
void work();
bool get_send_tensor_hash() const { return flags & RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH; }
@@ -641,26 +644,31 @@ void rpc_dispatcher::busy_spin_release() {
GGML_ASSERT(previous > 0);
}
-void rpc_dispatcher::start(const std::string & endpoint) {
+bool rpc_dispatcher::start(const std::string & endpoint) {
std::string host;
int port;
if (!parse_endpoint(endpoint, host, port)) {
- GGML_ABORT("Failed to parse endpoint: %s\n", endpoint.c_str());
+ LOG_ERROR("Failed to parse endpoint: %s\n", endpoint.c_str());
+ return false;
}
if (!rpc_transport_init()) {
- GGML_ABORT("RPC transport initialization failed\n");
+ LOG_ERROR("RPC transport initialization failed\n");
+ return false;
}
sock = socket_t::connect(host.c_str(), port);
if (sock == nullptr) {
- GGML_ABORT("Failed to connect to %s\n", endpoint.c_str());
+ LOG_ERROR("Failed to connect to %s\n", endpoint.c_str());
+ return false;
}
if (!negotiate_hello(sock, flags)) {
- GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str());
+ LOG_ERROR("RPC handshake failed for %s\n", endpoint.c_str());
+ return false;
}
LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str());
running = true;
thread = std::thread(rpc_dispatcher_trampoline, this);
+ return true;
}
void rpc_dispatcher::work() {
@@ -724,7 +732,9 @@ static std::shared_ptr<rpc_dispatcher> get_dispatcher(const std::string & endpoi
}
auto dispatcher = std::make_shared<rpc_dispatcher>();
- dispatcher->start(endpoint);
+ if (!dispatcher->start(endpoint)) {
+ return nullptr;
+ }
dispatchers[endpoint] = dispatcher;
return dispatcher;
}
@@ -999,6 +1009,9 @@ static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_back
rpc_msg_alloc_buffer_rsp response;
auto dispatcher = get_dispatcher(buft_ctx->endpoint);
+ if (dispatcher == nullptr) {
+ return nullptr;
+ }
dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), nullptr, 0, &response, sizeof(response));
if (response.remote_ptr != 0) {
LOG_DBG("[%s] endpoint: %s, device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n",
@@ -1119,6 +1132,9 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty
LOG_DBG2("[%s] cache miss: %s, querying server\n", __func__, rpc_log_format_tensor(tensor).c_str());
rpc_msg_get_alloc_size_rsp response;
auto dispatcher = get_dispatcher(buft_ctx->endpoint);
+ if (dispatcher == nullptr) {
+ return min_size;
+ }
dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), nullptr, 0, &response, sizeof(response));
{
@@ -1339,6 +1355,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u
return it->second;
}
auto dispatcher = get_dispatcher(endpoint);
+ if (dispatcher == nullptr) {
+ return nullptr;
+ }
size_t alignment = get_alignment(dispatcher, device);
size_t max_size = get_max_size(dispatcher, device);
ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context {
@@ -1363,6 +1382,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u
ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) {
std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]";
auto dispatcher = get_dispatcher(endpoint);
+ if (dispatcher == nullptr) {
+ return nullptr;
+ }
ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context {
/* .dispatcher = */ dispatcher,
/* .endpoint = */ endpoint,
@@ -1386,6 +1408,11 @@ bool ggml_backend_is_rpc(ggml_backend_t backend) {
void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) {
auto dispatcher = get_dispatcher(endpoint);
+ if (dispatcher == nullptr) {
+ *free = 0;
+ *total = 0;
+ return;
+ }
auto request = std::make_shared<rpc_msg_get_device_memory_req>();
request->device = device;
rpc_msg_get_device_memory_rsp response;
@@ -3103,18 +3130,27 @@ static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_b
static ggml_backend_event_t ggml_backend_rpc_device_event_new(ggml_backend_dev_t dev) {
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
auto dispatcher = get_dispatcher(ctx->endpoint);
+ if (dispatcher == nullptr) {
+ return nullptr;
+ }
return dispatcher->event_new(dev);
}
static void ggml_backend_rpc_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
auto dispatcher = get_dispatcher(ctx->endpoint);
+ if (dispatcher == nullptr) {
+ return;
+ }
dispatcher->event_free(event);
}
static void ggml_backend_rpc_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
auto dispatcher = get_dispatcher(ctx->endpoint);
+ if (dispatcher == nullptr) {
+ return;
+ }
dispatcher->event_synchronize(event);
}
@@ -3334,6 +3370,9 @@ ggml_backend_reg_t ggml_backend_rpc_reg(void) {
static uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) {
auto dispatcher = get_dispatcher(endpoint);
+ if (dispatcher == nullptr) {
+ return 0;
+ }
rpc_msg_device_count_rsp response;
dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, nullptr, 0, &response, sizeof(response));
return response.device_count;
diff --git a/tests/test-rpc-unreachable.cpp b/tests/test-rpc-unreachable.cpp
new file mode 100644
index 000000000..50e1f2d44
--- /dev/null
+++ b/tests/test-rpc-unreachable.cpp
@@ -0,0 +1,33 @@
+#include "ggml-backend.h"
+#include "ggml-rpc.h"
+#include "ggml.h"
+
+#include <cstdio>
+
+// Regression test: an unreachable or malformed RPC endpoint must be reported as an
+// error instead of aborting the calling process.
+//
+// Before the fix, ggml_backend_rpc_add_server() called GGML_ABORT (SIGABRT) when the
+// endpoint could not be parsed, the transport failed to initialize, the socket could
+// not connect, or the HELLO handshake failed - so a typo in an endpoint or a node
+// that was still booting killed the whole application, and this test would die
+// instead of failing cleanly.
+//
+// After the fix it returns nullptr.
+//
+// ref: https://github.com/ggml-org/llama.cpp/pull/30415
+
+int main(void) {
+ ggml_backend_load_all();
+
+ // 1) nothing listens on this port: the connect must fail, not abort
+ ggml_backend_reg_t reg = ggml_backend_rpc_add_server("127.0.0.1:1");
+ GGML_ASSERT(reg == nullptr);
+
+ // 2) a malformed endpoint must take the parse-failure path and also return nullptr
+ reg = ggml_backend_rpc_add_server("this-is-not-an-endpoint");
+ GGML_ASSERT(reg == nullptr);
+
+ printf("ok\n");
+ return 0;
+}
diff --git a/tools/rpc/CMakeLists.txt b/tools/rpc/CMakeLists.txt
index 2891c7d03..20c1f087e 100644
--- a/tools/rpc/CMakeLists.txt
+++ b/tools/rpc/CMakeLists.txt
@@ -13,6 +13,11 @@ if (LLAMA_BUILD_TESTS AND UNIX AND NOT GGML_BACKEND_DL)
$<TARGET_FILE:ggml-rpc-server>
$<TARGET_FILE:test-rpc-multi-server>)
set_property(TEST test-rpc-multi-server PROPERTY LABELS main)
+ add_executable(test-rpc-unreachable ${PROJECT_SOURCE_DIR}/tests/test-rpc-unreachable.cpp)
+ target_link_libraries(test-rpc-unreachable PRIVATE ggml ggml-rpc)
+ target_include_directories(test-rpc-unreachable PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
+ add_test(NAME test-rpc-unreachable COMMAND test-rpc-unreachable)
+ set_property(TEST test-rpc-unreachable PROPERTY LABELS main)
endif()
if(LLAMA_TOOLS_INSTALL)