Commit a5d22276b for llama.cpp
commit a5d22276be846d95958412e7042af20b0fc245f3
Author: Tim Wang <149349643+timothywang21@users.noreply.github.com>
Date: Thu Oct 22 07:27:24 2026 -0400
server : support typed content input for RERANK endpoint (#29625)
* server : support typed content input for rerank endpoint
Assisted-by: Opencode/Qwen3.8 27B
* server : address rerank multimodal review findings
- reject multimodal rerank input when the model has no rerank chat template (images were silently dropped)
- translate handle_media download/base64 failures to 400 instead of 500
- reject negative top_n with 400 instead of letting resize throw
- fix comment typos, redundant comments, indentation and trailing whitespace
- README: document that TEI input accepts the same shape as Jina; drop stray blank line
Assisted-by: opencode
* Trimmed README
---------
Co-authored-by: timothywang21 <timothywang21@users.noreply.github.com>
diff --git a/tools/server/README.md b/tools/server/README.md
index 71310d903..56d1ea5fe 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -771,37 +771,81 @@ This endpoint also supports multimodal embeddings. See the documentation for the
### POST `/reranking`: Rerank documents according to a given query
+> [!IMPORTANT]
+>
+> This endpoint is **not** OAI-compatible.
Similar to https://jina.ai/reranker/ but might change in the future.
-Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` options.
-
-*Options:*
+Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` or `--reranking` options.
-`query`: The query against which the documents will be ranked.
+`query` and each `documents`/`texts` element accept a plain string, or
+(for multimodal rerankers such as Qwen3-VL-Reranker) an object of the form
+`{"text": string | [strings], "image": url | data-uri | raw-base64 | [urls]}`.
+All keys are optional; a plain string is treated as `{"text": "..."}`.
-`documents`: An array strings representing the documents to be ranked.
+Options: `top_n` (default: all), `model` (echoed in the Jina response), `return_text` (TEI only).
*Aliases:*
- `/rerank`
- `/v1/rerank`
- `/v1/reranking`
-*Examples:*
+The format is selected by which key carries the documents: `texts` (TEI, same input shape as Jina,
+set `return_text: true` to echo document text) or `documents` (Jina, multimodal supported).
+
+**Text only (TEI format)** - `query` + `texts`; set `return_text: true` to echo document text:
```shell
curl http://127.0.0.1:8012/v1/rerank \
-H "Content-Type: application/json" \
-d '{
- "model": "some-model",
- "query": "What is panda?",
- "top_n": 3,
- "documents": [
- "hi",
- "it is a bear",
- "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China."
- ]
+ "query": "What is machine learning?",
+ "texts": [
+ "Machine learning is a subset of artificial intelligence.",
+ "A python is a large snake."
+ ],
+ "return_text": true
}' | jq
```
+**Multimodal (Jina Format)** - mix text and images in `query` and/or `documents`:
+
+```shell
+curl http://127.0.0.1:8012/v1/rerank \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "Qwen/Qwen3-VL-Reranker-2B",
+ "query": {
+ "text": "A city skyline with tall buildings and a river",
+ "image": "data:image/jpeg;base64,<base64>"
+ },
+ "documents": [
+ "a plain string document is also valid",
+ {
+ "text": "A photograph of the Boston skyline at dusk",
+ "image": "https://example.com/skyline.jpg"
+ },
+ {
+ "text": "caption for the two-figure spread",
+ "image": ["https://example.com/fig1.png", "data:image/jpeg;base64,<base64>"]
+ }
+ ],
+ "top_n": 2
+ }' | jq
+```
+
+Jina format returns a wrapped object with `model`, `object`, `usage` (`prompt_tokens` is the sum
+of `tokens_evaluated` across all scored pairs), and a `results` array of `{index, relevance_score}`.
+TEI format returns a bare array of `{index, score}` (+ `text` when `return_text: true`).
+Both are sorted descending by score and truncated to `top_n`; the score is the match probability in `[0, 1]`.
+
+Images never appear in any response shape.
+
+> **NOTE:** **Input size limit:** A bidirectional-encoder reranker (e.g. `bge-reranker-v2-m3`) must fit the whole prompt within
+`--ubatch-size`; if the input is too large to process, increase `--ubatch-size`.
+Causal decoder rerankers (e.g. `Qwen3-VL-Reranker`) have no such limitation.
+
+
+
### POST `/tts`: Generate speech audio from text
Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index baaca58eb..4f1acba1a 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -1265,7 +1265,6 @@ void handle_media(
out_files.push_back(decoded_data);
}
}
-
// load media files from an OAI content array, then replace each media part with a media marker text part
static void oaicompat_content_load_media(json & content, const server_chat_params & opt, std::vector<raw_buffer> & out_files) {
for (auto & p : content) {
@@ -1336,6 +1335,7 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context
if (files.empty()) {
return server_tokens(common_tokenize(vocab, prompt, add_special, parse_special), false);
}
+
return process_mtmd_prompt(mctx, prompt, files, init_opt);
}
@@ -1951,18 +1951,49 @@ server_tokens format_prompt_rerank(
mtmd_context * mctx,
const std::string & query,
const std::string & doc,
+ const std::vector<raw_buffer> & query_files,
+ const std::vector<raw_buffer> & doc_files,
const mtmd_helper_init_opt & init_opt) {
server_tokens result = {};
+ // media must be spliced through the mtmd pipeline
+ if ((!query_files.empty() || !doc_files.empty()) && mctx == nullptr) {
+ throw std::invalid_argument("multimodal rerank input requires --mmproj");
+ }
+
const char * rerank_prompt = llama_model_chat_template(model, "rerank");
if (rerank_prompt != nullptr) {
+ // multimodal: append one media marker per image (arrays already flattened by caller)
+ std::string q = query;
+ for (size_t i = 0; i < query_files.size(); i++) {
+ q += get_media_marker();
+ }
+ std::string d = doc;
+ for (size_t i = 0; i < doc_files.size(); i++) {
+ d += get_media_marker();
+ }
+
std::string prompt = rerank_prompt;
- string_replace_all(prompt, "{query}" , query);
- string_replace_all(prompt, "{document}", doc );
- server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
- result.push_back(tokens);
+ string_replace_all(prompt, "{query}" , q);
+ string_replace_all(prompt, "{document}", d );
+
+ if (query_files.empty() && doc_files.empty()) {
+ server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
+ result.push_back(tokens);
+ } else {
+ // process_mtmd_prompt returns a complete mtmd server_tokens; return it directly
+ // (push_back into a non-mtmd result would hit the NULL placeholder tokens)
+ std::vector<raw_buffer> files;
+ files.insert(files.end(), query_files.begin(), query_files.end());
+ files.insert(files.end(), doc_files.begin(), doc_files.end());
+ return process_mtmd_prompt(mctx, prompt, files, init_opt);
+ }
} else {
+ if (!query_files.empty() || !doc_files.empty()) {
+ throw std::invalid_argument("multimodal rerank input requires a model with a rerank chat template");
+ }
+
// Get EOS token - use SEP token as fallback if EOS is not available
server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);
server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false, init_opt);
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 538e008e9..8bbff7726 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -303,8 +303,11 @@ server_tokens process_mtmd_prompt(
mtmd_context * mctx,
const std::string & prompt,
const std::vector<raw_buffer> & files,
- const mtmd_helper_init_opt & init_opt,
- bool is_placeholder = false);
+ const mtmd_helper_init_opt & init_opt,
+ bool is_placeholder = false);
+
+// decode one media url (http/file/data-uri) into out_files; throws on failure
+void handle_media(std::vector<raw_buffer> & out_files, const std::string & url, const std::string & media_path);
/**
* break the input "prompt" object into multiple prompt if needed, then tokenize them
@@ -590,12 +593,16 @@ llama_tokens format_prompt_infill(
const llama_tokens & tokens_prompt);
// format rerank task: [BOS]query[EOS][SEP]doc[EOS].
+// query_files/doc_files carry decoded media for multimodal (Qwen) inputs; when non-empty the
+// prompt is built with get_media_marker() sentinels and spliced via process_mtmd_prompt.
server_tokens format_prompt_rerank(
const struct llama_model * model,
const struct llama_vocab * vocab,
mtmd_context * mctx,
const std::string & query,
const std::string & doc,
+ const std::vector<raw_buffer> & query_files,
+ const std::vector<raw_buffer> & doc_files,
const mtmd_helper_init_opt & init_opt);
// simple implementation of a pipe
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 296893f7a..a0acf1655 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6284,83 +6284,7 @@ void server_routes::init_routes() {
};
this->post_rerank = [this](const server_http_req & req) {
- auto res = create_response();
- if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {
- res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));
- return res;
- }
-
- const json body = json::parse(req.body);
-
- // if true, use TEI API format, otherwise use Jina API format
- // Jina: https://jina.ai/reranker/
- // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank
- bool is_tei_format = body.contains("texts");
-
- json query;
- if (body.count("query") == 1) {
- query = body.at("query");
- if (!query.is_string()) {
- res->error(format_error_response("\"query\" must be a string", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- } else {
- res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
-
- std::vector<std::string> documents = json_value(body, "documents",
- json_value(body, "texts", std::vector<std::string>()));
- if (documents.empty()) {
- res->error(format_error_response("\"documents\" must be a non-empty string array", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
-
- int top_n = json_value(body, "top_n", (int)documents.size());
-
- // create and queue the task
- json responses = json::array();
- auto & rd = res->rd;
- {
- std::vector<server_task> tasks;
- tasks.reserve(documents.size());
- for (size_t i = 0; i < documents.size(); i++) {
- auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);
- server_task task = server_task(SERVER_TASK_TYPE_RERANK);
- task.id = rd.get_new_id();
- task.tokens = std::move(tmp);
- tasks.push_back(std::move(task));
- }
- rd.post_tasks(std::move(tasks));
- }
-
- // wait for the results
- auto all_results = rd.wait_for_all(req.should_stop);
-
- // collect results
- if (all_results.is_terminated) {
- return res; // connection is closed
- } else if (all_results.error) {
- res->error(all_results.error->to_json());
- return res;
- } else {
- for (auto & res : all_results.results) {
- GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
- responses.push_back(res->to_json());
- }
- }
-
- // write JSON response
- json root = format_response_rerank(
- body,
- meta->model_name,
- responses,
- is_tei_format,
- documents,
- top_n);
-
- res->ok(root);
- return res;
+ return handle_rerank_impl(req);
};
this->post_tts = [this](const server_http_req & req) {
@@ -6822,6 +6746,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
return tokenize_input_subprompt(ctx_server.vocab, ctx_server.mctx, p, true, true, ctx_server.init_opt);
};
+ // dispatch to the appropriate tokenizer based on the shape of the input
std::vector<server_tokens> tokenized_prompts;
if (prompt.is_array() && !json_is_array_and_contains_numbers(prompt)) {
for (const auto & p : prompt) {
@@ -6895,6 +6820,154 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
return res;
}
+std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const server_http_req & req) {
+ auto res = create_response();
+ if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {
+ res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
+ const json body = json::parse(req.body);
+
+ // if true, use TEI API format, otherwise use Jina API format
+ // Jina: https://jina.ai/reranker/
+ // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank
+ bool is_tei_format = body.contains("texts");
+
+ // expand one rerank input item: plain string or {"text": str|[str], "image": url|data-uri|raw-base64|[urls]}
+ auto parse_item = [&](const json & item, std::string & out_text, std::vector<raw_buffer> & out_files) {
+ if (item.is_string()) {
+ out_text = item.get<std::string>();
+ return;
+ }
+ if (!item.is_object()) {
+ throw std::invalid_argument("rerank input must be a string or an object of the form {\"text\": ..., \"image\": ...}");
+ }
+ if (item.contains("video") || item.contains("fps") || item.contains("max_frames")) {
+ throw std::invalid_argument("video/fps/max_frames are not supported for rerank input");
+ }
+ json text = item.value("text", json());
+ if (text.is_string()) {
+ out_text += text.get<std::string>();
+ } else if (text.is_array()) {
+ for (const auto & t : text) {
+ if (!t.is_string()) {
+ throw std::invalid_argument("\"text\" array elements must be strings");
+ }
+ out_text += t.get<std::string>();
+ }
+ } else if (!text.is_null()) {
+ throw std::invalid_argument("\"text\" must be a string or an array of strings");
+ }
+ json image = item.value("image", json());
+ auto add_image = [&](const std::string & url) {
+ // marker is appended by format_prompt_rerank, one per decoded file
+ try {
+ handle_media(out_files, url, params.media_path);
+ } catch (const std::runtime_error & e) {
+ throw std::invalid_argument(e.what());
+ }
+ };
+ if (image.is_string()) {
+ add_image(image.get<std::string>());
+ } else if (image.is_array()) {
+ for (const auto & u : image) {
+ if (!u.is_string()) {
+ throw std::invalid_argument("\"image\" array elements must be strings");
+ }
+ add_image(u.get<std::string>());
+ }
+ } else if (!image.is_null()) {
+ throw std::invalid_argument("\"image\" must be a string or an array of strings");
+ }
+ };
+
+ // Parse query
+ std::string query_str;
+ std::vector<raw_buffer> query_files;
+ if (body.count("query") == 1) {
+ try {
+ parse_item(body.at("query"), query_str, query_files);
+ } catch (const std::invalid_argument & e) {
+ res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ } else {
+ res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ // Resolve Jina/TEI format ambiguity
+ json documents_json = json_value(body, "documents",
+ json_value(body, "texts", json::array()));
+ if (!documents_json.is_array() || documents_json.empty()) {
+ res->error(format_error_response("\"documents\" must be a non-empty array of strings or objects", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ // Parse documents
+ std::vector<std::string> documents(documents_json.size());
+ std::vector<std::vector<raw_buffer>> document_files(documents_json.size());
+ for (size_t i = 0; i < documents_json.size(); i++) {
+ try {
+ parse_item(documents_json[i], documents[i], document_files[i]);
+ } catch (const std::invalid_argument & e) {
+ res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ }
+
+ int top_n = json_value(body, "top_n", (int)documents.size());
+ if (top_n < 0) {
+ res->error(format_error_response("\"top_n\" must be >= 0", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ // create and queue the task
+ json responses = json::array();
+ auto & rd = res->rd;
+ {
+ std::vector<server_task> tasks;
+ tasks.reserve(documents.size());
+ for (size_t i = 0; i < documents.size(); i++) {
+ auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query_str, documents[i], query_files, document_files[i], ctx_server.init_opt);
+ server_task task = server_task(SERVER_TASK_TYPE_RERANK);
+ task.id = rd.get_new_id();
+ task.tokens = std::move(tmp);
+ tasks.push_back(std::move(task));
+ }
+ rd.post_tasks(std::move(tasks));
+ }
+
+ // wait for the results
+ auto all_results = rd.wait_for_all(req.should_stop);
+
+ // collect results
+ if (all_results.is_terminated) {
+ return res; // connection is closed
+ } else if (all_results.error) {
+ res->error(all_results.error->to_json());
+ return res;
+ } else {
+ for (auto & res : all_results.results) {
+ GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
+ responses.push_back(res->to_json());
+ }
+ }
+
+ // write JSON response
+ json root = format_response_rerank(
+ body,
+ meta->model_name,
+ responses,
+ is_tei_format,
+ documents,
+ top_n);
+
+ res->ok(root);
+ return res;
+}
+
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {
auto res = create_response();
std::vector<raw_buffer> files;
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index a579b0eca..4a8e327fc 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -177,6 +177,7 @@ private:
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+ std::unique_ptr<server_res_generator> handle_rerank_impl(const server_http_req & req);
std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
// using unique_ptr to allow late initialization of const