Commit 5d78740 for stable-diffusion.cpp
commit 5d787408c0e053568c4558e3c9043e8ebabd4cae
Author: leejet <leejet714@gmail.com>
Date: Sat Nov 14 01:24:05 2026 +0800
feat: support interleaved image prompts for vision LLMs (#2341)
diff --git a/docs/prompt_images.md b/docs/prompt_images.md
new file mode 100644
index 0000000..01a8703
--- /dev/null
+++ b/docs/prompt_images.md
@@ -0,0 +1,106 @@
+# Images in prompts
+
+Use `<image:source>` to place an image between text fragments. Multiple images
+are read in prompt order, including adjacent or repeated references:
+
+```text
+Use the subject from <image:subject.png> with the colors from <image:palette.png>.
+```
+
+Inline images are available to conditioners using Qwen2.5-VL, Qwen3-VL, or
+Qwen3.5 with compatible vision weights. Supply the vision encoder with
+`--llm_vision` or use a combined text/vision encoder. Each model keeps its own
+prompt template, text weighting, padding, and output-layer selection. Missing
+vision weights or a text-only LLM produce an error. See [MOSS-Image setup](moss_image.md)
+for a Qwen3.5 example.
+
+## CLI
+
+Pass image markers in the usual `-p` or `-n` argument. Relative image paths are
+resolved against the current working directory. Spaces, colons, and parentheses
+are allowed inside paths, including Windows paths:
+
+```text
+-p "Draw <image:inputs/subject.png> in the style of <image:C:\references\ink study.png>"
+```
+
+A path must be nonempty and cannot contain `>`. Empty or unclosed markers are
+errors. Write `\<image:example.png>` to include a marker as literal text. A marker
+is escaped when preceded by an odd number of backslashes; one backslash is
+removed. Backslashes inside image paths are preserved.
+
+Images contribute visual information at their specified positions. They do not
+automatically become edit source images or determine the output dimensions.
+Continue using `--ref-image` and `ref_images` for existing editing workflows.
+Positive and negative prompts resolve their own image markers independently.
+
+The context limit includes visual tokens and the model's prompt template. An
+image-containing prompt that exceeds the limit fails; images are never partially
+truncated.
+
+Each prompt supports at most 32 unique image sources, checked before loading any
+of them. Repeated sources share their resolved pixels. Qwen2.5-VL also limits
+each resized image to 8192 vision patches (2048 visual tokens), keeping its
+window mask within 256 MiB. Larger grids fail before vision encoding; use a
+smaller source image if this limit is exceeded.
+
+## C API
+
+Register a context-local image resolver with `sd_set_prompt_image_callback()`,
+then call the existing `generate_image()` or `generate_video()` function. The
+library passes each marker's `source` string unchanged to the callback, allowing
+paths, application resource IDs, or in-memory images. The library does not open
+files or fetch URLs itself.
+
+```c
+bool load_prompt_image(const char* source, void* user_data, sd_image_t* image_out);
+
+params.prompt = "Draw the subject from <image:subject> in a watercolor landscape.";
+sd_image_generation_result_t* result = NULL;
+sd_set_prompt_image_callback(ctx, load_prompt_image, application_images);
+bool ok = generate_image(ctx, ¶ms, &result);
+sd_set_prompt_image_callback(ctx, NULL, NULL);
+```
+
+The loader returns `true` with an image containing interleaved 8-bit pixels and
+one to four channels. The caller owns the pixels. The library copies them before
+calling the loader again, so a loader may reuse its buffer. Repeated sources
+within one prompt are loaded once and appear at every specified position.
+Resolved image pixels participate in conditioning-cache comparisons.
+
+Register or clear the callback while the context is idle. Different contexts
+have independent callbacks. Generation parameter structures and generation
+entry points are unchanged.
+
+## Server
+
+Native `/sdcpp/v1` image and video requests accept a `prompt_images` object
+mapping IDs to base64 images or image data URIs:
+
+```json
+{
+ "prompt": "Use <image:subject> with the colors from <image:palette>.",
+ "prompt_images": {
+ "subject": "data:image/png;base64,...",
+ "palette": "data:image/png;base64,..."
+ }
+}
+```
+
+For OpenAI-compatible `/v1/images/generations`, `/v1/images/edits`, and
+`/sdapi/v1` image requests, pass `prompt_images` only through the existing
+`sd_cpp_extra_args` block embedded in `prompt`:
+
+```json
+{
+ "prompt": "Use <image:subject>. <sd_cpp_extra_args>{\"prompt_images\":{\"subject\":\"data:image/png;base64,...\"}}</sd_cpp_extra_args>"
+}
+```
+
+For multipart `/v1/images/edits` requests, put the same block in the `prompt`
+form field. Standard `image` and `image[]` uploads continue to be edit references.
+Compatibility APIs do not read separate `prompt_images` fields or file uploads.
+
+The server resolves IDs only against the images owned by that request. An ID
+never opens a server file or fetches a URL, and queued requests retain their own
+image data until execution finishes.
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index 2bff050..44462eb 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -976,6 +976,20 @@ int main(int argc, const char* argv[]) {
return 1;
}
+ SDImageOwner prompt_image;
+ sd_set_prompt_image_callback(
+ sd_ctx.get(), [](const char* source, void* data, sd_image_t* image_out) -> bool {
+ auto& image = *static_cast<SDImageOwner*>(data);
+ sd_image_t loaded{};
+ if (!load_sd_image_from_file(&loaded, source, 0, 0, 3)) {
+ return false;
+ }
+ image.reset(loaded);
+ *image_out = loaded;
+ return true;
+ },
+ &prompt_image);
+
sd_img_gen_params_t img_gen_params{};
const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
if (use_img_gen_params) {
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 102bf95..d92c819 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -2199,6 +2199,26 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str
return true;
}
+bool SDGenerationParams::parse_prompt_images_json(const std::string& json_str) {
+ auto images = json::parse(json_str, nullptr, false);
+ if (!images.is_object()) {
+ return false;
+ }
+ std::map<std::string, SDImageOwner> decoded;
+ for (auto it = images.begin(); it != images.end(); ++it) {
+ if (it.key().empty() || it.key().find('>') != std::string::npos || !it.value().is_string()) {
+ return false;
+ }
+ SDImageOwner image;
+ if (!decode_base64_image(it.value().get<std::string>(), 3, 0, 0, image)) {
+ return false;
+ }
+ decoded.emplace(it.key(), std::move(image));
+ }
+ prompt_images = std::move(decoded);
+ return true;
+}
+
/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */
bool SDGenerationParams::from_json_str(
const std::string& json_str,
@@ -2213,6 +2233,8 @@ bool SDGenerationParams::from_json_str(
if (j.contains("image_preprocess") && !parse_image_preprocess_json(j["image_preprocess"].dump()))
return false;
+ if (j.contains("prompt_images") && !parse_prompt_images_json(j["prompt_images"].dump()))
+ return false;
auto load_if_exists = [&](const char* key, auto& out) {
if (j.contains(key)) {
diff --git a/examples/common/common.h b/examples/common/common.h
index d4df4fa..43e11ef 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -299,6 +299,7 @@ struct SDGenerationParams {
SDImageOwner init_image;
SDImageOwner end_image;
std::vector<SDImageOwner> ref_images;
+ std::map<std::string, SDImageOwner> prompt_images;
std::vector<std::vector<SDImageOwner>> ref_videos;
std::vector<SDAudioOwner> ref_video_audios;
std::vector<SDAudioOwner> ref_audios;
@@ -325,6 +326,7 @@ struct SDGenerationParams {
bool from_json_str(const std::string& json_str,
const std::function<std::string(const std::string&)>& lora_path_resolver = {});
bool parse_image_preprocess_json(const std::string& json_str);
+ bool parse_prompt_images_json(const std::string& json_str);
bool initialize_cache_params();
bool extract_and_remove_lora(const std::string& lora_model_dir);
bool width_and_height_are_set() const;
diff --git a/examples/server/api.md b/examples/server/api.md
index f6096c6..fe43128 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -11,6 +11,14 @@ The server currently exposes three API families:
The `sdcpp API` is the native API surface.
Its request schema is the same schema used by `sd_cpp_extra_args`.
+Image and video requests can include interleaved images in `prompt` or
+`negative_prompt` using `<image:ID>`. The native `sdcpp API` accepts a
+`prompt_images` object mapping each ID to a base64 image or image data URI.
+For `OpenAI API` and `sdapi`, pass `prompt_images` through `sd_cpp_extra_args`
+embedded inside `prompt`. IDs are resolved against that request's images.
+See [images in prompts](../../docs/prompt_images.md) for examples and supported
+LLM encoders.
+
Global LoRA rule:
- Server APIs do not parse LoRA tags embedded inside `prompt`.
diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp
index c730b9c..3513c4a 100644
--- a/examples/server/async_jobs.cpp
+++ b/examples/server/async_jobs.cpp
@@ -355,6 +355,7 @@ bool execute_img_gen_job(ServerRuntime& runtime,
}
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
JobCallbacksGuard callbacks(runtime.sd_ctx, job);
+ PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.img_gen.gen_params);
sd_image_generation_result_t* generated = nullptr;
generate_image(runtime.sd_ctx, ¶ms, &generated);
results.adopt(generated, runtime.ctx_params->lora_model_dir);
@@ -437,6 +438,7 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
}
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
JobCallbacksGuard callbacks(runtime.sd_ctx, job);
+ PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.vid_gen.gen_params);
sd_video_generation_result_t* generated = nullptr;
if (generate_video(runtime.sd_ctx, ¶ms, &generated)) {
output_fps = generated->fps;
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index f721e14..30ca7f3 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -94,10 +94,6 @@ static bool build_openai_edit_request(const httplib::Request& req,
size_t image_count = req.form.get_file_count("image[]");
bool has_legacy_image = req.form.has_file("image");
- if (image_count == 0 && !has_legacy_image) {
- error_message = "at least one image[] required";
- return false;
- }
std::vector<std::vector<uint8_t>> images_bytes;
for (size_t i = 0; i < image_count; ++i) {
@@ -204,6 +200,10 @@ static bool build_openai_edit_request(const httplib::Request& req,
error_message = "invalid sd_cpp_extra_args";
return false;
}
+ if (images_bytes.empty() && request.gen_params.prompt_images.empty()) {
+ error_message = "at least one image[] or prompt_images in sd_cpp_extra_args required";
+ return false;
+ }
// Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
@@ -223,6 +223,7 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
{
std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+ PromptImageCallbackGuard prompt_images(runtime.sd_ctx, request.gen_params);
sd_image_generation_result_t* generated = nullptr;
generate_image(runtime.sd_ctx, &img_gen_params, &generated);
results.adopt(generated, runtime.ctx_params->lora_model_dir);
diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp
index 0d37898..26ed7c9 100644
--- a/examples/server/routes_sdapi.cpp
+++ b/examples/server/routes_sdapi.cpp
@@ -600,6 +600,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
{
std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
+ PromptImageCallbackGuard prompt_images(runtime->sd_ctx, request.gen_params);
const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty();
SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing);
sd_image_generation_result_t* generated = nullptr;
diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp
index 5d08bba..499693a 100644
--- a/examples/server/runtime.cpp
+++ b/examples/server/runtime.cpp
@@ -19,6 +19,26 @@
namespace fs = std::filesystem;
+bool PromptImageCallbackGuard::load_image(const char* source, void* data, sd_image_t* image_out) {
+ const auto& params = static_cast<PromptImageCallbackGuard*>(data)->params_;
+ const auto it = params.prompt_images.find(source);
+ if (it == params.prompt_images.end()) {
+ LOG_ERROR("Unknown prompt image ID: %s", source);
+ return false;
+ }
+ *image_out = it->second.get();
+ return image_out->data != nullptr;
+}
+
+PromptImageCallbackGuard::PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params)
+ : ctx_(ctx), params_(params) {
+ sd_set_prompt_image_callback(ctx_, load_image, this);
+}
+
+PromptImageCallbackGuard::~PromptImageCallbackGuard() {
+ sd_set_prompt_image_callback(ctx_, nullptr, nullptr);
+}
+
bool validate_server_img_gen_request(const ServerRuntime& runtime,
const ImgGenJobRequest& request,
std::string& error_message) {
diff --git a/examples/server/runtime.h b/examples/server/runtime.h
index ba9ddc4..b21f6f5 100644
--- a/examples/server/runtime.h
+++ b/examples/server/runtime.h
@@ -94,6 +94,19 @@ struct ImgGenJobRequest {
}
};
+class PromptImageCallbackGuard {
+ sd_ctx_t* ctx_;
+ const SDGenerationParams& params_;
+
+ static bool load_image(const char* source, void* data, sd_image_t* image_out);
+
+public:
+ PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params);
+ ~PromptImageCallbackGuard();
+ PromptImageCallbackGuard(const PromptImageCallbackGuard&) = delete;
+ PromptImageCallbackGuard& operator=(const PromptImageCallbackGuard&) = delete;
+};
+
constexpr int k_max_generation_dimension = 4096;
constexpr int k_max_adetailer_steps = 100;
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index 7453a3f..3a06aea 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -517,6 +517,11 @@ typedef struct {
} sd_preview_info_t;
typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data);
+// Resolves <image:source> markers. Pixels are borrowed and copied before the
+// next callback; the caller retains ownership. Register/clear while ctx is idle.
+typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, sd_image_t* image_out);
+SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data);
+
SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
// Context-local generation events, independent of preview and the legacy progress callback.
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index 31aa493..eeebb77 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -4,10 +4,12 @@
#include <cinttypes>
#include <cmath>
#include <iomanip>
+#include <iterator>
#include <limits>
#include <optional>
#include <sstream>
#include <stdexcept>
+#include "conditioning/prompt_input.h"
#include "core/ggml_tensor_utils.h"
#include "core/tensor_ggml.hpp"
@@ -132,6 +134,7 @@ static inline sd::Tensor<float> apply_token_weights(sd::Tensor<float> hidden_sta
struct ConditionerParams {
std::string text;
+ const sd::PromptInput* prompt_input = nullptr;
int clip_skip = -1;
int width = -1;
int height = -1;
@@ -143,6 +146,7 @@ struct ConditionerParams {
struct Conditioner {
virtual int resolve_clip_skip(int) const { return 0; }
+ virtual bool supports_prompt_images() const { return false; }
virtual ~Conditioner() = default;
public:
@@ -1979,6 +1983,10 @@ struct LLMEmbedder : public Conditioner {
std::shared_ptr<T5Runner> byt5;
std::shared_ptr<PH3ImageTextConnectorRunner> ph3_connector;
+ bool supports_prompt_images() const override {
+ return llm->supports_prompt_images();
+ }
+
LLMEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
SDVersion version = VERSION_QWEN_IMAGE,
@@ -2024,6 +2032,14 @@ struct LLMEmbedder : public Conditioner {
} else if (version == VERSION_MOSS_IMAGE) {
arch = LLM::LLMArch::QWEN3_5;
}
+ if (arch == LLM::LLMArch::QWEN2_5_VL || arch == LLM::LLMArch::QWEN3_VL || arch == LLM::LLMArch::QWEN3_5) {
+ for (const auto& entry : tensor_storage_map) {
+ if (starts_with(entry.first, "text_encoders.llm.visual.")) {
+ enable_vision = true;
+ break;
+ }
+ }
+ }
llm = std::make_shared<LLM::LLMRunner>(arch,
backend,
tensor_storage_map,
@@ -2184,13 +2200,9 @@ struct LLMEmbedder : public Conditioner {
}
}
- std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> tokenize(std::string text,
- const std::pair<int, int>& attn_range,
- size_t min_length = 0,
- size_t max_length = 100000000,
- bool spell_quotes = false,
- const std::string& suffix = "",
- const std::vector<std::pair<int, sd::Tensor<float>>>& image_embeds = {}) {
+ static std::vector<std::pair<std::string, float>> parse_text_attention(const std::string& text,
+ const std::pair<int, int>& attn_range,
+ bool spell_quotes) {
std::vector<std::pair<std::string, float>> parsed_attention;
if (attn_range.first >= 0 && attn_range.second > 0) {
if (attn_range.first > 0) {
@@ -2212,6 +2224,18 @@ struct LLMEmbedder : public Conditioner {
parsed_attention.emplace_back(text, 1.f);
}
+ return parsed_attention;
+ }
+
+ std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> tokenize(std::string text,
+ const std::pair<int, int>& attn_range,
+ size_t min_length = 0,
+ size_t max_length = 100000000,
+ bool spell_quotes = false,
+ const std::string& suffix = "",
+ const std::vector<std::pair<int, sd::Tensor<float>>>& image_embeds = {}) {
+ auto parsed_attention = parse_text_attention(text, attn_range, spell_quotes);
+
{
std::stringstream ss;
ss << "[";
@@ -2287,6 +2311,57 @@ struct LLMEmbedder : public Conditioner {
return {tokens, weights, mask};
}
+ bool prepare_prompt_tokens(int n_threads,
+ const sd::PromptInputTemplate& input_template,
+ const std::string& prompt,
+ const std::pair<int, int>& attn_range,
+ int min_length,
+ int max_length,
+ bool spell_quotes,
+ const std::string& suffix,
+ const LLM::ImageEmbeds& images,
+ const LLM::DeepStackImageEmbeds& deepstack,
+ const std::vector<LLM::ImageGrid>& grids,
+ LLM::PromptTokens& result) {
+ sd::PromptInput input;
+ auto text = parse_text_attention(prompt, attn_range, spell_quotes);
+ text.emplace_back(suffix, 1.f);
+ for (const auto& span : text) {
+ auto parts = input_template.expand(span.first, span.second);
+ input.insert(input.end(), std::make_move_iterator(parts.begin()), std::make_move_iterator(parts.end()));
+ }
+ max_length = std::min(max_length, static_cast<int>(llm->config.max_position_embeddings));
+ if (!llm->prepare_prompt(n_threads, *tokenizer, input, max_length, result, images, deepstack, grids)) {
+ return false;
+ }
+ auto expanded = result.tokens;
+ for (const auto& image : result.image_embeds) {
+ expanded.insert(expanded.begin() + image.first, image.second.shape()[1] - 1, expanded[image.first]);
+ }
+ if (sd_version_is_flux3(version)) {
+ min_length = std::min(max_length, ((result.expanded_length + 79) / 80) * 80);
+ }
+ tokenizer->pad_tokens(expanded, &result.weights, &result.mask, min_length, max_length);
+ const int leading_padding = static_cast<int>(std::find(result.mask.begin(), result.mask.end(), 1.f) - result.mask.begin());
+ for (auto& image : result.image_embeds) {
+ image.first += leading_padding;
+ }
+ for (auto& layer : result.deepstack_image_embeds) {
+ for (auto& image : layer) {
+ image.first += leading_padding;
+ }
+ }
+ for (auto& grid : result.image_grids) {
+ grid.index += leading_padding;
+ }
+ for (auto it = result.image_embeds.rbegin(); it != result.image_embeds.rend(); ++it) {
+ expanded.erase(expanded.begin() + it->first + 1, expanded.begin() + it->first + it->second.shape()[1]);
+ }
+ result.tokens = std::move(expanded);
+ result.expanded_length = static_cast<int>(result.weights.size());
+ return true;
+ }
+
sd::Tensor<float> encode_prompt(int n_threads,
const std::string prompt,
const std::pair<int, int>& prompt_attn_range,
@@ -2300,8 +2375,18 @@ struct LLMEmbedder : public Conditioner {
const LLM::DeepStackImageEmbeds& deepstack_image_embeds = {},
const std::vector<LLM::ImageGrid>& image_grids = {},
const std::string& prompt_suffix = "",
- sd::Tensor<float>* output_mask = nullptr) {
- auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes, prompt_suffix, image_embeds);
+ sd::Tensor<float>* output_mask = nullptr,
+ const sd::PromptInputTemplate* input_template = nullptr,
+ LLM::PromptTokens* output_prompt_tokens = nullptr) {
+ LLM::PromptTokens prepared;
+ if (input_template != nullptr &&
+ !prepare_prompt_tokens(n_threads, *input_template, prompt, prompt_attn_range, min_length, max_length,
+ spell_quotes, prompt_suffix, image_embeds, deepstack_image_embeds, image_grids, prepared)) {
+ return {};
+ }
+ auto tokens_weights_mask = input_template == nullptr
+ ? tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes, prompt_suffix, image_embeds)
+ : std::make_tuple(prepared.tokens, prepared.weights, prepared.mask);
auto& tokens = std::get<0>(tokens_weights_mask);
auto& weights = std::get<1>(tokens_weights_mask);
auto& mask = std::get<2>(tokens_weights_mask);
@@ -2332,21 +2417,24 @@ struct LLMEmbedder : public Conditioner {
sd::Tensor<int32_t> position_ids;
if (sd_version_is_flux3(version)) {
position_ids = sd::Tensor<int32_t>::zeros({static_cast<int64_t>(weights.size() * 4)});
+ if (input_template != nullptr) {
+ position_ids.values() = LLM::LLMRunner::build_position_ids(llm->config, weights.size(), prepared.image_grids);
+ }
for (size_t axis = 0; axis < 4; ++axis) {
for (size_t i = 0; i < weights.size(); ++i) {
- position_ids[axis * weights.size() + i] = mask[i] == 0.f ? 1 : static_cast<int32_t>(i);
+ position_ids[axis * weights.size() + i] = mask[i] == 0.f ? 1 : (input_template == nullptr ? static_cast<int32_t>(i) : position_ids[axis * weights.size() + i]);
}
}
}
auto hidden_states = llm->compute(n_threads,
input_ids,
attention_mask,
- image_embeds,
+ input_template == nullptr ? image_embeds : prepared.image_embeds,
out_layers,
false,
false,
- deepstack_image_embeds,
- image_grids,
+ input_template == nullptr ? deepstack_image_embeds : prepared.deepstack_image_embeds,
+ input_template == nullptr ? image_grids : prepared.image_grids,
position_ids);
if (hidden_states.empty()) {
LOG_ERROR("LLM prompt encoding failed");
@@ -2386,6 +2474,9 @@ struct LLMEmbedder : public Conditioner {
output_mask->data()[i - prompt_template_encode_start_idx] = mask[i];
}
}
+ if (output_prompt_tokens != nullptr && input_template != nullptr) {
+ *output_prompt_tokens = std::move(prepared);
+ }
return new_hidden_states;
}
@@ -2482,7 +2573,16 @@ struct LLMEmbedder : public Conditioner {
}
SDCondition get_learned_condition(int n_threads,
- const ConditionerParams& conditioner_params) override {
+ const ConditionerParams& params) override {
+ ConditionerParams conditioner_params = params;
+ sd::PromptInputTemplate input_template(params.prompt_input);
+ if (params.prompt_input != nullptr) {
+ if (!supports_prompt_images()) {
+ LOG_ERROR("This conditioner does not support inline prompt images");
+ return {};
+ }
+ conditioner_params.text = input_template.text;
+ }
std::string prompt;
std::pair<int, int> prompt_attn_range;
std::vector<std::string> extra_prompts;
@@ -3270,6 +3370,7 @@ struct LLMEmbedder : public Conditioner {
GGML_ABORT("unknown version %d", version);
}
+ LLM::PromptTokens prompt_tokens;
auto hidden_states = encode_prompt(n_threads,
prompt,
prompt_attn_range,
@@ -3283,7 +3384,9 @@ struct LLMEmbedder : public Conditioner {
deepstack_image_embeds,
image_grids,
prompt_suffix,
- version == VERSION_IRIS ? &text_mask : nullptr);
+ version == VERSION_IRIS ? &text_mask : nullptr,
+ params.prompt_input == nullptr ? nullptr : &input_template,
+ &prompt_tokens);
if (hidden_states.empty()) {
return {};
}
@@ -3350,10 +3453,11 @@ struct LLMEmbedder : public Conditioner {
result.extra_c_crossattns = std::move(extra_hidden_states_vec);
result.c_vector = std::move(text_mask);
if (version == VERSION_QWEN_IMAGE_2_1) {
- auto slots = sd::Tensor<int32_t>::zeros({result.c_crossattn.shape()[1]});
+ auto slots = sd::Tensor<int32_t>::zeros({result.c_crossattn.shape()[1]});
+ const auto& references = params.prompt_input == nullptr ? image_embeds : prompt_tokens.image_embeds;
for (size_t i = 0; i < image_embeds.size(); ++i) {
- int64_t begin = image_embeds[i].first - prompt_template_encode_start_idx;
- int64_t end = begin + image_embeds[i].second.shape()[1];
+ int64_t begin = references[i].first - prompt_template_encode_start_idx;
+ int64_t end = begin + references[i].second.shape()[1];
if (begin < 0 || end > slots.numel()) {
LOG_ERROR("Qwen Image 2.1 image slots exceed the encoded prompt");
return {};
@@ -3366,7 +3470,8 @@ struct LLMEmbedder : public Conditioner {
}
if (sd_version_is_minimax_h3(version)) {
std::vector<int32_t> tags(static_cast<size_t>(result.c_crossattn.shape()[1]), 1);
- for (const auto& [index, image_embed] : image_embeds) {
+ const auto& images = params.prompt_input == nullptr ? image_embeds : prompt_tokens.image_embeds;
+ for (const auto& [index, image_embed] : images) {
int64_t begin = std::max<int64_t>(0, index - 1);
int64_t end = std::min<int64_t>(static_cast<int64_t>(tags.size()),
index + image_embed.shape()[1] + 1);
diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h
index 34648c9..58f539a 100644
--- a/src/conditioning/conditioning_cache.h
+++ b/src/conditioning/conditioning_cache.h
@@ -11,11 +11,16 @@ class ConditioningCache {
struct Entry {
ConditionerParams params;
std::vector<sd::Tensor<float>> ref_images;
+ sd::PromptInput prompt_input;
std::vector<MiniMaxH3PresentationItem> references;
SDCondition condition;
Entry(const ConditionerParams& input, const SDCondition& output)
: params(input), condition(output) {
+ if (input.prompt_input != nullptr) {
+ prompt_input = *input.prompt_input;
+ params.prompt_input = &prompt_input;
+ }
// Request-owned reference pointers must not outlive the request.
if (input.ref_images != nullptr) {
ref_images = *input.ref_images;
@@ -52,10 +57,21 @@ class ConditioningCache {
r.ref_image_size);
};
if (fields(a) != fields(b) ||
+ (a.prompt_input == nullptr) != (b.prompt_input == nullptr) ||
(a.ref_images == nullptr) != (b.ref_images == nullptr) ||
(a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) {
return false;
}
+ if (a.prompt_input != nullptr &&
+ !std::equal(a.prompt_input->begin(), a.prompt_input->end(),
+ b.prompt_input->begin(), b.prompt_input->end(),
+ [](const sd::PromptPart& x, const sd::PromptPart& y) {
+ return x.type == y.type && x.content == y.content && x.weight == y.weight &&
+ (x.image == y.image ||
+ (x.image && y.image && x.image->shape() == y.image->shape() && x.image->values() == y.image->values()));
+ })) {
+ return false;
+ }
if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) {
return false;
}
diff --git a/src/conditioning/fibo2.hpp b/src/conditioning/fibo2.hpp
index 43b2ee0..db124ea 100644
--- a/src/conditioning/fibo2.hpp
+++ b/src/conditioning/fibo2.hpp
@@ -132,7 +132,12 @@ struct Fibo2Conditioner : public LLMEmbedder {
return result;
}
- SDCondition get_learned_condition(int n_threads, const ConditionerParams& p) override {
+ SDCondition get_learned_condition(int n_threads, const ConditionerParams& params) override {
+ auto p = params;
+ sd::PromptInputTemplate input_template(p.prompt_input);
+ if (p.prompt_input != nullptr) {
+ p.text = input_template.text;
+ }
int image_count = p.ref_images ? static_cast<int>(p.ref_images->size()) : 0;
if (image_count && !llm->enable_vision) {
LOG_ERROR("Fibo 2.0 editing requires vision weights");
@@ -169,7 +174,8 @@ struct Fibo2Conditioner : public LLMEmbedder {
}
}
const std::set<int> layers = {5, 7, 9, 11, 13, 16, 18, 20, 22, 24, 27, 29, 31, 33, 35};
- auto states = encode_prompt(n_threads, prompt, {0, 0}, 0, 0, images, layers, 0, false, 4096, deepstack, grids);
+ auto states = encode_prompt(n_threads, prompt, {0, 0}, 0, 0, images, layers, 0, false, 4096, deepstack, grids,
+ "", nullptr, p.prompt_input == nullptr ? nullptr : &input_template);
if (states.empty()) {
return {};
}
diff --git a/src/conditioning/prompt_input.cpp b/src/conditioning/prompt_input.cpp
new file mode 100644
index 0000000..0748e9c
--- /dev/null
+++ b/src/conditioning/prompt_input.cpp
@@ -0,0 +1,155 @@
+#include "conditioning/prompt_input.h"
+
+#include <algorithm>
+#include <map>
+#include <stdexcept>
+
+#include "core/util.h"
+
+namespace sd {
+
+ PromptInputTemplate::PromptInputTemplate(const PromptInput* input)
+ : input_(input) {
+ if (input == nullptr) {
+ return;
+ }
+ uint32_t codepoint = 0xf0000;
+ for (size_t i = 0; i < input->size(); ++i) {
+ const auto& part = (*input)[i];
+ if (part.type == PromptPart::Type::TEXT) {
+ text += part.content;
+ continue;
+ }
+ // One private-use code point survives JSON formatting and quote spelling.
+ std::string marker;
+ do {
+ if (codepoint > 0xffffd) {
+ throw std::invalid_argument("Too many prompt image markers");
+ }
+ marker = {static_cast<char>(0xf0 | (codepoint >> 18)),
+ static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f)),
+ static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f)),
+ static_cast<char>(0x80 | (codepoint & 0x3f))};
+ ++codepoint;
+ } while (std::any_of(input->begin(), input->end(), [&](const PromptPart& source) {
+ return source.content.find(marker) != std::string::npos;
+ }));
+ image_markers_.emplace(marker, i);
+ text += marker;
+ }
+ }
+
+ PromptInput PromptInputTemplate::expand(const std::string& formatted_text, float weight) const {
+ PromptInput result;
+ size_t cursor = 0;
+ size_t search = 0;
+ while (input_ != nullptr && search < formatted_text.size()) {
+ const size_t begin = formatted_text.find(static_cast<char>(0xf3), search);
+ if (begin == std::string::npos) {
+ break;
+ }
+ const auto marker = image_markers_.find(formatted_text.substr(begin, 4));
+ search = begin + 1;
+ if (marker == image_markers_.end()) {
+ continue;
+ }
+ if (begin > cursor) {
+ result.push_back({PromptPart::Type::TEXT, formatted_text.substr(cursor, begin - cursor), {}, weight});
+ }
+ result.push_back((*input_)[marker->second]);
+ cursor = begin + marker->first.size();
+ search = cursor;
+ }
+ if (cursor < formatted_text.size()) {
+ result.push_back({PromptPart::Type::TEXT, formatted_text.substr(cursor), {}, weight});
+ }
+ return result;
+ }
+
+ PromptInput parse_prompt_input(const std::string& text) {
+ PromptInput parts;
+ std::string plain_text;
+ size_t cursor = 0;
+ while (cursor < text.size()) {
+ const size_t begin = text.find("<image:", cursor);
+ if (begin == std::string::npos) {
+ plain_text += text.substr(cursor);
+ break;
+ }
+ size_t slashes = 0;
+ while (begin > slashes && text[begin - slashes - 1] == '\\') {
+ ++slashes;
+ }
+ if (slashes % 2 != 0) {
+ plain_text += text.substr(cursor, begin - cursor - 1);
+ plain_text += "<image:";
+ cursor = begin + 7;
+ continue;
+ }
+ const size_t end = text.find('>', begin + 7);
+ if (end == std::string::npos || end == begin + 7) {
+ throw std::invalid_argument("Invalid <image:...> prompt marker at byte " + std::to_string(begin));
+ }
+ plain_text += text.substr(cursor, begin - cursor);
+ if (!plain_text.empty()) {
+ parts.push_back({PromptPart::Type::TEXT, std::move(plain_text), {}});
+ plain_text.clear();
+ }
+ parts.push_back({PromptPart::Type::IMAGE, text.substr(begin + 7, end - begin - 7), {}});
+ cursor = end + 1;
+ }
+ if (!plain_text.empty() || parts.empty()) {
+ parts.push_back({PromptPart::Type::TEXT, std::move(plain_text), {}});
+ }
+ return parts;
+ }
+
+ bool prompt_has_images(const PromptInput& input) {
+ return std::any_of(input.begin(), input.end(), [](const PromptPart& part) {
+ return part.type == PromptPart::Type::IMAGE;
+ });
+ }
+
+ std::string prompt_input_text(const PromptInput& input) {
+ std::string text;
+ for (const auto& part : input) {
+ if (part.type == PromptPart::Type::TEXT) {
+ text += part.content;
+ }
+ }
+ return text;
+ }
+
+ bool resolve_prompt_images(PromptInput& input, const PromptImageLoader& loader) {
+ constexpr size_t max_image_sources = 32;
+ std::map<std::string, std::shared_ptr<const Tensor<float>>> images;
+ for (const auto& part : input) {
+ if (part.type == PromptPart::Type::IMAGE) {
+ images.try_emplace(part.content);
+ if (images.size() > max_image_sources) {
+ LOG_ERROR("A prompt supports at most %zu unique image sources", max_image_sources);
+ return false;
+ }
+ }
+ }
+ for (auto& part : input) {
+ if (part.type != PromptPart::Type::IMAGE) {
+ continue;
+ }
+ if (!loader) {
+ return false;
+ }
+ auto& resolved = images.at(part.content);
+ if (!resolved) {
+ auto image = loader(part.content);
+ if (image.empty()) {
+ return false;
+ }
+ resolved = std::make_shared<const Tensor<float>>(std::move(image));
+ }
+ part.image = resolved;
+ }
+ return true;
+ }
+
+} // namespace sd
diff --git a/src/conditioning/prompt_input.h b/src/conditioning/prompt_input.h
new file mode 100644
index 0000000..ed5fa2d
--- /dev/null
+++ b/src/conditioning/prompt_input.h
@@ -0,0 +1,45 @@
+#ifndef __SD_CONDITIONING_PROMPT_INPUT_H__
+#define __SD_CONDITIONING_PROMPT_INPUT_H__
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/tensor.hpp"
+
+namespace sd {
+
+ struct PromptPart {
+ enum class Type { TEXT,
+ IMAGE };
+
+ Type type = Type::TEXT;
+ std::string content;
+ std::shared_ptr<const Tensor<float>> image;
+ float weight = 1.f;
+ };
+
+ using PromptInput = std::vector<PromptPart>;
+ using PromptImageLoader = std::function<Tensor<float>(const std::string&)>;
+
+ class PromptInputTemplate {
+ const PromptInput* input_;
+ std::map<std::string, size_t> image_markers_;
+
+ public:
+ std::string text;
+
+ explicit PromptInputTemplate(const PromptInput* input);
+ PromptInput expand(const std::string& formatted_text, float weight = 1.f) const;
+ };
+
+ PromptInput parse_prompt_input(const std::string& text);
+ bool prompt_has_images(const PromptInput& input);
+ std::string prompt_input_text(const PromptInput& input);
+ bool resolve_prompt_images(PromptInput& input, const PromptImageLoader& loader);
+
+} // namespace sd
+
+#endif // __SD_CONDITIONING_PROMPT_INPUT_H__
diff --git a/src/model/integrations/moss_image.cpp b/src/model/integrations/moss_image.cpp
index 4a85d16..6a2114d 100644
--- a/src/model/integrations/moss_image.cpp
+++ b/src/model/integrations/moss_image.cpp
@@ -8,7 +8,7 @@ static bool build_moss_image(const sd::model_builders::Context& ctx,
const TokenizerConfig& tokenizers,
sd::model_builders::CoreRunners& runners) {
runners.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- ctx.tensor_storage_map, ctx.version, "", false,
+ ctx.tensor_storage_map, ctx.version, "", true,
ctx.weight_manager, tokenizers);
runners.diffusion = std::make_shared<MOSSImage::MOSSImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
ctx.tensor_storage_map, "model.diffusion_model", ctx.weight_manager);
diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp
index 3bba320..647ff51 100644
--- a/src/model/te/llm.hpp
+++ b/src/model/te/llm.hpp
@@ -20,6 +20,7 @@
#include <utility>
#include <vector>
+#include "conditioning/prompt_input.h"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
@@ -110,6 +111,16 @@ namespace LLM {
using ImageEmbeds = std::vector<std::pair<int, sd::Tensor<float>>>;
using DeepStackImageEmbeds = std::vector<ImageEmbeds>;
+ struct PromptTokens {
+ std::vector<int> tokens;
+ int expanded_length = 0;
+ std::vector<float> weights;
+ std::vector<float> mask;
+ ImageEmbeds image_embeds;
+ DeepStackImageEmbeds deepstack_image_embeds;
+ std::vector<ImageGrid> image_grids;
+ };
+
struct LLMConfig {
LLMArch arch = LLMArch::QWEN2_5_VL;
int64_t num_layers = 28;
@@ -176,18 +187,25 @@ namespace LLM {
config.qkv_bias = false;
config.rms_norm_eps = 1e-5f;
} else if (arch == LLMArch::QWEN3_5) {
- config.hidden_size = 4096;
- config.intermediate_size = 12288;
- config.head_dim = 256;
- config.num_heads = 16;
- config.num_kv_heads = 4;
- config.qkv_bias = false;
- config.qk_norm = true;
- config.rms_norm_add = true;
- config.partial_rotary = 0.25f;
- config.rope_thetas = {10000000.f};
- config.max_position_embeddings = 262144;
- config.vocab_size = 248320;
+ config.hidden_size = 4096;
+ config.intermediate_size = 12288;
+ config.head_dim = 256;
+ config.num_heads = 16;
+ config.num_kv_heads = 4;
+ config.qkv_bias = false;
+ config.qk_norm = true;
+ config.rms_norm_add = true;
+ config.partial_rotary = 0.25f;
+ config.rope_thetas = {10000000.f};
+ config.max_position_embeddings = 262144;
+ config.vocab_size = 248320;
+ config.vision.arch = LLMVisionArch::QWEN3_VL;
+ config.vision.num_layers = 27;
+ config.vision.hidden_size = 1152;
+ config.vision.intermediate_size = 4304;
+ config.vision.out_hidden_size = 4096;
+ config.vision.patch_size = 16;
+ config.vision.num_position_embeddings = 2304;
} else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) {
config.head_dim = 128;
config.num_heads = 32;
@@ -1800,6 +1818,11 @@ namespace LLM {
1.f,
32.f,
1.f);
+ } else if (arch == LLMArch::QWEN3_5 && input_pos->ne[0] == n_token * 4) {
+ int sections[4] = {11, 11, 10, 0};
+ int rotary_dim = rope_pairs * 2;
+ q = ggml_rope_multi(ctx->ggml_ctx, q, input_pos, nullptr, rotary_dim, sections, GGML_ROPE_TYPE_IMROPE, static_cast<int>(max_position_embeddings), rope_thetas[0], 1.f, 0.f, 1.f, 32.f, 1.f);
+ k = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, rotary_dim, sections, GGML_ROPE_TYPE_IMROPE, static_cast<int>(max_position_embeddings), rope_thetas[0], 1.f, 0.f, 1.f, 32.f, 1.f);
} else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::QWEN3_5) {
// LLaDA2 slices the head (query[..., :rotary_dim]) instead of zero-padding
// inv_freq like gemma does, so rotate_half pairs i with i + rotary_dim/2 and the
@@ -2207,6 +2230,16 @@ namespace LLM {
std::array<std::vector<int32_t>, 4> pos_embed_idx_data_;
std::array<std::vector<float>, 4> pos_embed_weight_data_;
+ static bool check_window_mask_size(int64_t grid_h, int64_t grid_w) {
+ // The dense float mask must fit within 256 MiB, including rectangular grids.
+ constexpr int64_t max_patches = 8192;
+ if (grid_h <= 0 || grid_w <= 0 || grid_h > max_patches || grid_w > max_patches / grid_h) {
+ LOG_ERROR("Qwen2.5-VL images support at most %" PRId64 " patches (256 MiB window mask)", max_patches);
+ return false;
+ }
+ return true;
+ }
+
static ggml_tensor* process_image_common(GGMLRunnerContext* runner_ctx,
ggml_tensor* image,
const LLMVisionConfig& vision_params) {
@@ -2317,8 +2350,12 @@ namespace LLM {
GGML_ASSERT(image->ne[1] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0);
GGML_ASSERT(image->ne[0] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0);
- int grid_h = static_cast<int>(image->ne[1]) / vision_params.patch_size;
- int grid_w = static_cast<int>(image->ne[0]) / vision_params.patch_size;
+ if (vision_params.arch == LLMVisionArch::QWEN2_5_VL &&
+ !check_window_mask_size(image->ne[1] / vision_params.patch_size, image->ne[0] / vision_params.patch_size)) {
+ throw std::invalid_argument("Qwen2.5-VL window mask exceeds its memory limit");
+ }
+ int grid_h = static_cast<int>(image->ne[1] / vision_params.patch_size);
+ int grid_w = static_cast<int>(image->ne[0] / vision_params.patch_size);
auto pixel_values = process_image_common(runner_ctx, image, vision_params);
int head_dim = static_cast<int>(vision_params.hidden_size / vision_params.num_heads);
@@ -2380,7 +2417,8 @@ namespace LLM {
runner->set_backend_tensor_data(window_index, window_index_vec.data());
runner->set_backend_tensor_data(window_inverse_index, window_inverse_index_vec.data());
- window_mask_vec.resize((grid_h * grid_w) * (grid_h * grid_w));
+ const size_t n_patches = static_cast<size_t>(grid_h) * grid_w;
+ window_mask_vec.resize(n_patches * n_patches);
int window_start_index = 0;
for (int seq_index = 0; seq_index < seqlens.size(); seq_index++) {
int window_end_index = window_start_index + seqlens[seq_index];
@@ -2391,8 +2429,9 @@ namespace LLM {
if (j >= window_start_index && j < window_end_index) {
mask_value = 0;
}
- GGML_ASSERT((i * (grid_h * grid_w) + j) < window_mask_vec.size());
- window_mask_vec[i * (grid_h * grid_w) + j] = mask_value;
+ const size_t index = static_cast<size_t>(i) * n_patches + j;
+ GGML_ASSERT(index < window_mask_vec.size());
+ window_mask_vec[index] = mask_value;
}
}
window_start_index = window_end_index;
@@ -2486,38 +2525,14 @@ namespace LLM {
return hidden_states;
}
- ggml_cgraph* build_graph(const sd::Tensor<int32_t>& input_ids_tensor,
- const sd::Tensor<float>& attention_mask_tensor,
- const ImageEmbeds& image_embeds_tensor,
- const DeepStackImageEmbeds& deepstack_image_embeds_tensor,
- const std::vector<ImageGrid>& image_grids,
- std::set<int> out_layers,
- bool return_all_hidden_states = false,
- const sd::Tensor<int32_t>& position_ids = {}) {
- ggml_cgraph* gf = new_graph_custom(LLM_GRAPH_SIZE);
- ggml_tensor* input_ids = make_input(input_ids_tensor);
- std::vector<std::pair<int, ggml_tensor*>> image_embeds;
- image_embeds.reserve(image_embeds_tensor.size());
- for (const auto& [idx, embed_tensor] : image_embeds_tensor) {
- ggml_tensor* embed = make_input(embed_tensor);
- image_embeds.emplace_back(idx, embed);
- }
- std::vector<std::vector<std::pair<int, ggml_tensor*>>> deepstack_image_embeds(deepstack_image_embeds_tensor.size());
- for (size_t layer = 0; layer < deepstack_image_embeds_tensor.size(); ++layer) {
- deepstack_image_embeds[layer].reserve(deepstack_image_embeds_tensor[layer].size());
- for (const auto& [idx, embed_tensor] : deepstack_image_embeds_tensor[layer]) {
- deepstack_image_embeds[layer].emplace_back(idx, make_input(embed_tensor));
- }
- }
-
- int64_t n_tokens = input_ids->ne[0];
- for (const auto& image_embed : image_embeds) {
- n_tokens += image_embed.second->ne[1] - 1;
- }
+ static std::vector<int> build_position_ids(const LLMConfig& config,
+ int64_t n_tokens,
+ const std::vector<ImageGrid>& image_grids) {
+ std::vector<int> input_pos_vec;
if (config.arch == LLMArch::MISTRAL_SMALL_3_2 ||
config.arch == LLMArch::MINISTRAL_3_3B ||
config.arch == LLMArch::QWEN3 ||
- config.arch == LLMArch::QWEN3_5 ||
+ (config.arch == LLMArch::QWEN3_5 && image_grids.empty()) ||
config.arch == LLMArch::GEMMA3_12B ||
config.arch == LLMArch::GEMMA4_12B ||
config.arch == LLMArch::GEMMA2_2B ||
@@ -2535,7 +2550,7 @@ namespace LLM {
input_pos_vec[2 * n_tokens + i] = i;
input_pos_vec[3 * n_tokens + i] = 0;
}
- if (config.arch == LLMArch::QWEN3_VL && !image_grids.empty()) {
+ if ((config.arch == LLMArch::QWEN2_5_VL || config.arch == LLMArch::QWEN3_VL || config.arch == LLMArch::QWEN3_5) && !image_grids.empty()) {
int offset = 0;
for (const auto& grid : image_grids) {
int end = grid.index + grid.size;
@@ -2561,6 +2576,39 @@ namespace LLM {
}
}
+ return input_pos_vec;
+ }
+
+ ggml_cgraph* build_graph(const sd::Tensor<int32_t>& input_ids_tensor,
+ const sd::Tensor<float>& attention_mask_tensor,
+ const ImageEmbeds& image_embeds_tensor,
+ const DeepStackImageEmbeds& deepstack_image_embeds_tensor,
+ const std::vector<ImageGrid>& image_grids,
+ std::set<int> out_layers,
+ bool return_all_hidden_states = false,
+ const sd::Tensor<int32_t>& position_ids = {}) {
+ ggml_cgraph* gf = new_graph_custom(LLM_GRAPH_SIZE);
+ ggml_tensor* input_ids = make_input(input_ids_tensor);
+ std::vector<std::pair<int, ggml_tensor*>> image_embeds;
+ image_embeds.reserve(image_embeds_tensor.size());
+ for (const auto& [idx, embed_tensor] : image_embeds_tensor) {
+ ggml_tensor* embed = make_input(embed_tensor);
+ image_embeds.emplace_back(idx, embed);
+ }
+ std::vector<std::vector<std::pair<int, ggml_tensor*>>> deepstack_image_embeds(deepstack_image_embeds_tensor.size());
+ for (size_t layer = 0; layer < deepstack_image_embeds_tensor.size(); ++layer) {
+ deepstack_image_embeds[layer].reserve(deepstack_image_embeds_tensor[layer].size());
+ for (const auto& [idx, embed_tensor] : deepstack_image_embeds_tensor[layer]) {
+ deepstack_image_embeds[layer].emplace_back(idx, make_input(embed_tensor));
+ }
+ }
+
+ int64_t n_tokens = input_ids->ne[0];
+ for (const auto& image_embed : image_embeds) {
+ n_tokens += image_embed.second->ne[1] - 1;
+ }
+ input_pos_vec = build_position_ids(config, n_tokens, image_grids);
+
if (!position_ids.empty()) {
GGML_ASSERT(position_ids.numel() == static_cast<int64_t>(input_pos_vec.size()));
input_pos_vec = position_ids.values();
@@ -2877,6 +2925,149 @@ namespace LLM {
return outputs;
}
+ bool supports_prompt_images() const {
+ return enable_vision && (config.arch == LLMArch::QWEN2_5_VL || config.arch == LLMArch::QWEN3_5 || config.arch == LLMArch::QWEN3_VL);
+ }
+
+ bool prepare_prompt(int n_threads,
+ Tokenizer& tokenizer,
+ const sd::PromptInput& input,
+ int max_length,
+ PromptTokens& result,
+ const ImageEmbeds& reference_embeds = {},
+ const DeepStackImageEmbeds& reference_deepstack = {},
+ const std::vector<ImageGrid>& reference_grids = {}) {
+ result = {};
+ if (!supports_prompt_images()) {
+ LOG_ERROR("%s requires compatible vision weights for inline prompt images", get_desc().c_str());
+ return false;
+ }
+ std::vector<int> image_token, vision_start, vision_end;
+ if (!tokenizer.encode("<|image_pad|>", image_token) || image_token.size() != 1 ||
+ !tokenizer.encode("<|vision_start|>", vision_start) || vision_start.size() != 1 ||
+ !tokenizer.encode("<|vision_end|>", vision_end) || vision_end.size() != 1) {
+ LOG_ERROR("The LLM tokenizer must encode vision markers as single special tokens");
+ return false;
+ }
+ size_t reference_index = 0;
+ result.deepstack_image_embeds.resize(reference_deepstack.size());
+ auto append_tokens = [&](const std::vector<int>& tokens, float weight) {
+ for (int token : tokens) {
+ const bool reference = reference_index < reference_embeds.size() &&
+ reference_embeds[reference_index].first == result.expanded_length;
+ const int count = reference ? static_cast<int>(reference_embeds[reference_index].second.shape()[1]) : 1;
+ if (count > max_length - result.expanded_length) {
+ LOG_ERROR("Multimodal prompt exceeds the %d-token context", max_length);
+ return false;
+ }
+ if (reference) {
+ const auto& embed = reference_embeds[reference_index++];
+ result.image_embeds.push_back(embed);
+ for (size_t layer = 0; layer < reference_deepstack.size(); ++layer) {
+ for (const auto& extra : reference_deepstack[layer]) {
+ if (extra.first == embed.first) {
+ result.deepstack_image_embeds[layer].push_back(extra);
+ }
+ }
+ }
+ for (const auto& grid : reference_grids) {
+ if (grid.index == embed.first) {
+ result.image_grids.push_back(grid);
+ }
+ }
+ }
+ result.tokens.push_back(token);
+ result.weights.insert(result.weights.end(), count, reference ? 1.f : weight);
+ result.expanded_length += count;
+ }
+ return true;
+ };
+ for (const auto& part : input) {
+ if (part.type == sd::PromptPart::Type::TEXT) {
+ std::vector<int> tokens;
+ if (!tokenizer.encode(part.content, tokens) || !append_tokens(tokens, part.weight)) {
+ return false;
+ }
+ continue;
+ }
+ if (!append_tokens(vision_start, 1.f)) {
+ return false;
+ }
+ if (!part.image || part.image->empty() || part.image->dim() != 4 || part.image->shape()[2] != 3 || part.image->shape()[3] != 1) {
+ LOG_ERROR("Prompt images must be RGB tensors with one frame");
+ return false;
+ }
+ const auto& source = *part.image;
+ const int factor = config.vision.patch_size * config.vision.spatial_merge_size;
+ const double height = static_cast<double>(source.shape()[1]);
+ const double width = static_cast<double>(source.shape()[0]);
+ if (std::max(height, width) / std::min(height, width) > 200.0) {
+ LOG_ERROR("Prompt image aspect ratio exceeds 200");
+ return false;
+ }
+ int h = std::max(factor, static_cast<int>(std::round(height / factor)) * factor);
+ int w = std::max(factor, static_cast<int>(std::round(width / factor)) * factor);
+ const bool qwen25 = config.arch == LLMArch::QWEN2_5_VL;
+ const double min_pixels = qwen25 ? 3136.0 : 65536.0;
+ const double max_pixels = qwen25 ? 12845056.0 : 16777216.0;
+ if (static_cast<double>(h) * w > max_pixels) {
+ const double scale = std::sqrt(height * width / max_pixels);
+ h = std::max(factor, static_cast<int>(std::floor(height / scale / factor)) * factor);
+ w = std::max(factor, static_cast<int>(std::floor(width / scale / factor)) * factor);
+ } else if (static_cast<double>(h) * w < min_pixels) {
+ const double scale = std::sqrt(min_pixels / (height * width));
+ h = static_cast<int>(std::ceil(height * scale / factor)) * factor;
+ w = static_cast<int>(std::ceil(width * scale / factor)) * factor;
+ }
+ const int count = (h / factor) * (w / factor);
+ if (count > max_length - result.expanded_length) {
+ LOG_ERROR("Prompt images exceed the %d-token context", max_length);
+ return false;
+ }
+ if (qwen25 && !check_window_mask_size(h / config.vision.patch_size, w / config.vision.patch_size)) {
+ return false;
+ }
+ auto image = sd::ops::interpolate(source, {w, h, 3, 1}, sd::ops::InterpolateMode::Bicubic, false, true);
+ const float means[] = {0.48145466f, 0.4578275f, 0.40821073f};
+ const float stds[] = {0.26862954f, 0.26130258f, 0.27577711f};
+ for (int channel = 0; channel < 3; ++channel) {
+ for (int64_t pixel = 0; pixel < int64_t(w) * h; ++pixel) {
+ float& value = image[pixel + int64_t(channel) * w * h];
+ value = (std::clamp(value, 0.f, 1.f) - (qwen25 ? means[channel] : 0.5f)) /
+ (qwen25 ? stds[channel] : 0.5f);
+ }
+ }
+ auto outputs = encode_image_outputs(n_threads, image, false);
+ if (outputs.empty() || outputs[0].shape()[0] != config.hidden_size || outputs[0].shape()[1] != count) {
+ LOG_ERROR("Prompt image encoder returned an invalid embedding shape");
+ return false;
+ }
+ const int index = result.expanded_length;
+ result.tokens.push_back(image_token[0]);
+ result.weights.insert(result.weights.end(), count, 1.f);
+ result.expanded_length += count;
+ result.image_embeds.emplace_back(index, std::move(outputs[0]));
+ if (result.deepstack_image_embeds.empty()) {
+ result.deepstack_image_embeds.resize(outputs.size() - 1);
+ }
+ if (result.deepstack_image_embeds.size() != outputs.size() - 1) {
+ return false;
+ }
+ for (size_t layer = 1; layer < outputs.size(); ++layer) {
+ result.deepstack_image_embeds[layer - 1].emplace_back(index, std::move(outputs[layer]));
+ }
+ result.image_grids.push_back({index, count, h / config.vision.patch_size, w / config.vision.patch_size});
+ if (!append_tokens(vision_end, 1.f)) {
+ return false;
+ }
+ }
+ if (reference_index != reference_embeds.size()) {
+ LOG_ERROR("Reference images do not match the tokenized prompt");
+ return false;
+ }
+ return true;
+ }
+
std::vector<sd::Tensor<float>> encode_video_block_outputs(const int n_threads,
const sd::Tensor<float>& frames,
bool auto_runner_end = false) {
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index e791afc..bd23231 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -1800,7 +1800,10 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
}
}
- if (version == VERSION_FIBO2 || version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
+ if (version == VERSION_FIBO2 || version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MOSS_IMAGE ||
+ version == VERSION_IRIS || sd_version_is_flux3(version) || sd_version_is_lingbot_video(version) ||
+ sd_version_is_ideogram4(version) || sd_version_is_sefi_image(version) || sd_version_is_boogu_image(version) ||
+ sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
const std::string hf_vision_prefix = "text_encoders.llm.model.visual.";
if (starts_with(name, hf_vision_prefix)) {
name = "text_encoders.llm.visual." + name.substr(hf_vision_prefix.size());
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
index 5666680..745c8ba 100644
--- a/src/pipeline/diffusion_engine.cpp
+++ b/src/pipeline/diffusion_engine.cpp
@@ -1875,7 +1875,41 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
}
SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) {
- return conditioning_cache_->get(*cond_stage_model, n_threads, params);
+ if (params.prompt_input != nullptr) {
+ return conditioning_cache_->get(*cond_stage_model, n_threads, params);
+ }
+ auto input = sd::parse_prompt_input(params.text);
+ auto resolved = params;
+ if (sd::prompt_has_images(input)) {
+ if (!cond_stage_model->supports_prompt_images()) {
+ LOG_ERROR("Inline prompt images require a compatible LLM and vision weights");
+ return {};
+ }
+ if (prompt_image_callback == nullptr) {
+ LOG_ERROR("Inline prompt images require sd_set_prompt_image_callback()");
+ return {};
+ }
+ if (!sd::resolve_prompt_images(input, [&](const std::string& source) -> sd::Tensor<float> {
+ sd_image_t image{};
+ if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &image) ||
+ image.data == nullptr || image.width == 0 || image.height == 0 || image.channel < 1 || image.channel > 4) {
+ LOG_ERROR("Failed to resolve prompt image: %s", source.c_str());
+ return {};
+ }
+ auto pixels = sd_image_to_tensor(image);
+ if (image.channel <= 2) {
+ auto gray = sd::ops::slice(pixels, 2, 0, 1);
+ return sd::ops::concat(sd::ops::concat(gray, gray, 2), gray, 2);
+ }
+ return image.channel == 3 ? pixels : sd::ops::slice(pixels, 2, 0, 3);
+ })) {
+ return {};
+ }
+ resolved.prompt_input = &input;
+ } else {
+ resolved.text = sd::prompt_input_text(input);
+ }
+ return conditioning_cache_->get(*cond_stage_model, n_threads, resolved);
}
void StableDiffusionGGML::reset_generation_extensions() {
diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h
index 990a8b6..d2a05e0 100644
--- a/src/pipeline/diffusion_engine.h
+++ b/src/pipeline/diffusion_engine.h
@@ -79,6 +79,8 @@ public:
std::shared_ptr<ControlNet> control_net;
std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
sd::Tensor<float> ip_adapter_tokens;
+ sd_prompt_image_cb_t prompt_image_callback = nullptr;
+ void* prompt_image_callback_data = nullptr;
sd::Tensor<float> ip_adapter_uncond_tokens;
float ip_adapter_strength = 1.0f;
std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index 4a7af2a..df2cfcb 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -698,6 +698,17 @@ struct sd_ctx_t {
StableDiffusionGGML* sd = nullptr;
};
+void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) {
+ if (ctx == nullptr || ctx->sd == nullptr) {
+ return;
+ }
+ std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
+ if (!ctx->sd->executing_) {
+ ctx->sd->prompt_image_callback = cb;
+ ctx->sd->prompt_image_callback_data = data;
+ }
+}
+
void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) {
if (ctx == nullptr || ctx->sd == nullptr) {
return;