Commit 2c678bc for stable-diffusion.cpp
commit 2c678bcfe99b6977ee4266a3f602da66c7411743
Author: Wagner Bruna <wbruna@users.noreply.github.com>
Date: Sun Nov 22 14:05:20 2026 -0300
feat: support for MOSS-Image edit LoRAs (#2203)
Co-authored-by: leejet <leejet714@gmail.com>
diff --git a/README.md b/README.md
index e27eacb..b5ddd2a 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,7 @@ API and command-line option may change frequently.***
- [Mage-Flow-Edit](./docs/models/mage_flow.md#image-editing)
- [LLaDA-Image Edit](./docs/models/llada_image.md#image-editing)
- [Fibo 2.0 Dev / Edit Turbo](./docs/models/fibo2.md#fibo-20-edit-turbo)
+ - [MOSS-Image Edit](./docs/models/moss_image.md#moss-image-edit)
- Video Models
- [Wan2.1/Wan2.2](./docs/models/wan.md)
- [MiniMax-H3](./docs/models/minimax_h3.md)
diff --git a/docs/dgl-builtins.md b/docs/dgl-builtins.md
index 4905a7f..073fbe7 100644
--- a/docs/dgl-builtins.md
+++ b/docs/dgl-builtins.md
@@ -70,7 +70,7 @@ Optional parameters are marked with `?`. Default values follow `=`. See [DGL syn
| `photomaker::encode_images` | `photomaker: photomaker`, `images: list<tensor>`, `id_embedding?: tensor` | `tensor` |
| `photomaker::encode_prompt` | `encoder: text_encoder`, `text: string`, `tokens: tensor`, `width?: int = 1024`, `height?: int = 1024`, `clip_skip?: int = -1` | `dict`<br>`condition: dict`<br>`condition.concat: tensor`<br>`condition.crossattn: tensor`<br>`condition.extra_crossattn: list<tensor>`<br>`condition.image_embed_indices: list<tensor>`<br>`condition.image_embeds: list<tensor>`<br>`condition.input_ids: tensor`<br>`condition.position_ids: tensor`<br>`condition.ref_audios: list<tensor>`<br>`condition.ref_images: list<tensor>`<br>`condition.reference_blocks: list<dict>`<br>`condition.reference_blocks.audio: int`<br>`condition.reference_blocks.kind: string`<br>`condition.reference_blocks.video: int`<br>`condition.t5_ids: tensor`<br>`condition.t5_weights: tensor`<br>`condition.token_types: tensor`<br>`condition.vector: tensor`<br>`condition.vinput_mask: tensor`<br>`mask: tensor`<br>`plain: dict`<br>`plain.concat: tensor`<br>`plain.crossattn: tensor`<br>`plain.extra_crossattn: list<tensor>`<br>`plain.image_embed_indices: list<tensor>`<br>`plain.image_embeds: list<tensor>`<br>`plain.input_ids: tensor`<br>`plain.position_ids: tensor`<br>`plain.ref_audios: list<tensor>`<br>`plain.ref_images: list<tensor>`<br>`plain.reference_blocks: list<dict>`<br>`plain.reference_blocks.audio: int`<br>`plain.reference_blocks.kind: string`<br>`plain.reference_blocks.video: int`<br>`plain.t5_ids: tensor`<br>`plain.t5_weights: tensor`<br>`plain.token_types: tensor`<br>`plain.vector: tensor`<br>`plain.vinput_mask: tensor` |
| `photomaker::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `photomaker` |
-| `predictor` | `model: denoiser`, `condition: dict`, `prediction?: string = "auto"`, `concat?: tensor`, `ref_latents?: list<tensor>`, `ref_audios?: list<tensor>`, `reference_blocks?: list<dict>`, `keyframe_indices?: list<int>`, `ref_index_mode?: string = "fixed"`, `force_ref_timestep_zero?: bool = false`, `guidance?: number = 3.5`, `skip_layers?: list<int>`, `vace_context?: tensor`, `vace_strength?: number = 1`, `audio_embedding?: tensor`, `video_positions?: tensor`, `audio_length?: int = 0`, `fps?: number = 24`, `video_sigma_shift?: number = 12`, `audio_sigma_shift?: number = 3`, `circular_x?: bool = false`, `circular_y?: bool = false`, `transform?: fn`, `transform_args?: dict` | `predictor` |
+| `predictor` | `model: denoiser`, `condition: dict`, `prediction?: string = "auto"`, `concat?: tensor`, `ref_latents?: list<tensor>`, `ref_audios?: list<tensor>`, `reference_blocks?: list<dict>`, `keyframe_indices?: list<int>`, `ref_index_mode?: string = "fixed"`, `force_ref_timestep_zero?: bool = false`, `ref_rope_anchor?: string = "default"`, `guidance?: number = 3.5`, `skip_layers?: list<int>`, `vace_context?: tensor`, `vace_strength?: number = 1`, `audio_embedding?: tensor`, `video_positions?: tensor`, `audio_length?: int = 0`, `fps?: number = 24`, `video_sigma_shift?: number = 12`, `audio_sigma_shift?: number = 3`, `circular_x?: bool = false`, `circular_y?: bool = false`, `transform?: fn`, `transform_args?: dict` | `predictor` |
| `preview` | `decoder?: vae`, `circular_x?: bool = false`, `circular_y?: bool = false` | `sampling_observer` |
| `pulid::apply` | `condition: dict`, `pulid: pulid`, `embedding: tensor`, `strength?: number = 1`, `start_sigma?: number`, `end_sigma?: number` | same type as `condition`<br>`pulid: pulid_bindings` |
| `pulid::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `pulid` |
diff --git a/docs/dgl.md b/docs/dgl.md
index 1adb624..17072e0 100644
--- a/docs/dgl.md
+++ b/docs/dgl.md
@@ -780,6 +780,10 @@ pixels to `encode_prompt` and prepared latents to `predictor` explicitly.
`predictor(ref_index_mode="increase")` selects reference position indexing;
the default is `"fixed"`. Use `"decrease"` for layered references and set
`force_ref_timestep_zero=true` when required by the checkpoint.
+For MOSS-Image, `ref_rope_anchor` aligns reference tokens with the start of the
+image (`"image_start"`) or text (`"text_start"`) along the RoPE sequence axis.
+`"default"` uses the model's alignment. See the
+[MOSS-Image editing examples](models/moss_image.md#dgl).
`prediction` belongs to `predictor`, for example
`pos = predictor(dit, condition, prediction="eps")`. `load_dit` and its
diff --git a/docs/dgl/README.md b/docs/dgl/README.md
index eaa998d..df8958c 100644
--- a/docs/dgl/README.md
+++ b/docs/dgl/README.md
@@ -65,7 +65,7 @@ and [NumPy tensor files](../dgl.md#tensor-files).
| [Ming-Image](../models/ming_image.md) | [0.1 Design text to image](ming_image/ming-image-0-1-design-txt2img.dgl) |
| [MiniMax-H3-Image](../models/minimax_h3_image.md) | [Text to image](minimax_h3_image/minimax-h3-image-txt2img.dgl) · [Image editing](minimax_h3_image/minimax-h3-image-edit.dgl) |
| [MiniT2I](../models/minit2i.md) | [CUDA](minit2i/minit2i-cuda-txt2img.dgl) · [Metal](minit2i/minit2i-metal-txt2img.dgl) (`--threads 8`) |
-| [MOSS-Image](../models/moss_image.md) | [Base: text to image](moss_image/moss-image-base-txt2img.dgl) · [SFT: text to image](moss_image/moss-image-sft-txt2img.dgl) · [Inline images](moss_image/moss-image-sft-inline-images.dgl) |
+| [MOSS-Image](../models/moss_image.md) | [Base: text to image](moss_image/moss-image-base-txt2img.dgl) · [SFT: text to image](moss_image/moss-image-sft-txt2img.dgl) · [Inline images](moss_image/moss-image-sft-inline-images.dgl) · [Image editing (HQ)](moss_image/moss-image-edit.dgl) · [Image editing (chinoll)](moss_image/moss-image-edit-chinoll.dgl) |
| [Ovis-Image](../models/ovis_image.md) | [Text to image](ovis_image/ovis-image-txt2img.dgl) |
| [P-H3-Image](../models/p_h3_image.md) | [Text to image](p_h3_image/p-h3-image-txt2img.dgl) · [Image editing](p_h3_image/p-h3-image-edit.dgl) |
| [PiD](../models/pid.md) | [512 to 2048 pixels](pid/pid-flux1-512-to-2048-txt2img.dgl) |
diff --git a/docs/dgl/moss_image/moss-image-edit-chinoll.dgl b/docs/dgl/moss_image/moss-image-edit-chinoll.dgl
new file mode 100644
index 0000000..d03021f
--- /dev/null
+++ b/docs/dgl/moss_image/moss-image-edit-chinoll.dgl
@@ -0,0 +1,63 @@
+# version: 0.0.0
+# docs/models/moss_image.md — chinoll/Moss-Image-Edit
+
+var {
+ model_path = "../models/diffusion_models/moss_image_sft_int8_convrot_attn.safetensors"
+ text_path = "../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_qwen3.5.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+ lora_path = "../models/loras/moss/moss-edit-lora.safetensors"
+ lora_weight = 1.0
+
+ reference_paths = ["../assets/flux/flux1-dev-q8_0.png"]
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'moss.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn encode_reference(path, vae, rng) {
+ image = image::to_f32(image::load(path), channels = 3)
+ shape = tensor::shape(image)
+ long_edge = shape[0] + shape[1] - math::min(shape[0], shape[1])
+ scale = math::min(1024, long_edge) / long_edge
+ rounded = [math::round(shape[0] * scale / 16) * 16, math::round(shape[1] * scale / 16) * 16]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], 16, dtype = "i64"))
+ pixels = tensor::interpolate(image, size = size, mode = "bilinear")
+ return vae_to_dit_latents(vae, vae_encode(vae, pixels, rng = rng))
+}
+
+fn main() {
+ base_model = load_dit(model_path, family = "moss_image")
+ model = apply_lora(base_model, load_lora(lora_path), weight = lora_weight, mode = "auto")
+ text = load_text_encoder(text_path, family = "moss_image", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "moss_image")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ references = map(reference_paths, process = encode_reference, vae = vae, rng = encode_rng)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, ref_latents = references,
+ ref_index_mode = "increase", ref_rope_anchor = "text_start", force_ref_timestep_zero = true)
+ neg = predictor(model, negative, ref_latents = references,
+ ref_index_mode = "increase", ref_rope_anchor = "text_start", force_ref_timestep_zero = true)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/moss_image/moss-image-edit.dgl b/docs/dgl/moss_image/moss-image-edit.dgl
new file mode 100644
index 0000000..2417f49
--- /dev/null
+++ b/docs/dgl/moss_image/moss-image-edit.dgl
@@ -0,0 +1,69 @@
+# version: 0.0.0
+# docs/models/moss_image.md — MOSS-Image-Edit-HQ
+
+var {
+ model_path = "../models/diffusion_models/moss_image_sft_int8_convrot_attn.safetensors"
+ text_path = "../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3.5-9B-mmproj-BF16.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_qwen3.5.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+ lora_path = "../models/loras/moss/armV5-moss-edit-lora.safetensors"
+ lora_weight = 1.0
+
+ reference_paths = ["../assets/flux/flux1-dev-q8_0.png"]
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'moss.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn load_reference(path) {
+ return image::to_f32(image::load(path), channels = 3)
+}
+
+fn encode_reference(image, vae, rng) {
+ shape = tensor::shape(image)
+ long_edge = shape[0] + shape[1] - math::min(shape[0], shape[1])
+ scale = math::min(1024, long_edge) / long_edge
+ rounded = [math::round(shape[0] * scale / 16) * 16, math::round(shape[1] * scale / 16) * 16]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], 16, dtype = "i64"))
+ pixels = tensor::interpolate(image, size = size, mode = "bilinear")
+ return vae_to_dit_latents(vae, vae_encode(vae, pixels, rng = rng))
+}
+
+fn main() {
+ base_model = load_dit(model_path, family = "moss_image")
+ model = apply_lora(base_model, load_lora(lora_path), weight = lora_weight, mode = "auto")
+ text = load_text_encoder(text_path, family = "moss_image",
+ tokenizer = tokenizer_path, vision_path = vision_path)
+ vae = load_vae(vae_path, family = "moss_image")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ images = map(reference_paths, process = load_reference)
+ references = map(images, process = encode_reference, vae = vae, rng = encode_rng)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height, media = images)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = images)
+ pos = predictor(model, positive, ref_latents = references,
+ ref_index_mode = "increase", ref_rope_anchor = "image_start", force_ref_timestep_zero = true)
+ neg = predictor(model, negative, ref_latents = references,
+ ref_index_mode = "increase", ref_rope_anchor = "image_start", force_ref_timestep_zero = true, skip_layers = [22])
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/guides/edit.md b/docs/guides/edit.md
index 2426cf0..ac7ef20 100644
--- a/docs/guides/edit.md
+++ b/docs/guides/edit.md
@@ -23,6 +23,7 @@ that do not support image guidance report an error.
| Fibo 2.0 | Shared center crops for VAE and vision encoding; multiple references use aspect-ratio buckets |
| [Anima Edit](../models/anima.md) | Matching community editing LoRA; VAE reference latents |
| [Krea2 Edit](../models/krea2.md) | Matching community editing LoRA; vision inputs and VAE reference latents |
+| [MOSS-Image Edit](../models/moss_image.md#moss-image-edit) | HQ LoRA with VAE reference latents and optional vision conditioning |
| LLaDA-Image | Reference pixels go to the text encoder's native image components and reference latents to the diffusion model |
| MiniMax-H3 | Aspect-preserving references aligned to the model grid |
diff --git a/docs/models/moss_image.md b/docs/models/moss_image.md
index 30cdb00..f56442a 100644
--- a/docs/models/moss_image.md
+++ b/docs/models/moss_image.md
@@ -1,6 +1,7 @@
# MOSS-Image
MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as the LLM text encoder. Both Base and SFT are supported. Prompt enhancement is not included.
+Image editing requires a matching LoRA.
## Model files
@@ -10,13 +11,15 @@ MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as th
| vae | [Safetensors](https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/vae) | `--vae` | Use matching VAE |
| Qwen3.5 9B | [GGUF](https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main) | `--llm` | Use matching text encoder |
| `tokenizer.json` | [Download](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base/tree/main/tokenizer) | `--tokenizer` | Required; match text encoder |
+| Qwen3.5 9B vision projector | [GGUF](https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/blob/main/mmproj-BF16.gguf) | `--llm_vision` | Required for vision inputs when the encoder uses separate vision weights |
+| MOSS-Image-Edit-HQ LoRA | [`armV5-moss-edit-lora.safetensors`](https://huggingface.co/akhaliq/MOSS-Image-Edit-HQ/blob/main/armV5-moss-edit-lora.safetensors) | `--lora-model-dir` | HQ editing with SFT |
+| chinoll Moss-Image-Edit LoRA | [`adapter_model.safetensors`](https://huggingface.co/chinoll/Moss-Image-Edit/blob/main/adapter_model.safetensors) | — | Editing with Base or SFT in DGL |
Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer.
-For [inline prompt images](../guides/prompt_images.md), use a matching Qwen3.5
-vision projector. Pass it with `--llm_vision` in the CLI or `vision_path` on
-`load_text_encoder` in dgl. See the
-[DGL example](../dgl/moss_image/moss-image-sft-inline-images.dgl).
+[Inline prompt images](../guides/prompt_images.md) require Qwen3.5 vision
+weights. HQ editing also uses them when available. Pass a separate projector
+with `--llm_vision` in the CLI or `vision_path` on `load_text_encoder` in DGL.
## Usage
@@ -43,6 +46,8 @@ Select optional components when required by your checkpoint or task.
| [Base: text to image](../dgl/moss_image/moss-image-base-txt2img.dgl) | MOSS-Image Base. |
| [SFT: text to image](../dgl/moss_image/moss-image-sft-txt2img.dgl) | MOSS-Image SFT. |
| [Inline images](../dgl/moss_image/moss-image-sft-inline-images.dgl) | Insert named images into the prompt. |
+| [Image editing (HQ)](../dgl/moss_image/moss-image-edit.dgl) | SFT with the armV5 LoRA and a Qwen3.5 vision projector. |
+| [Image editing (chinoll)](../dgl/moss_image/moss-image-edit-chinoll.dgl) | Base or SFT with the chinoll LoRA. |
#### Run a script
@@ -140,4 +145,50 @@ CLI examples use the [shared path and shell conventions](README.md#example-conve
<img src="../../assets/moss/sft.png" />
-Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The CLI and C API set `skip_layers = [22]` on the negative predictor when translating requests to DGL, matching the upstream pipeline. Handwritten DGL scripts specify this zero-based layer index explicitly, as shown in the examples.
+#### MOSS-Image Edit
+
+The CLI uses the HQ workflow with SFT. Save the LoRA as
+`models/loras/moss/armV5-moss-edit-lora.safetensors` or adjust the LoRA tag below.
+Pass `-r` for each reference image.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe `
+ --diffusion-model ..\models\diffusion_models\moss_image_sft_int8_convrot_attn.safetensors `
+ --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf `
+ --llm_vision ..\models\text_encoders\Qwen3.5-9B-mmproj-BF16.gguf `
+ --vae ..\models\vae\flux2_ae.safetensors `
+ --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json `
+ --lora-model-dir ..\models\loras `
+ -r ..\assets\flux\flux1-dev-q8_0.png `
+ -p "change 'flux.cpp' to 'moss.cpp'<lora:moss/armV5-moss-edit-lora:1.0>" `
+ --cfg-scale 3.0 `
+ -v
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli \
+ --diffusion-model ../models/diffusion_models/moss_image_sft_int8_convrot_attn.safetensors \
+ --llm ../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf \
+ --llm_vision ../models/text_encoders/Qwen3.5-9B-mmproj-BF16.gguf \
+ --vae ../models/vae/flux2_ae.safetensors \
+ --tokenizer ../models/tokenizers/tokenizer_qwen3.5.json \
+ --lora-model-dir ../models/loras \
+ -r ../assets/flux/flux1-dev-q8_0.png \
+ -p "change 'flux.cpp' to 'moss.cpp'<lora:moss/armV5-moss-edit-lora:1.0>" \
+ --cfg-scale 3.0 \
+ -v
+```
+
+## Notes and limitations
+
+Use output dimensions divisible by 16. The editing examples cap the reference
+long edge at 1024 pixels. The discrete scheduler defaults to a flow shift of 5.
+Use `--flow-shift` to override it. Prompt weights use `(phrase:1.2)` syntax.
+
+For text-to-image and HQ editing, the CLI and C API set `skip_layers = [22]` on
+the negative predictor. The corresponding DGL examples set the same zero-based
+layer index.
diff --git a/src/api/generation_image.cpp b/src/api/generation_image.cpp
index cdaa3c6..a5168ba 100644
--- a/src/api/generation_image.cpp
+++ b/src/api/generation_image.cpp
@@ -27,6 +27,10 @@ namespace sd::api {
const double scale = version == VERSION_P_H3_IMAGE ? std::min(1.0, std::sqrt(double(target_width) * target_height / (double(width) * height))) : double(std::clamp(std::min(width, height), 1024, 2048)) / std::min(width, height);
width = std::max(32, int(std::nearbyint(width * scale / 32)) * 32);
height = std::max(32, int(std::nearbyint(height * scale / 32)) * 32);
+ } else if (version == VERSION_MOSS_IMAGE) {
+ const double scale = std::min(1.0, 1024.0 / std::max(width, height));
+ width = std::max(factor, int(std::round(width * scale / factor)) * factor);
+ height = std::max(factor, int(std::round(height * scale / factor)) * factor);
} else if (version != VERSION_ANIMA && !sd_version_is_pid(version)) {
if (sd_version_is_llada_image(version) || sd_version_is_mage_flow(version) || (version == VERSION_FIBO2 && request.image.ref_images_count == 1)) {
width = target_width;
@@ -50,9 +54,7 @@ namespace sd::api {
return pixels;
}
- static bool qwen_prompt_has_vision(const GenerationTemplate& request) {
- if (request.definition.version == VERSION_QWEN_IMAGE_LAYERED)
- return false;
+ static bool prompt_has_vision(const GenerationTemplate& request) {
const auto* vision_path = request.context.config.context.llm_vision_path;
if (vision_path && *vision_path)
return true;
@@ -72,6 +74,11 @@ namespace sd::api {
const auto refs = bound_tensors(request.options, "ref_images");
if (refs.empty() && !sd_version_is_unet_edit(version))
return;
+ const bool moss_edit = version == VERSION_MOSS_IMAGE;
+ const bool qwen_edit = sd_version_is_qwen_image(version) && version != VERSION_QWEN_IMAGE_LAYERED;
+ const bool has_vision = (moss_edit || qwen_edit) && prompt_has_vision(request);
+ if (moss_edit && !has_vision)
+ LOG_INFO("MOSS-Image HQ edit is using text-only prompt conditioning; set llm_vision_path (--llm_vision) to add reference images to Qwen3.5");
std::string pixels, latents;
const bool encode = version != VERSION_HIDREAM_O1;
for (size_t i = 0; i < refs.size(); ++i) {
@@ -90,7 +97,7 @@ namespace sd::api {
request.fields["negative_predictor_options"] += ", reference_blocks = " + literal(blocks);
request.fields["reference_predictor_options"] += ", reference_blocks = " + literal(blocks);
}
- if ((sd_version_is_qwen_image(version) && qwen_prompt_has_vision(request)) || sd_version_is_minimax_h3_image(version) ||
+ if (has_vision || sd_version_is_minimax_h3_image(version) ||
version == VERSION_FIBO2 || sd_version_is_llada_image(version) || sd_version_is_krea2(version) ||
sd_version_is_mage_flow(version) || sd_version_is_longcat(version) || sd_version_is_boogu_image(version)) {
request.fields["prompt_options"] += ", media = [" + pixels + "]";
@@ -98,14 +105,16 @@ namespace sd::api {
request.fields["negative_prompt_options"] += ", media = [" + pixels + "]";
}
if (encode && !sd_version_is_unet_edit(version)) {
- const bool increase = sd_version_is_flux2(version) || sd_version_is_flux3(version) ||
+ const bool increase = sd_version_is_flux2(version) || sd_version_is_flux3(version) ||
sd_version_is_qwen_image(version) || sd_version_is_minimax_h3_image(version) ||
version == VERSION_FIBO2 || sd_version_is_krea2(version) ||
- version == VERSION_ANIMA || sd_version_is_mage_flow(version);
+ version == VERSION_ANIMA || sd_version_is_mage_flow(version) || moss_edit;
std::string options = ", ref_latents = [" + latents + "], ref_index_mode = " +
literal(version == VERSION_QWEN_IMAGE_LAYERED ? "decrease" : increase ? "increase"
: "fixed") +
- ", force_ref_timestep_zero = " + literal(sd_version_is_llada_image(version) || sd_version_is_krea2(version));
+ ", force_ref_timestep_zero = " + literal(sd_version_is_llada_image(version) || sd_version_is_krea2(version) || moss_edit);
+ if (moss_edit)
+ options += ", ref_rope_anchor = \"image_start\"";
request.fields["predictor_options"] += options;
request.fields["negative_predictor_options"] += options;
request.fields["reference_predictor_options"] += options;
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index 7e2fcf8..64f8007 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -14,6 +14,7 @@
#include "conditioning/textual_inversion.h"
#include "core/ggml_tensor_utils.h"
#include "core/image_geometry.h"
+#include "core/image_tensor.h"
#include "core/tensor_ggml.hpp"
#include "core/util.h"
@@ -1993,7 +1994,8 @@ struct LLMEmbedder : public Conditioner {
return 0;
return sd_version_is_minimax_h3(version) || sd_version_is_lingbot_video(version) ||
sd_version_is_qwen_image(version) || sd_version_is_mage_flow(version) ||
- sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_longcat(version)
+ sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_longcat(version) ||
+ version == VERSION_MOSS_IMAGE
? std::numeric_limits<size_t>::max()
: 0;
}
@@ -2560,8 +2562,28 @@ struct LLMEmbedder : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& params) override {
ConditionerParams conditioner_params = params;
- sd::PromptInputTemplate input_template(params.prompt_input);
- if (params.prompt_input != nullptr) {
+ sd::PromptInput prompt_with_references;
+ if (version == VERSION_MOSS_IMAGE && params.ref_images && !params.ref_images->empty()) {
+ for (size_t i = 0; i < params.ref_images->size(); ++i) {
+ const auto& source = (*params.ref_images)[i];
+ const auto width = source.shape()[0], height = source.shape()[1];
+ const double scale = std::min(1.0, std::sqrt(384.0 * 384.0 / (double(width) * height)));
+ const auto w = std::max<int64_t>(32, std::llround(width * scale));
+ const auto h = std::max<int64_t>(32, std::llround(height * scale));
+ auto rgb = sd::ensure_image_tensor_channels(source, 3);
+ if (w != width || h != height)
+ rgb = sd::ops::interpolate(rgb, {w, h, 3, 1}, sd::ops::InterpolateMode::Bicubic, false, true);
+ prompt_with_references.push_back({sd::PromptPart::Type::TEXT, "Picture " + std::to_string(i + 1) + ": "});
+ prompt_with_references.push_back({sd::PromptPart::Type::IMAGE, {}, std::make_shared<sd::Tensor<float>>(std::move(rgb))});
+ }
+ if (params.prompt_input)
+ prompt_with_references.insert(prompt_with_references.end(), params.prompt_input->begin(), params.prompt_input->end());
+ else
+ prompt_with_references.push_back({sd::PromptPart::Type::TEXT, params.text});
+ conditioner_params.prompt_input = &prompt_with_references;
+ }
+ sd::PromptInputTemplate input_template(conditioner_params.prompt_input);
+ if (conditioner_params.prompt_input != nullptr) {
if (!supports_prompt_images()) {
LOG_ERROR("This conditioner does not support inline prompt images");
return {};
@@ -3170,8 +3192,11 @@ struct LLMEmbedder : public Conditioner {
} else if (version == VERSION_MOSS_IMAGE) {
prompt_template_encode_start_idx = 0;
max_length = 4096;
- prompt = system_prefix("<|im_start|>user\n") + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<think>\n";
- prompt_attn_range = {0, 0};
+ prompt = system_prefix("<|im_start|>user\n");
+ prompt_attn_range.first = static_cast<int>(prompt.size());
+ prompt += conditioner_params.text;
+ prompt_attn_range.second = static_cast<int>(prompt.size());
+ prompt += "<|im_end|>\n<|im_start|>assistant\n<think>\n";
} else if (sd_version_is_flux3(version)) {
prompt_template_encode_start_idx = 0;
max_length = 4096;
@@ -3329,7 +3354,7 @@ struct LLMEmbedder : public Conditioner {
image_grids,
prompt_suffix,
version == VERSION_IRIS ? &text_mask : nullptr,
- params.prompt_input == nullptr ? nullptr : &input_template,
+ conditioner_params.prompt_input == nullptr ? nullptr : &input_template,
&prompt_tokens);
if (hidden_states.empty()) {
return {};
diff --git a/src/dgl/bindings/predictor.cpp b/src/dgl/bindings/predictor.cpp
index 562b449..5e20df0 100644
--- a/src/dgl/bindings/predictor.cpp
+++ b/src/dgl/bindings/predictor.cpp
@@ -136,6 +136,12 @@ namespace sd::dgl {
: ref_index_mode == "decrease" ? Rope::RefIndexMode::DECREASE
: Rope::RefIndexMode::FIXED;
output.options.force_ref_timestep_zero = get_bool(args, "force_ref_timestep_zero");
+ const auto ref_rope_anchor = get_string(args, "ref_rope_anchor");
+ if (ref_rope_anchor != "default" && engine.version() != VERSION_MOSS_IMAGE)
+ throw std::runtime_error("reference RoPE anchoring is not configurable for this model");
+ output.options.ref_rope_anchor = ref_rope_anchor == "image_start" ? Rope::RefRoPEAnchor::IMAGE_START
+ : ref_rope_anchor == "text_start" ? Rope::RefRoPEAnchor::TEXT_START
+ : Rope::RefRoPEAnchor::MODEL_DEFAULT;
output.options.guidance = float(get_number(args, "guidance"));
bind_predictor_options(output, args);
const auto& condition = args.at("condition").fields;
diff --git a/src/dgl/bindings/registration.cpp b/src/dgl/bindings/registration.cpp
index f06ebb2..6a066ef 100644
--- a/src/dgl/bindings/registration.cpp
+++ b/src/dgl/bindings/registration.cpp
@@ -578,6 +578,7 @@ namespace sd::dgl {
optional_list("ref_latents", Kind::Tensor), optional_list("ref_audios", Kind::Tensor),
optional_list("reference_blocks", Kind::Dict), optional_list("keyframe_indices", Kind::Integer),
choice("ref_index_mode", {"fixed", "increase", "decrease"}, "fixed"), optional("force_ref_timestep_zero", Kind::Boolean, false),
+ choice("ref_rope_anchor", {"default", "image_start", "text_start"}, "default"),
number("guidance", 0, 1000, 3.5), optional_list("skip_layers", Kind::Integer),
optional("vace_context", Kind::Tensor), number("vace_strength", 0, 100, 1),
optional("audio_embedding", Kind::Tensor), optional("video_positions", Kind::Tensor),
diff --git a/src/dgl/models/diffusion_prediction.h b/src/dgl/models/diffusion_prediction.h
index 4a7cb78..94bc6d1 100644
--- a/src/dgl/models/diffusion_prediction.h
+++ b/src/dgl/models/diffusion_prediction.h
@@ -41,6 +41,7 @@ namespace sd::dgl::models {
float audio_sigma_shift = 3.f;
int audio_length = 0;
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED;
+ Rope::RefRoPEAnchor ref_rope_anchor = Rope::RefRoPEAnchor::MODEL_DEFAULT;
bool force_ref_timestep_zero = false;
std::function<PredictionParameters(PredictionParameters)> transform;
};
diff --git a/src/dgl/models/dit.cpp b/src/dgl/models/dit.cpp
index 9999f46..73e7bba 100644
--- a/src/dgl/models/dit.cpp
+++ b/src/dgl/models/dit.cpp
@@ -359,6 +359,7 @@ namespace sd::dgl::models {
params.c_concat = present(condition.c_concat);
params.ref_latents = references.empty() ? &condition.c_ref_images : &references;
params.ref_index_mode = options.ref_index_mode;
+ params.ref_rope_anchor = options.ref_rope_anchor;
params.force_ref_timestep_zero = options.force_ref_timestep_zero;
UNetControls unet_controls;
std::vector<ZImageControlInput> z_image_controls;
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index 168b68e..ddb2f24 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -8,6 +8,7 @@
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
+#include "json.hpp"
#include "model.h"
#include "model/adapter/lora_ops.h"
#include "model_loader.h"
@@ -66,6 +67,38 @@ struct LoraModel : public GGMLRunner {
ggml_backend_t backend() const { return runtime_backend; }
+ void load_moss_edit_scales(const std::string& adapter_config) {
+ try {
+ const auto config = nlohmann::json::parse(adapter_config);
+ if (config.value("format", "") != "moss-edit-lora" || config.value("format_version", 0) != 1)
+ return;
+ const auto modules = config.find("modules");
+ if (modules == config.end() || !modules->is_object())
+ return;
+ for (const auto& [target, spec] : modules->items()) {
+ const auto alpha = spec.find("alpha");
+ if (alpha == spec.end() || !alpha->is_number())
+ continue;
+ const float value = alpha->get<float>();
+ if (!std::isfinite(value) || value < 0.f)
+ continue;
+
+ auto name = convert_tensor_name("lora." + target + ".scale", source_version);
+ const auto base = name.substr(0, name.size() - (sizeof(".scale") - 1));
+ const auto down = lora_tensors.find(base + ".lora_down");
+ if (down == lora_tensors.end() || lora_tensors.count(name) || lora_tensors.count(base + ".alpha"))
+ continue;
+
+ // Use the loaded linear tensor's rank and the same scale path as file-backed scalars.
+ auto* scale = ggml_new_tensor_1d(params_ctx, GGML_TYPE_F32, 1);
+ scale->data = &scalar_values.emplace(scale, value / down->second->ne[1]).first->second;
+ lora_tensors.emplace(std::move(name), scale);
+ }
+ } catch (const nlohmann::json::exception& e) {
+ LOG_WARN("invalid MOSS-Image Edit adapter_config in '%s': %s", file_path.c_str(), e.what());
+ }
+ }
+
bool init_params(int n_threads, filter_t filter = nullptr) {
if (!weights)
return false;
@@ -76,7 +109,8 @@ struct LoraModel : public GGMLRunner {
const auto metadata = weights->loader().file_metadata(source_file);
const auto alpha = metadata.find("alpha");
// Converters that write "baked_scale" have already folded alpha / rank into lora_up.
- if (alpha != metadata.end() && metadata.find("baked_scale") == metadata.end()) {
+ const bool baked_scale = metadata.count("baked_scale") != 0;
+ if (alpha != metadata.end() && !baked_scale) {
try {
size_t parsed = 0;
metadata_alpha = std::stof(alpha->second, &parsed);
@@ -126,6 +160,9 @@ struct LoraModel : public GGMLRunner {
if (!weights->loader().load_file_tensors(source_file, source_version, callback, scalar_names))
return false;
}
+ const auto adapter_config = metadata.find("adapter_config");
+ if (source_version == VERSION_MOSS_IMAGE && !baked_scale && metadata_alpha < 0.f && adapter_config != metadata.end())
+ load_moss_edit_scales(adapter_config->second);
if (!weights->register_param_tensors(ModelComponent::LoRA, tensors, residency_mode,
runtime_backend, params_backend, nullptr, false, params_follow_compute,
nullptr, source_file, source_version))
diff --git a/src/model/common/rope.hpp b/src/model/common/rope.hpp
index b6f3111..c3d638b 100644
--- a/src/model/common/rope.hpp
+++ b/src/model/common/rope.hpp
@@ -62,6 +62,12 @@ namespace Rope {
DECREASE,
};
+ enum class RefRoPEAnchor {
+ MODEL_DEFAULT,
+ IMAGE_START,
+ TEXT_START,
+ };
+
template <class T>
__STATIC_INLINE__ std::vector<T> linspace(T start, T end, int num) {
std::vector<T> result(num);
diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp
index b717192..993624b 100644
--- a/src/model/diffusion/model.hpp
+++ b/src/model/diffusion/model.hpp
@@ -178,6 +178,7 @@ struct DiffusionParams {
const sd::Tensor<float>* y = nullptr;
const std::vector<sd::Tensor<float>>* ref_latents = nullptr;
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED;
+ Rope::RefRoPEAnchor ref_rope_anchor = Rope::RefRoPEAnchor::MODEL_DEFAULT;
bool force_ref_timestep_zero = false;
DiffusionExtraParams extra = std::monostate{};
};
diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp
index 4bb9077..3e87433 100644
--- a/src/model/diffusion/moss_image.hpp
+++ b/src/model/diffusion/moss_image.hpp
@@ -2,8 +2,8 @@
#define __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__
#include <algorithm>
-#include <array>
#include <cinttypes>
+#include <stdexcept>
#include "core/pe_cache.h"
@@ -94,6 +94,26 @@ namespace MOSSImage {
}
};
+ static ggml_tensor* attend(GGMLRunnerContext* ctx, ggml_tensor* q, ggml_tensor* k, ggml_tensor* v, ggml_tensor* pe, int64_t ref_start) {
+ const int64_t tokens = q->ne[2];
+ if (ref_start == tokens) {
+ return Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false);
+ }
+
+ const int64_t heads = q->ne[1];
+ q = Rope::apply_rope(ctx, q, pe, false);
+ k = Rope::apply_rope(ctx, k, pe, false);
+ // Image and text queries see the whole sequence. Reference queries see only references.
+ auto attention = [&](ggml_tensor* query, ggml_tensor* key, ggml_tensor* value) {
+ return ggml_ext_attention_ext(ctx, query, key, value, heads, nullptr, true, ctx->flash_attn_enabled);
+ };
+ auto body = attention(ggml_ext_slice(ctx, q, 1, 0, ref_start, false), k, v);
+ auto refs = attention(ggml_ext_slice(ctx, q, 1, ref_start, tokens, false),
+ ggml_ext_slice(ctx, k, 1, ref_start, tokens, false),
+ ggml_ext_slice(ctx, v, 2, ref_start, tokens, false));
+ return ggml_concat(ctx->ggml_ctx, body, refs, 1);
+ }
+
struct Block : public GGMLBlock {
MOSSImageConfig config;
@@ -109,13 +129,12 @@ namespace MOSSImage {
blocks["proj.ffn_down"] = std::make_shared<Linear>(config.ffn_dim, config.dim, false);
}
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::array<ggml_tensor*, 6>& mod) {
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<ggml_tensor*>& mod, int64_t ref_start) {
auto c = ctx->ggml_ctx;
auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
auto norm = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<RMSNorm>(blocks[name])->forward(ctx, h); };
- auto part = [&](int i) { return mod[i]; };
auto y = norm("norm1", x);
- y = ggml_ext_modulate(ctx, y, part(0), part(1));
+ y = ggml_ext_modulate(ctx, y, mod[0], mod[1]);
auto qkv = linear("proj.qkv", y);
auto split = [&](int i) {
return ggml_view_4d(c, qkv, config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2],
@@ -124,16 +143,16 @@ namespace MOSSImage {
auto q = norm("q_norm", split(0));
auto k = norm("k_norm", split(1));
auto v = split(2);
- y = Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false);
+ y = attend(ctx, q, k, v, pe, ref_start);
y = linear("proj.o", y);
- y = ggml_mul(c, y, part(2));
+ y = ggml_mul(c, y, mod[2]);
x = ggml_add(c, x, y);
y = norm("norm2", x);
- y = ggml_ext_modulate(ctx, y, part(3), part(4));
+ y = ggml_ext_modulate(ctx, y, mod[3], mod[4]);
auto up_gate = linear("proj.ffn_up_gate", y);
y = ggml_ext_situ(ctx, up_gate);
y = linear("proj.ffn_down", y);
- y = ggml_mul(c, y, part(5));
+ y = ggml_mul(c, y, mod[5]);
return ggml_add(c, x, y);
}
};
@@ -159,31 +178,41 @@ namespace MOSSImage {
blocks["final_linear_video"] = std::make_shared<Linear>(config.dim, config.channels, false);
}
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* time, ggml_tensor* text, ggml_tensor* pe, const std::vector<int>& skip) {
- auto c = ctx->ggml_ctx;
- auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
- int64_t image_len = image->ne[0] * image->ne[1];
- auto pixels = ggml_reshape_3d(c, image, image_len, image->ne[2], image->ne[3]);
- auto x = linear("video_embedder", ggml_cont(c, ggml_transpose(c, pixels)));
- text = linear("text_embedder.proj", text);
- auto text_pe = ggml_ext_slice(ctx, pe, 3, image_len, image_len + text->ne[1], false);
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* time, ggml_tensor* text, ggml_tensor* pe, const std::vector<int>& skip, const std::vector<ggml_tensor*>& refs, ggml_tensor* modulation_rows) {
+ auto c = ctx->ggml_ctx;
+ auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
+ auto embed_image = [&](ggml_tensor* latent) {
+ auto pixels = ggml_reshape_3d(c, latent, latent->ne[0] * latent->ne[1], latent->ne[2], latent->ne[3]);
+ return linear("video_embedder", ggml_cont(c, ggml_transpose(c, pixels)));
+ };
+ auto x = embed_image(image);
+ const int64_t image_len = x->ne[1];
+ text = linear("text_embedder.proj", text);
+ auto text_pe = ggml_ext_slice(ctx, pe, 3, image_len, image_len + text->ne[1], false);
for (int i = 0; i < 2; ++i) {
text = std::dynamic_pointer_cast<TextBlock>(blocks["text_embedder.blocks." + std::to_string(i)])->forward(ctx, text, text_pe);
}
- time = linear("time_embedding.2", ggml_silu(c, linear("time_embedding.0", time)));
- auto mod = linear("shared_block_modulation", time);
- auto head = linear("video_head_modulation", time);
- std::array<ggml_tensor*, 6> parts;
- for (int i = 0; i < 6; ++i) {
- parts[i] = ggml_ext_slice(ctx, mod, 0, i * config.dim, (i + 1) * config.dim, false);
+ x = ggml_concat(c, x, text, 1);
+ const int64_t ref_start = x->ne[1];
+ for (auto* ref : refs) {
+ x = ggml_concat(c, x, embed_image(ref), 1);
+ }
+
+ time = linear("time_embedding.2", ggml_silu(c, linear("time_embedding.0", time)));
+ auto head = linear("video_head_modulation", ggml_ext_slice(ctx, time, 1, 0, 1, false));
+ auto parts = ggml_ext_chunk(ctx, linear("shared_block_modulation", time), 6, 0, false);
+ for (auto*& part : parts) {
+ if (modulation_rows) {
+ part = ggml_get_rows(c, part, modulation_rows);
+ }
if (ctx->graph != nullptr) {
- ggml_build_forward_expand(ctx->graph, parts[i]);
+ ggml_build_forward_expand(ctx->graph, part);
}
}
- x = ggml_concat(c, x, text, 1);
+
for (int64_t i = 0; i < config.depth; ++i) {
if (std::find(skip.begin(), skip.end(), i) == skip.end()) {
- x = std::dynamic_pointer_cast<Block>(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, parts);
+ x = std::dynamic_pointer_cast<Block>(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, parts, ref_start);
sd::ggml_graph_cut::mark_graph_cut(x, "moss_image.layers." + std::to_string(i), "x");
}
}
@@ -218,49 +247,83 @@ namespace MOSSImage {
LOG_ERROR("MOSS-Image requires latents, timesteps and text features");
return {};
}
- std::vector<int> skip;
- if (auto extra = std::get_if<SkipLayerDiffusionExtra>(&inputs.extra)) {
- if (extra->skip_layers) {
- skip = *extra->skip_layers;
+ const auto* extra = std::get_if<SkipLayerDiffusionExtra>(&inputs.extra);
+ const std::vector<int> no_skips;
+ const auto& skip = extra && extra->skip_layers ? *extra->skip_layers : no_skips;
+ const auto* references = inputs.ref_latents;
+ if (references) {
+ for (const auto& ref : *references) {
+ if (ref.empty() || ref.dim() != 4 || ref.shape()[2] != config.channels || ref.shape()[3] != 1) {
+ throw std::runtime_error("MOSS-Image reference latents must have shape [width, height, channels, 1] with the model's latent channel count");
+ }
}
}
+ sd::Tensor<int32_t> modulation_rows;
auto graph = [&]() {
auto gf = new_graph_custom(65536);
auto x = make_input(*inputs.x);
auto text = make_input(*inputs.context);
GGML_ASSERT(x->ne[3] == 1 && x->ne[2] == config.channels && text->ne[0] == config.text_dim);
- int64_t h = x->ne[1], w = x->ne[0], nt = text->ne[1];
+ const auto h = x->ne[1], w = x->ne[0], nt = text->ne[1];
+ const auto ref_start = h * w + nt;
+ auto tokens = ref_start;
+ // ai-toolkit aligns references with image positions. Chinoll aligns them with the start of the text.
+ const auto text_offset = references && !references->empty() && inputs.ref_rope_anchor != Rope::RefRoPEAnchor::TEXT_START ? -nt : 0;
sd::PECacheKey pe_key("moss_image.pe");
- pe_key.add(h).add(w).add(nt).add(config.head_dim).add(circular_x_enabled).add(circular_y_enabled);
+ pe_key.add(h).add(w).add(nt).add(text_offset).add(config.head_dim).add(circular_x_enabled).add(circular_y_enabled);
+ std::vector<ggml_tensor*> refs;
+ if (references) {
+ for (const auto& ref : *references) {
+ refs.push_back(make_input(ref));
+ pe_key.add(ref.shape()[1]).add(ref.shape()[0]);
+ tokens += ref.shape()[1] * ref.shape()[0];
+ }
+ }
auto pe = make_input(pe_cache_.get_or_create(pe_key, [&]() {
Rope::Embedding embedding;
- embedding.positions.append_image(static_cast<int>(h), static_cast<int>(w));
- embedding.positions.images.back().height_axis = 2;
- embedding.positions.images.back().width_axis = 3;
- embedding.positions.append_tokens(nt);
- for (int64_t token = 0; token < h * w + nt; ++token) {
- float pos[4] = {0.f, float(token - h * w), 0.f, 0.f};
- if (token < h * w) {
- pos[1] = float(nt);
- pos[2] = float(token / w) - float(h - 1) / 2.f;
- pos[3] = float(token % w) - float(w - 1) / 2.f;
+ embedding.ids.reserve(tokens);
+ auto append_image = [&](int64_t height, int64_t width, float ref_index, float text_position) {
+ embedding.positions.append_image(static_cast<int>(height), static_cast<int>(width));
+ auto& region = embedding.positions.images.back();
+ region.height_axis = 2;
+ region.width_axis = 3;
+ for (int64_t i = 0; i < height * width; ++i) {
+ embedding.ids.push_back({ref_index, text_position,
+ float(i / width) - float(height - 1) / 2.f,
+ float(i % width) - float(width - 1) / 2.f});
}
- embedding.ids.push_back({pos[0], pos[1], pos[2], pos[3]});
+ };
+ append_image(h, w, 0.f, float(nt + text_offset));
+ embedding.positions.append_tokens(nt);
+ for (int64_t i = 0; i < nt; ++i) {
+ embedding.ids.push_back({0.f, float(i + text_offset), 0.f, 0.f});
+ }
+ for (size_t i = 0; i < refs.size(); ++i) {
+ append_image(refs[i]->ne[1], refs[i]->ne[0], float(i + 1), 0.f);
}
embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, std::vector<int>(4, static_cast<int>(config.head_dim / 4)), embedding.layout, &embedding.frequencies);
- auto values = finish_rope_pe(std::move(embedding));
- return sd::Tensor<float>({2, 2, config.head_dim / 2, h * w + nt}, std::move(values));
+ return sd::Tensor<float>({2, 2, config.head_dim / 2, tokens}, finish_rope_pe(std::move(embedding)));
}));
- time_data.resize(config.freq_dim);
- for (int64_t i = 0; i < config.freq_dim / 2; ++i) {
- double angle = inputs.timesteps->values()[0] * std::pow(10000.0, -double(i) / double(config.freq_dim / 2));
- time_data[i] = float(std::cos(angle));
- time_data[i + config.freq_dim / 2] = float(std::sin(angle));
+ const auto half = config.freq_dim / 2;
+ const int time_count = refs.empty() ? 1 : 2;
+ time_data.assign(time_count * config.freq_dim, 0.f);
+ for (int64_t i = 0; i < half; ++i) {
+ const double angle = inputs.timesteps->values()[0] * std::pow(10000.0, -double(i) / double(half));
+ time_data[i] = float(std::cos(angle));
+ time_data[i + half] = float(std::sin(angle));
+ }
+ ggml_tensor* rows = nullptr;
+ if (!refs.empty()) {
+ // The second row is the timestep-zero embedding for reference tokens.
+ std::fill_n(time_data.begin() + config.freq_dim, half, 1.f);
+ modulation_rows = sd::Tensor<int32_t>::zeros({tokens});
+ std::fill_n(modulation_rows.data() + ref_start, tokens - ref_start, 1);
+ rows = make_input(modulation_rows);
}
- auto time = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.freq_dim, 1, 1);
+ auto time = ggml_new_tensor_2d(compute_ctx, GGML_TYPE_F32, config.freq_dim, time_count);
set_backend_tensor_data(time, time_data.data());
auto ctx = get_context(gf);
- ggml_build_forward_expand(gf, model.forward(&ctx, x, time, text, pe, skip));
+ ggml_build_forward_expand(gf, model.forward(&ctx, x, time, text, pe, skip, refs, rows));
return gf;
};
return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim());
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index a4bf8f6..3032597 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -1076,6 +1076,8 @@ static std::string convert_diffusers_dit_to_original_pixart(std::string name) {
}
static std::string convert_moss_image_dit_name(std::string name) {
+ if (starts_with(name, "transformer.layers."))
+ name.erase(0, sizeof("transformer.") - 1);
static const std::vector<std::pair<std::string, std::string>> name_map = {
{".proj.q.weight", ".proj.qkv.weight"},
{".proj.k.weight", ".proj.qkv.weight.1"},
@@ -1937,7 +1939,8 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
if (starts_with(name, "down_blocks.") || starts_with(name, "up_blocks.") ||
starts_with(name, "mid_block.") || starts_with(name, "conv_in.") ||
starts_with(name, "conv_out.") || starts_with(name, "time_embedding.") ||
- starts_with(name, "conv_norm_out.")) {
+ starts_with(name, "conv_norm_out.") ||
+ (version == VERSION_MOSS_IMAGE && starts_with(name, "layers."))) {
const std::string& canonical_prefix = diffuison_model_prefix_vec.front();
name = convert_diffusion_model_name(name, canonical_prefix, version);
name = canonical_prefix + name;