Commit 3e92bf9 for stable-diffusion.cpp
commit 3e92bf990ad70566212ee5eb27ad7570fb7ed2af
Author: leejet <leejet714@gmail.com>
Date: Wed Nov 4 23:24:51 2026 +0800
feat: add P-H3-Image generation and editing support (#2272)
diff --git a/README.md b/README.md
index 1337c5b..28af00a 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,7 @@ API and command-line option may change frequently.***
- [Z-Image](./docs/z_image.md)
- [Z-Image L2P](./docs/z_image_l2p.md)
- [MiniT2I](./docs/minit2i.md)
+ - [P-H3-Image](./docs/p_h3_image.md)
- [SenseNova U1.5](./docs/sensenova_u1.md)
- [Ovis-Image](./docs/ovis_image.md)
- [Anima](./docs/anima.md)
diff --git a/docs/p_h3_image.md b/docs/p_h3_image.md
new file mode 100644
index 0000000..20de6fb
--- /dev/null
+++ b/docs/p_h3_image.md
@@ -0,0 +1,34 @@
+# How to Use
+
+P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax-H3 diffusion transformer, a single-frame VAE, and Qwen3-VL-4B with a text connector. Prompt enhancement is not included.
+
+## Download weights
+
+- Download P-H3-Image
+ - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/diffusion_models
+- Download P-H3-Image VAE
+ - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/vae
+- Download Qwen3-VL-4B with the text connector
+ - https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders
+
+Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`.
+
+Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE is needed.
+
+## Examples
+
+### Text-to-image
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -p "a lovely cat holding a sign says 'p-h3-image.cpp'" -v --offload-to-cpu --rng cpu -H 2912 -W 2912
+```
+
+### Image editing
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -r source.png -r reference.png -p "Edit <Picture 1>. Give the cat the scarf from <Picture 2>, preserving its face and the background." -v --offload-to-cpu --rng cpu -H 2912 -W 2912
+```
+
+Reference images are numbered in argument order as `<Picture 1>`, `<Picture 2>`, and so on. A plain edit instruction gets `Edit <Picture 1>. ` prepended automatically. Prompts starting with `Edit <Picture N>` or `subject_definitions:` are used directly.
+
+Use dimensions divisible by 32. The upstream default is 2912 × 2912; smaller canvases reduce memory use. Use CFG 1 and `--sigmas p-h3-image` for the upstream 8-step schedule. Keep the default VAE tiling size of 256 pixels.
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 637bbb6..d5496f6 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -464,7 +464,7 @@ ArgOptions SDContextParams::get_options() {
&uncond_diffusion_model_path},
{"",
"--embeddings-connectors",
- "path to LTXAV embeddings connectors",
+ "path to embeddings connectors (LTXAV or P-H3-Image)",
0,
&embeddings_connectors_path},
{"",
@@ -1530,6 +1530,8 @@ ArgOptions SDGenerationParams::get_options() {
std::string sigmas_str = value;
if (sigmas_str == "qwen-image-2.1-turbo") {
sigmas_str = "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0";
+ } else if (sigmas_str == "p-h3-image") {
+ sigmas_str = "1.0,0.984753,0.967905,0.951688,0.925372,0.881923,0.810083,0.528718,0.0";
}
if (!sigmas_str.empty() && sigmas_str.front() == '[') {
sigmas_str.erase(0, 1);
@@ -1777,11 +1779,11 @@ ArgOptions SDGenerationParams::get_options() {
on_scheduler_arg},
{"",
"--sigmas",
- "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo.",
+ "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image.",
on_sigmas_arg},
{"",
"--hires-sigmas",
- "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo.",
+ "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image.",
on_hires_sigmas_arg},
{"",
"--skip-layers",
diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md
index e20a2cd..efe596c 100644
--- a/examples/server/recipes/defaults.md
+++ b/examples/server/recipes/defaults.md
@@ -71,6 +71,7 @@ For community edit adapters and other optional extensions, these are the base-mo
| `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. |
| `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) |
| `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT) |
+| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. |
| `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion) |
| `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B) |
| `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
diff --git a/examples/server/recipes/p-h3-image.json b/examples/server/recipes/p-h3-image.json
new file mode 100644
index 0000000..1098bab
--- /dev/null
+++ b/examples/server/recipes/p-h3-image.json
@@ -0,0 +1,55 @@
+{
+ "schema_version": 1,
+ "id": "p-h3-image",
+ "name": "P-H3-Image",
+ "size_multiple": 32,
+ "description": "Supports text-to-image and reference-image editing with the fixed 8-step schedule. Use the P-H3 single-frame VAE and Qwen3-VL-4B with its vision tower. Supply the P-H3 text connector separately unless it is included in the text encoder.",
+ "model_url": "https://huggingface.co/PrunaAI/P-H3-Image",
+ "slots": [
+ {
+ "id": "diffusion_model",
+ "required": true
+ },
+ {
+ "id": "llm",
+ "required": true,
+ "label": "Qwen 3 VL 4B Text Encoder"
+ },
+ {
+ "id": "llm_vision",
+ "required": false,
+ "label": "Vision projector (required when not included in the text encoder)"
+ },
+ {
+ "id": "embeddings_connectors",
+ "required": false,
+ "label": "P-H3 Text Connector (required when not included in the text encoder)"
+ },
+ {
+ "id": "vae",
+ "required": true,
+ "label": "P-H3 Image VAE"
+ }
+ ],
+ "defaults": {
+ "sampling_method": "euler",
+ "cfg_scale": 1,
+ "steps": 8,
+ "width": 1024,
+ "height": 1024,
+ "custom_sigmas": [
+ 1.0,
+ 0.984753,
+ 0.967905,
+ 0.951688,
+ 0.925372,
+ 0.881923,
+ 0.810083,
+ 0.528718,
+ 0.0
+ ]
+ },
+ "features": [
+ "ref_images"
+ ]
+}
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index e3832a1..7c8b749 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -17,6 +17,7 @@
#include "model/te/llada_image_te.hpp"
#include "model/te/llm.hpp"
#include "model/te/ming_image_te.hpp"
+#include "model/te/ph3_image_connector.hpp"
#include "model/te/t5.hpp"
#include "model_loader.h"
#include "tokenizers/sensenova_u1_tokenizer.h"
@@ -1976,6 +1977,7 @@ struct LLMEmbedder : public Conditioner {
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
std::shared_ptr<T5Runner> byt5;
+ std::shared_ptr<PH3ImageTextConnectorRunner> ph3_connector;
LLMEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
@@ -2022,12 +2024,18 @@ struct LLMEmbedder : public Conditioner {
} else if (version == VERSION_MOSS_IMAGE) {
arch = LLM::LLMArch::QWEN3_5;
}
- llm = std::make_shared<LLM::LLMRunner>(arch,
- backend,
- tensor_storage_map,
- "text_encoders.llm",
- enable_vision,
- weight_manager);
+ llm = std::make_shared<LLM::LLMRunner>(arch,
+ backend,
+ tensor_storage_map,
+ "text_encoders.llm",
+ enable_vision,
+ weight_manager);
+ if (version == VERSION_P_H3_IMAGE) {
+ if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) {
+ throw std::runtime_error("P-H3-Image requires the Qwen3-VL-4B text encoder");
+ }
+ ph3_connector = std::make_shared<PH3ImageTextConnectorRunner>(backend, tensor_storage_map, weight_manager);
+ }
int pad_id = 151643;
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
pad_id = 11;
@@ -2063,6 +2071,9 @@ struct LLMEmbedder : public Conditioner {
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
llm->get_param_tensors(tensors, "text_encoders.llm");
+ if (ph3_connector) {
+ ph3_connector->get_param_tensors(tensors);
+ }
if (byt5) {
byt5->get_param_tensors(tensors, "text_encoders.t5xxl.transformer");
}
@@ -2074,6 +2085,9 @@ struct LLMEmbedder : public Conditioner {
void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
llm->set_max_graph_vram_bytes(max_vram_bytes);
+ if (ph3_connector) {
+ ph3_connector->set_max_graph_vram_bytes(max_vram_bytes);
+ }
if (byt5) {
byt5->set_max_graph_vram_bytes(max_vram_bytes);
}
@@ -2081,6 +2095,9 @@ struct LLMEmbedder : public Conditioner {
void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
llm->set_runtime_backends(backends);
+ if (ph3_connector) {
+ ph3_connector->set_runtime_backends(backends);
+ }
if (byt5) {
byt5->set_runtime_backends(backends);
}
@@ -2113,6 +2130,9 @@ struct LLMEmbedder : public Conditioner {
void set_flash_attention_enabled(bool enabled) override {
llm->set_flash_attention_enabled(enabled);
+ if (ph3_connector) {
+ ph3_connector->set_flash_attention_enabled(enabled);
+ }
if (byt5) {
byt5->set_flash_attention_enabled(enabled);
}
@@ -2120,12 +2140,18 @@ struct LLMEmbedder : public Conditioner {
void set_scale_overrides(float linear_scale, float attn_scale) override {
llm->set_scale_overrides(linear_scale, attn_scale);
+ if (ph3_connector) {
+ ph3_connector->set_scale_overrides(linear_scale, attn_scale);
+ }
if (byt5) {
byt5->set_scale_overrides(linear_scale, attn_scale);
}
}
void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+ if (ph3_connector) {
+ ph3_connector->set_weight_adapter(adapter);
+ }
if (llm) {
llm->set_weight_adapter(adapter);
}
@@ -2135,6 +2161,9 @@ struct LLMEmbedder : public Conditioner {
}
void runner_end() override {
+ if (ph3_connector) {
+ ph3_connector->runner_end();
+ }
if (llm) {
llm->runner_end();
}
@@ -2311,6 +2340,12 @@ struct LLMEmbedder : public Conditioner {
LOG_ERROR("LLM prompt encoding failed");
return {};
}
+ if (ph3_connector) {
+ hidden_states = ph3_connector->compute(n_threads, hidden_states);
+ if (hidden_states.empty()) {
+ return {};
+ }
+ }
hidden_states = apply_token_weights(std::move(hidden_states), weights);
GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx);
@@ -2457,8 +2492,9 @@ struct LLMEmbedder : public Conditioner {
if (sd_version_is_minimax_h3(version)) {
prompt_template_encode_start_idx = 0;
- out_layers = {50};
- prompt_attn_range = {0, 0};
+ // The final tap is normalized; LLMRunner exposes it at num_layers + 1.
+ out_layers = version == VERSION_P_H3_IMAGE ? std::set<int>{13, 19, 25, 31, 37} : std::set<int>{50};
+ prompt_attn_range = {0, 0};
if (llm->enable_vision) {
const int patch_size = llm->config.vision.patch_size;
@@ -2564,7 +2600,17 @@ struct LLMEmbedder : public Conditioner {
}
}
}
- prompt += conditioner_params.text;
+ auto text = conditioner_params.text;
+ if (version == VERSION_P_H3_IMAGE && !image_embeds.empty()) {
+ // Preserve explicit picture targets and structured briefs from the upstream upsampler.
+ static const std::regex verbatim(R"(^\s*(Edit <Picture \d+>|subject_definitions:))");
+ if (!std::regex_search(text, verbatim)) {
+ text = "Edit <Picture 1>. " + text;
+ } else {
+ text = trim(text);
+ }
+ }
+ prompt += text;
} else if (sd_version_is_hunyuan_video(version)) {
prompt_template_encode_start_idx = 98;
out_layers = {26};
diff --git a/src/core/layer_split_partition.cpp b/src/core/layer_split_partition.cpp
index 6ef2435..ff8aaeb 100644
--- a/src/core/layer_split_partition.cpp
+++ b/src/core/layer_split_partition.cpp
@@ -206,7 +206,7 @@ namespace sd {
for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
ggml_backend_t backend = backend_by_segment[seg_idx];
const auto& segment = plan.segments[seg_idx];
- auto assign_node = [&](int node_index) {
+ auto assign_node = [&](int node_index) {
if (node_index < 0 || node_index >= n_nodes) {
return;
}
diff --git a/src/core/vulkan/shaders/modulate.comp b/src/core/vulkan/shaders/modulate.comp
index df07626..81f910d 100644
--- a/src/core/vulkan/shaders/modulate.comp
+++ b/src/core/vulkan/shaders/modulate.comp
@@ -1,16 +1,25 @@
#version 450
layout(local_size_x = 256) in;
-layout(binding = 0) readonly buffer X { float x[]; };
-layout(binding = 1) readonly buffer S { float scale[]; };
-layout(binding = 2) readonly buffer H { float shift[]; };
-layout(binding = 3) writeonly buffer D { float dst[]; };
+layout(binding = 0) readonly buffer X {
+ float x[];
+};
+layout(binding = 1) readonly buffer S {
+ float scale[];
+};
+layout(binding = 2) readonly buffer H {
+ float shift[];
+};
+layout(binding = 3) writeonly buffer D {
+ float dst[];
+};
layout(push_constant) uniform Parameters {
uint ne[4], scale_ne[4], shift_ne[4];
uint x_stride[4], scale_stride[4], shift_stride[4];
uint offsets[4];
uint count, has_shift;
-} p;
+}
+p;
void main() {
uint group = gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x;
diff --git a/src/core/vulkan/shaders/modulate_convrot.comp b/src/core/vulkan/shaders/modulate_convrot.comp
index 3cac110..4e85b1d 100644
--- a/src/core/vulkan/shaders/modulate_convrot.comp
+++ b/src/core/vulkan/shaders/modulate_convrot.comp
@@ -2,8 +2,8 @@
#extension GL_EXT_control_flow_attributes : require
-layout(constant_id = 0) const uint K = 4096;
-layout(constant_id = 1) const bool AFFINE = false;
+layout(constant_id = 0) const uint K = 4096;
+layout(constant_id = 1) const bool AFFINE = false;
layout(constant_id = 2) const bool HAS_SHIFT = true;
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout(binding = 0) readonly buffer X {
diff --git a/src/model.h b/src/model.h
index 12bc22e..4695f5c 100644
--- a/src/model.h
+++ b/src/model.h
@@ -70,6 +70,7 @@ enum SDVersion {
VERSION_MOSS_IMAGE,
VERSION_FIBO2,
VERSION_CHROMA,
+ VERSION_P_H3_IMAGE,
VERSION_COUNT,
};
@@ -166,7 +167,7 @@ static inline bool sd_version_is_ltxav(SDVersion version) {
}
static inline bool sd_version_is_minimax_h3(SDVersion version) {
- return version == VERSION_MINIMAX_H3;
+ return version == VERSION_MINIMAX_H3 || version == VERSION_P_H3_IMAGE;
}
static inline bool sd_version_is_wan(SDVersion version) {
diff --git a/src/model/integrations/minimax_h3.cpp b/src/model/integrations/minimax_h3.cpp
index 2569b62..35a1f67 100644
--- a/src/model/integrations/minimax_h3.cpp
+++ b/src/model/integrations/minimax_h3.cpp
@@ -40,6 +40,15 @@ const std::vector<ModelDefinition>& minimax_h3_definitions() {
definition.build = build_minimax_h3;
result.push_back(std::move(definition));
}
+ {
+ ModelDefinition definition = result.front();
+ definition.version = VERSION_P_H3_IMAGE;
+ definition.name = "P-H3-Image";
+ definition.traits.supports_video_generation = false;
+ definition.sampling.default_flow_shift = 1.f;
+ definition.ref_image_preset = "qwen";
+ result.push_back(std::move(definition));
+ }
return result;
}();
return definitions;
diff --git a/src/model/te/ph3_image_connector.hpp b/src/model/te/ph3_image_connector.hpp
new file mode 100644
index 0000000..1e74e8e
--- /dev/null
+++ b/src/model/te/ph3_image_connector.hpp
@@ -0,0 +1,70 @@
+#ifndef __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__
+#define __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__
+
+#include "core/ggml_runner.h"
+#include "model/common/ggml_block.hpp"
+
+struct PH3ImageTextConnector : public UnaryBlock {
+ static constexpr int64_t input_dim = 5 * 2560;
+ static constexpr int64_t output_dim = 5120;
+
+ PH3ImageTextConnector() {
+ blocks["proj"] = std::make_shared<Linear>(input_dim, output_dim, false, true, true);
+ }
+
+ void init_params(ggml_context* ctx,
+ const String2TensorStorage& tensor_storage_map = {},
+ const std::string prefix = "") override {
+ for (const auto* name : {"input_mean", "input_std"}) {
+ params[name] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, input_dim);
+ }
+ for (const auto* name : {"output_mean", "output_std", "sink_embedding"}) {
+ params[name] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, output_dim);
+ }
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+ auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
+ x = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, x, params["input_mean"]), params["input_std"]);
+ x = proj->forward(ctx, x);
+ x = ggml_add(ctx->ggml_ctx, ggml_mul(ctx->ggml_ctx, x, params["output_std"]), params["output_mean"]);
+ auto sink = ggml_repeat(ctx->ggml_ctx, params["sink_embedding"], ggml_ext_slice(ctx->ggml_ctx, x, 1, 0, 1));
+ if (x->ne[1] == 1) {
+ return sink;
+ }
+ return ggml_concat(ctx->ggml_ctx, sink, ggml_ext_slice(ctx->ggml_ctx, x, 1, 1, x->ne[1]), 1);
+ }
+};
+
+struct PH3ImageTextConnectorRunner : public GGMLRunner {
+ PH3ImageTextConnector model;
+ static constexpr const char* prefix = "text_encoders.ph3_image_connector";
+
+ PH3ImageTextConnectorRunner(ggml_backend_t backend,
+ const String2TensorStorage& tensor_storage_map,
+ std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+ : GGMLRunner(backend, weight_manager) {
+ model.init(params_ctx, tensor_storage_map, prefix);
+ }
+
+ std::string get_desc() override {
+ return "ph3_image_text_connector";
+ }
+
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
+ model.get_param_tensors(tensors, prefix);
+ }
+
+ sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& hidden_states) {
+ auto get_graph = [&]() {
+ auto gf = ggml_new_graph(compute_ctx);
+ auto x = make_input(hidden_states);
+ auto runner_ctx = get_context();
+ ggml_build_forward_expand(gf, model.forward(&runner_ctx, x));
+ return gf;
+ };
+ return take_or_empty(GGMLRunner::compute(get_graph, n_threads, false));
+ }
+};
+
+#endif // __SD_MODEL_TE_PH3_IMAGE_CONNECTOR_HPP__
diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp
index 1e06558..d5bb01a 100644
--- a/src/model/vae/minimax_h3_vae.hpp
+++ b/src/model/vae/minimax_h3_vae.hpp
@@ -10,6 +10,7 @@
#include <utility>
#include <vector>
+#include "core/rng_mt19937.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/dit.hpp"
#include "model/vae/vae.hpp"
@@ -18,6 +19,34 @@ namespace MiniMaxH3VAE {
constexpr int H3_VIDEO_VAE_GRAPH_SIZE = 262144;
+ struct MiniMaxH3VAEConfig {
+ int num_layers = 36;
+ int patch_size_t = 4;
+
+ static MiniMaxH3VAEConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+ MiniMaxH3VAEConfig config;
+ auto proj = tensors.find(prefix + ".decoder.proj_out.weight");
+ if (proj != tensors.end()) {
+ config.patch_size_t = static_cast<int>(proj->second.ne[1] / (3 * 16 * 16));
+ if (config.patch_size_t != 1 && config.patch_size_t != 4) {
+ throw std::runtime_error("unsupported MiniMax-H3 VAE temporal patch size");
+ }
+ int layers = 0;
+ const auto layer_prefix = prefix + ".decoder.transformer_blocks.";
+ for (const auto& [name, _] : tensors) {
+ if (starts_with(name, layer_prefix)) {
+ layers = std::max(layers, std::atoi(name.c_str() + layer_prefix.size()) + 1);
+ }
+ }
+ if (layers > 0) {
+ config.num_layers = layers;
+ }
+ LOG_VERBOSE("minimax_h3_vae: layers=%d, patch_size_t=%d", config.num_layers, config.patch_size_t);
+ }
+ return config;
+ }
+ };
+
struct CausalConv3d : public Conv3d {
std::tuple<int, int, int> temporal_padding;
@@ -361,13 +390,14 @@ namespace MiniMaxH3VAE {
};
struct Decoder : public GGMLBlock {
- static constexpr int dim = 2048;
- static constexpr int num_layers = 36;
+ static constexpr int dim = 2048;
+ int num_layers;
static constexpr int num_register_tokens = 4;
static constexpr int patch_size = 16;
- static constexpr int patch_size_t = 4;
+ int patch_size_t;
- Decoder() {
+ explicit Decoder(const MiniMaxH3VAEConfig& config)
+ : num_layers(config.num_layers), patch_size_t(config.patch_size_t) {
blocks["x_embedder"] = std::make_shared<Linear>(24, dim, true);
for (int i = 0; i < num_layers; ++i) {
blocks["transformer_blocks." + std::to_string(i)] =
@@ -389,7 +419,9 @@ namespace MiniMaxH3VAE {
GGML_TYPE_F32,
dim,
num_register_tokens);
- params["mask_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim);
+ if (patch_size_t != 1) {
+ params["mask_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim);
+ }
}
ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -443,7 +475,10 @@ namespace MiniMaxH3VAE {
};
struct MiniMaxH3VideoVAE : public GGMLBlock {
- MiniMaxH3VideoVAE() {
+ bool single_frame;
+
+ explicit MiniMaxH3VideoVAE(const MiniMaxH3VAEConfig& config)
+ : single_frame(config.patch_size_t == 1) {
blocks["encoder"] = std::make_shared<Encoder>();
blocks["quant_conv"] = std::make_shared<Conv3d>(48,
48,
@@ -451,7 +486,7 @@ namespace MiniMaxH3VAE {
blocks["post_quant_conv"] = std::make_shared<Conv3d>(24,
24,
std::tuple{1, 1, 1});
- blocks["decoder"] = std::make_shared<Decoder>();
+ blocks["decoder"] = std::make_shared<Decoder>(config);
}
ggml_tensor* encode(GGMLRunnerContext* ctx,
@@ -464,7 +499,7 @@ namespace MiniMaxH3VAE {
auto encoder = std::dynamic_pointer_cast<Encoder>(blocks["encoder"]);
auto quant = std::dynamic_pointer_cast<Conv3d>(blocks["quant_conv"]);
auto moments = quant->forward(ctx, encoder->forward(ctx, pixels));
- return ggml_ext_slice(ctx->ggml_ctx, moments, 3, 0, 24);
+ return single_frame ? moments : ggml_ext_slice(ctx->ggml_ctx, moments, 3, 0, 24);
}
ggml_tensor* decode(GGMLRunnerContext* ctx,
@@ -483,6 +518,7 @@ namespace MiniMaxH3VAE {
};
struct MiniMaxH3VideoVAERunner : public VAE {
+ MiniMaxH3VAEConfig config;
MiniMaxH3VideoVAE model;
sd::Tensor<float> pixel_mean;
sd::Tensor<float> pixel_std;
@@ -495,6 +531,8 @@ namespace MiniMaxH3VAE {
const std::string& prefix = "first_stage_model",
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: VAE(VERSION_MINIMAX_H3, backend, prefix, weight_manager),
+ config(MiniMaxH3VAEConfig::detect_from_weights(tensor_storage_map, prefix)),
+ model(config),
pixel_mean({1, 1, 1, 3}, {0.485f, 0.456f, 0.406f}),
pixel_std({1, 1, 1, 3}, {0.229f, 0.224f, 0.225f}),
latents_mean({1, 1, 1, 24},
@@ -516,12 +554,12 @@ namespace MiniMaxH3VAE {
}
std::string get_desc() override {
- return "minimax_h3_video_vae";
+ return config.patch_size_t == 1 ? "p_h3_image_vae" : "minimax_h3_video_vae";
}
int get_encoder_output_channels(int input_channels) override {
SD_UNUSED(input_channels);
- return 24;
+ return config.patch_size_t == 1 ? 48 : 24;
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
@@ -531,6 +569,15 @@ namespace MiniMaxH3VAE {
sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output,
std::shared_ptr<RNG> rng) override {
SD_UNUSED(rng);
+ if (config.patch_size_t == 1) {
+ auto mean = sd::ops::slice(vae_output, 3, 0, 24);
+ auto logvar = sd::ops::slice(vae_output, 3, 24, 48);
+ auto noise = MT19937RNG(42).randn(static_cast<uint32_t>(mean.numel()));
+ for (int64_t i = 0; i < mean.numel(); ++i) {
+ mean[i] += std::exp(0.5f * std::clamp(logvar[i], -30.f, 20.f)) * noise[i];
+ }
+ return mean;
+ }
return vae_output;
}
@@ -610,6 +657,10 @@ namespace MiniMaxH3VAE {
bool circular_y = false) override {
auto input = ensure_video_shape(x);
auto tiling = resolve_tiling_params(tiling_params);
+ if (config.patch_size_t == 1 && input.shape()[2] != 1) {
+ LOG_ERROR("P-H3-Image VAE encodes single-frame images only");
+ return {};
+ }
if (input.shape()[2] == 1) {
auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y);
if (!encoded.empty() && encoded.shape()[2] > 1) {
@@ -654,6 +705,10 @@ namespace MiniMaxH3VAE {
bool silent = false) override {
auto input = ensure_video_shape(x);
auto tiling = resolve_tiling_params(tiling_params);
+ if (config.patch_size_t == 1 && input.shape()[2] != 1) {
+ LOG_ERROR("P-H3-Image VAE decodes single-frame latents only");
+ return {};
+ }
if (input.shape()[2] == 1) {
auto decoded = VAE::decode(n_threads,
input,
@@ -752,7 +807,9 @@ namespace MiniMaxH3VAE {
ids.reserve(static_cast<size_t>(width * height * num_frames + 5));
constexpr float two_pi = 6.28318530717958647692f;
for (int64_t t = 0; t < num_frames; ++t) {
- float pt = (2.f * ((t + 0.5f) / num_frames) - 1.f) * two_pi;
+ float pt = (config.patch_size_t == 1 ? 2.f * (0.5f / 7.f) - 1.f
+ : 2.f * ((t + 0.5f) / num_frames) - 1.f) *
+ two_pi;
for (int64_t h = 0; h < height; ++h) {
float ph = (2.f * ((h + 0.5f) / height) - 1.f) * two_pi;
for (int64_t w = 0; w < width; ++w) {
diff --git a/src/model_loader.cpp b/src/model_loader.cpp
index 5576067..407885b 100644
--- a/src/model_loader.cpp
+++ b/src/model_loader.cpp
@@ -578,6 +578,25 @@ SDVersion ModelLoader::get_sd_version() const {
}
if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&
tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) {
+ // P-H3 uses the same DiT architecture as H3; its VAE or connector identifies the image pipeline.
+ auto decoder = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight");
+ if (decoder == tensor_storage_map.end()) {
+ decoder = tensor_storage_map.find("vae.decoder.proj_out.weight");
+ }
+ if (decoder != tensor_storage_map.end() && decoder->second.ne[1] == 768) {
+ return VERSION_P_H3_IMAGE;
+ }
+ for (const std::string prefix : {"", "embeddings_connectors."}) {
+ if (tensor_storage_map.count(prefix + "text_encoders.ph3_image_connector.proj.weight") ||
+ tensor_storage_map.count(prefix + "text_encoders.llm.text_connector.proj.weight")) {
+ return VERSION_P_H3_IMAGE;
+ }
+ }
+ auto connector = tensor_storage_map.find("embeddings_connectors.proj.weight");
+ if (connector != tensor_storage_map.end() && connector->second.ne[0] == 12800 &&
+ connector->second.ne[1] == 5120 && tensor_storage_map.count("embeddings_connectors.sink_embedding")) {
+ return VERSION_P_H3_IMAGE;
+ }
return VERSION_MINIMAX_H3;
}
if (name == "model.diffusion_model.y_embedder.layer_pool.weight" &&
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index 57a41c5..2e14734 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -1699,12 +1699,19 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
}
}
+ const std::vector<std::pair<std::string, std::string>> connector_prefix_map = {
+ {"embeddings_connectors.text_encoders.", "text_encoders."},
+ {"embeddings_connectors.", version == VERSION_P_H3_IMAGE ? "text_encoders.ph3_image_connector." : ""},
+ };
+ replace_with_prefix_map(name, connector_prefix_map);
+
std::unordered_map<std::string, std::string> prefix_map = {
{"diffusion_model.", "model.diffusion_model."},
{"unet.", "model.diffusion_model."},
{"transformer.", "model.diffusion_model."}, // dit
{"vae.", "first_stage_model."},
{"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."},
+ {"text_encoders.llm.text_connector.", "text_encoders.ph3_image_connector."},
{"text_encoder.", "cond_stage_model.transformer."},
{"te.", "cond_stage_model.transformer."},
{"text_encoder.2.", "cond_stage_model.1.transformer."},
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
index d161d9e..98e80e1 100644
--- a/src/pipeline/diffusion_engine.cpp
+++ b/src/pipeline/diffusion_engine.cpp
@@ -778,7 +778,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) {
LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path);
- if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) {
+ if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path, "embeddings_connectors.")) {
LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path);
}
}
@@ -1491,7 +1491,7 @@ bool StableDiffusionGGML::build_denoiser() {
}
case H3_AV_FLOW_PRED: {
LOG_INFO("running in MiniMax H3 AV FLOW mode");
- denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
+ denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, version == VERSION_P_H3_IMAGE ? 1.f : 3.f, get_latent_channel());
break;
}
case MING_IMAGE_FLOW_PRED: {
@@ -2594,9 +2594,9 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
condition.c_position_ids.empty() ? nullptr : &condition.c_position_ids,
condition.c_ref_audios.empty() ? nullptr : &condition.c_ref_audios,
condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
- audio_length,
+ version == VERSION_P_H3_IMAGE ? 2 : audio_length,
std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
- 3.f};
+ version == VERSION_P_H3_IMAGE ? 1.f : 3.f};
} else if (sd_version_is_ltxav(version)) {
diffusion_params.extra = LTXAVDiffusionExtra{
nullptr,
@@ -3007,7 +3007,10 @@ sd::Tensor<float> StableDiffusionGGML::decode_first_stage(const sd::Tensor<float
if (sd_version_is_pid(version) || sd_version_is_minit2i(version) || version == VERSION_IRIS) {
return sd::ops::clamp((x + 1.f) * 0.5f, 0.0f, 1.0f);
}
- auto latents = first_stage_model->diffusion_to_vae_latents(x);
+ auto image_latents = version == VERSION_P_H3_IMAGE && x.dim() == 5
+ ? sd::ops::slice(x, 3, 0, get_latent_channel())
+ : x;
+ auto latents = first_stage_model->diffusion_to_vae_latents(image_latents);
auto tiling_params = first_stage_model->resolve_tiling_params(vae_tiling_params);
const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 &&
first_stage_model->can_temporal_tile_decode();
@@ -3021,6 +3024,9 @@ sd::Tensor<float> StableDiffusionGGML::decode_first_stage(const sd::Tensor<float
return {};
}
auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y);
+ if (version == VERSION_P_H3_IMAGE && !decoded.empty() && decoded.dim() == 5) {
+ decoded = decoded.squeeze(2);
+ }
if (!decoded.empty() ||
!sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling,
first_stage_model->last_compute_status(),
@@ -3220,7 +3226,7 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
if (params.force_ref_timestep_zero && !sd_version_is_krea2(version)) {
LOG_WARN("force_ref_timestep_zero is only supported by Krea2 architecture for now");
}
- if (sd_version_is_minimax_h3(version)) {
+ if (version == VERSION_MINIMAX_H3) {
// H3 consumes references as VAE latents in its own packed sequence rather than
// through the generic conditioner/DiT reference path, so ref_image_size is the
// only arg it acts on. Everything else is reported as ineffective rather than
diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp
index 607ec18..0763e38 100644
--- a/src/pipeline/image.cpp
+++ b/src/pipeline/image.cpp
@@ -144,6 +144,34 @@ namespace sd::pipeline {
return padded;
}
+ static sd::Tensor<float> resize_ph3_reference(const sd::Tensor<float>& image, int64_t pixels) {
+ double scale = std::min(1.0, std::sqrt(static_cast<double>(pixels) / (image.shape()[0] * image.shape()[1])));
+ int64_t width = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[0] * scale / 32)) * 32);
+ int64_t height = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[1] * scale / 32)) * 32);
+ if (width == image.shape()[0] && height == image.shape()[1]) {
+ return image;
+ }
+ return sd::ops::interpolate(image, {width, height, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Lanczos);
+ }
+
+ static sd::Tensor<float> pack_ph3_image_latent(const sd::Tensor<float>& image,
+ const sd::Tensor<float>& previous = {}) {
+ auto video = image.dim() == 4 ? image.unsqueeze(2) : image;
+ GGML_ASSERT(video.dim() == 5 && video.shape()[2] == 1 && video.shape()[3] == 24 && video.shape()[4] == 1);
+ const int64_t spatial = video.shape()[0] * video.shape()[1];
+ // The image DiT still attends to two stereo audio latents (32 channels).
+ auto shape = video.shape();
+ shape[3] += (128 + spatial - 1) / spatial;
+ auto packed = sd::zeros<float>(shape);
+ std::copy_n(video.data(), video.numel(), packed.data());
+ if (!previous.empty()) {
+ const int64_t offset = previous.shape()[0] * previous.shape()[1] * 24;
+ GGML_ASSERT(previous.numel() >= offset + 128);
+ std::copy_n(previous.data() + offset, 128, packed.data() + video.numel());
+ }
+ return packed;
+ }
+
static std::optional<ImageGenerationLatents> prepare_image_generation_latents(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
GenerationRequest* request,
@@ -323,7 +351,10 @@ namespace sd::pipeline {
continue;
}
sd::Tensor<float> ref_latent;
- if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) {
+ if (sd->version == VERSION_P_H3_IMAGE) {
+ auto resized = resize_ph3_reference(ref_images[i], int64_t(request->width) * request->height);
+ ref_latent = sd->encode_first_stage(resized);
+ } else if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) {
LOG_VERBOSE("auto resize ref images");
double vae_width;
double vae_height;
@@ -450,6 +481,16 @@ namespace sd::pipeline {
}
ImageGenerationLatents latents;
+ if (sd->version == VERSION_P_H3_IMAGE) {
+ init_latent = pack_ph3_image_latent(init_latent);
+ latent_mask = latent_mask.unsqueeze(2);
+ for (size_t i = 0; i < ref_latents.size(); ++i) {
+ latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE, static_cast<int32_t>(i), -1});
+ }
+ for (auto& image : ref_images) {
+ image = resize_ph3_reference(image, 1000000);
+ }
+ }
latents.init_latent = std::move(init_latent);
latents.concat_latent = std::move(concat_latent);
latents.img_uncond_concat_latent = std::move(img_uncond_concat_latent);
@@ -502,6 +543,9 @@ namespace sd::pipeline {
if (cond.c_concat.empty() && ref_image_params.pass_to_dit) {
cond.c_concat = latents->concat_latent; // TODO: optimize
}
+ if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) {
+ cond.c_reference_blocks = latents->minimax_reference_blocks;
+ }
bool use_ref_latent_img_cfg = request->use_img_uncond &&
!latents->ref_images.empty() &&
@@ -546,6 +590,9 @@ namespace sd::pipeline {
if (uncond.c_concat.empty() && ref_image_params.pass_to_dit) {
uncond.c_concat = latents->concat_latent; // TODO: optimize
}
+ if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) {
+ uncond.c_reference_blocks = latents->minimax_reference_blocks;
+ }
}
SDCondition img_uncond;
@@ -1047,13 +1094,19 @@ namespace sd::pipeline {
sd->rng->manual_seed(cur_seed);
sd->sampler_rng->manual_seed(cur_seed);
+ auto hires_source = sd->version == VERSION_P_H3_IMAGE
+ ? sd::ops::slice(final_latents[b], 3, 0, 24)
+ : final_latents[b];
sd::Tensor<float> upscaled = upscale_hires_latent(sd,
- final_latents[b],
+ hires_source,
request,
hires_upscaler.get());
if (upscaled.empty()) {
return false;
}
+ if (sd->version == VERSION_P_H3_IMAGE) {
+ upscaled = pack_ph3_image_latent(upscaled, final_latents[b]);
+ }
sd::Tensor<float> noise = sd::randn_like<float>(upscaled, sd->rng);
diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp
index 8b85f9b..e78a0d6 100644
--- a/src/pipeline/model_builders.cpp
+++ b/src/pipeline/model_builders.cpp
@@ -110,7 +110,7 @@ namespace sd::model_builders {
return false;
}
- bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) {
+ bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) try {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
@@ -159,6 +159,13 @@ namespace sd::model_builders {
version,
weight_manager);
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::MINIMAX_H3) {
+ if (version == VERSION_P_H3_IMAGE) {
+ // The video VAE also loads successfully, but uses different temporal decoding and posterior sampling.
+ auto proj = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight");
+ if (proj == tensor_storage_map.end() || proj->second.ne[1] != 768) {
+ throw std::runtime_error("P-H3-Image requires its single-frame VAE; pass --vae p-h3-image_vae_bf16.safetensors");
+ }
+ }
return std::make_shared<MiniMaxH3VAE::MiniMaxH3VideoVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
@@ -240,6 +247,9 @@ namespace sd::model_builders {
}
runners = std::move(result);
return true;
+ } catch (const std::exception& error) {
+ LOG_ERROR("failed to build VAE runners: %s", error.what());
+ return false;
}
bool build_control_net_runner(const Context& ctx, std::shared_ptr<ControlNet>& runner) {