Commit a85db96 for stable-diffusion.cpp
commit a85db96aa5ce2c72bd5aec326f6c5c857839a50a
Author: leejet <leejet714@gmail.com>
Date: Sun Oct 25 03:18:03 2026 +0800
feat: return generation metadata with core results (#2206)
diff --git a/docs/generation_metadata.md b/docs/generation_metadata.md
new file mode 100644
index 0000000..1f7fc33
--- /dev/null
+++ b/docs/generation_metadata.md
@@ -0,0 +1,165 @@
+# Generation results and metadata
+
+`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return
+owned results containing the media and its generation record. This is a breaking
+C API change: applications and bindings must update their calls and rebuild.
+
+```c
+sd_img_gen_params_t params;
+sd_img_gen_params_init(¶ms);
+params.prompt = "a small house";
+
+sd_image_generation_result_t* result = NULL;
+if (generate_image(ctx, ¶ms, &result)) {
+ for (int i = 0; i < result->count; ++i) {
+ const sd_image_result_t* item = &result->items[i];
+ /* Consume item->image and item->generation_info together. */
+ }
+ free_sd_image_generation_result(result);
+}
+```
+
+`result_out` is required and is set to `NULL` on failure. A successful result
+owns its pixels and UTF-8 JSON strings. The video result additionally owns its
+audio and contains the effective `fps` and `frame_count`. Results remain valid
+after freeing the generation context. Use the matching library result destructor;
+do not separately free fields unless transferring ownership and clearing them.
+Both result destructors accept `NULL`.
+
+## Reproduction parameters
+
+The `sdcpp.generation/v1` JSON is a snapshot of generation parameters for the
+same loaded context and the same input images, masks, reference media and other
+assets. It is created after `GenerationRequest` resolves the request, before
+preprocessing or sampling. A failed call returns no result or metadata.
+
+Fields follow the C generation parameter structs: `prompt`, `negative_prompt`,
+`width`, `height`, `seed`, `sample_params`, `hires`, `cache`,
+`vae_tiling_params`, `image_preprocess`, and the image/video-specific options.
+`kind` distinguishes image and video parameters. There is no inferred `task`.
+Enums use their API string names. Seeds are decimal strings to preserve their
+full integer range. Omitted optional fields retain the parameter initializer's
+defaults, including automatic values represented by infinity in the C API.
+
+Disabled optional features are omitted entirely: `cache`, `hires`, VAE tiling,
+zero/nonpositive `shifted_timestep`, inactive SLG, false circular axes, and empty
+preprocessing/reference arguments. SLG layers remain present when `slg_uncond`
+is enabled even with zero SLG scale. Enabled caches store their mode's parameters;
+TaylorSeer retains the shared DBCache thresholds and limits used by the runtime.
+Image CFG at its neutral value of 1 is omitted and resolves back to 1 by default.
+
+Image strength, ControlNet/IP-Adapter strength and PhotoMaker settings are stored
+when their corresponding input images are supplied. PuLID requires an embedding
+path. Layer count is stored for layered generation; video VACE and high-noise
+settings require those models. High-noise settings omit scheduler, custom sigmas,
+flow shift and timestep shifting because that pass shares the base configuration.
+Explicit nonempty argument strings are retained without rewriting their contents.
+Core sampling values such as seed, sampler, scheduler, steps and guidance remain
+explicit even when equal to their defaults. ADetailer always retains its inpaint
+strength because its input crops are created later.
+Model-required VAE tiling is resolved before serialization, so forced tiling
+(such as MiniMax-H3) remains present even if the caller did not enable it.
+
+The request fixes the seed, dimensions, clip skip, guidance, sampler, scheduler
+and eta before taking the snapshot. `custom_sigmas` is stored only when supplied
+as an input. Automatically computed schedules, preprocessing transforms,
+conditioning/cache decisions, detector regions and output statistics are not
+stored: they are recomputed from the same parameters and inputs.
+
+Each image records the seed used to sample its latent. `batch_count` is omitted;
+replay image parameters with `batch_count = 1`. Layers or frames decoded from
+the same latent share its seed, so layered generation reproduces that group of
+layers. Video frames share the video request. Partial batch cancellation returns
+metadata only for completed outputs. Full cancellation returns no result.
+
+The current image pipeline prepares input latents once using the starting seed
+and shares them across a batch. With a stochastic VAE encoder, replaying a later
+image alone using its sampling seed can change those input latents. Per-image
+seeds therefore do not guarantee independent reproduction for these paths.
+
+To replay a record, initialize the matching C parameter struct, populate its
+fields from the JSON, restore the same input buffers, and call generation with
+the same context. Keep custom-sigma/SLG arrays and string storage alive for the
+call. Paths supplied as generation parameters (LoRAs, hires models and identity
+embeddings) are retained. `models` identifies successfully loaded context model
+files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using
+filenames with extensions and omitting directories. Optional components disabled
+for the request and preview-only TAE are omitted. WebUI text includes `Model`
+(the diffusion model when supplied separately, otherwise the checkpoint, without
+the extension) and `VAE` for a separate active VAE. These names are identifiers,
+not hashes or sufficient configuration to rebuild a context; replay still requires
+the same model files and context settings. The JSON does not embed media buffers
+or model weights.
+
+When `--lora-model-dir` is configured, CLI/server output records express LoRA
+paths relative to that directory, preserving subdirectories and file extensions
+and using `/` separators. This applies to file metadata, API responses and
+ADetailer records. Resolve these paths against the same LoRA directory before
+passing them back to the C API. Core results retain the supplied loading paths.
+Files outside the directory retain `..` components; paths on another volume,
+or records without a configured LoRA directory, retain their original paths.
+
+## Postprocessing
+
+Upscale records `upscale.upscale_factor`. ADetailer records `adetailer` and
+`inpaint_params`, with a concrete starting seed chosen before detection. These
+operations require the same upscaler/detector context and source image on replay.
+They do not store detection results or per-region generation traces.
+
+The CLI/server append operation parameter snapshots to `postprocess` in call
+order. The source generation parameters remain unchanged. Frame postprocessing
+uses the same operation parameters for each video frame. Standalone
+postprocessing does not invent source diffusion parameters for an input image.
+
+## File metadata
+
+Image encoders accept the native record and derive WebUI infotext from it.
+Images use the existing metadata locations and combined text layout:
+`<WebUI infotext>, SDCPP: <native JSON>`. Standalone postprocessing without
+source diffusion parameters stores only native JSON. Video metadata stores
+only native JSON.
+
+| Container | Metadata location |
+| --- | --- |
+| PNG | `tEXt`, keyword `parameters` |
+| JPEG | COM segment, `parameters` label followed by a NUL byte |
+| WebP image / animated WebP | XMP property `sdcpp:parameters` |
+| AVI | RIFF INFO `ICMT` |
+| WebM | `COMMENT` tag |
+
+Inspect images or videos with `sd-cli -M metadata --image <file>`; the same
+command supports all containers above, with `--metadata-format json` for a
+structured report. Video metadata inspection does not require a model or video
+decoder.
+Matroska containers are also supported, including `.webm` outputs whose audio
+codec causes the muxer to select the Matroska document type.
+
+The XMP namespace is `https://github.com/leejet/stable-diffusion.cpp/ns/1.0/`.
+PNG and JPEG retain the existing UTF-8 text payload convention. Memory encoding
+and file saving use the same implementation. The core result and API
+`generation_info` fields remain pure JSON; the combined layout applies only
+to embedded image metadata.
+
+WebUI conversion is lossy: known samplers and schedules have explicit name
+mappings; unsupported ones use `SDCPP Sampler`/`SDCPP Schedule` fields. Hires
+keeps the base request size and separate configured hires size/steps. The native
+record retains the full generation parameters. Standalone postprocessing has no WebUI
+infotext because its original diffusion settings are unknown.
+
+WebUI prompt text appends ordinary LoRAs as `<lora:filename_without_extension:weight>`.
+Directory components are omitted to match WebUI's filename lookup; the native
+`loras` array retains relative paths and weights. High-noise-only LoRAs and names
+that cannot be represented by this tag syntax remain in the native record only.
+The native prompt is not modified, and postprocessing LoRAs are not added to the
+source generation prompt.
+
+JPEG requires the COM payload, including the label and terminating NUL bytes,
+to fit within 65,533 bytes. Larger records fail encoding with an error rather
+than being truncated. Use PNG or WebP for larger records.
+
+Disabling file metadata omits both image representations, but does not disable
+the core generation record or its API response. Native async image results and
+OpenAI image response items contain `generation_info`; video results contain a
+sequence-level `generation_info`. The SD API `info` object contains per-output
+`generation_info` alongside derived `infotexts` and each image's seed. Its outer
+`parameters` field continues to echo the request.
diff --git a/docs/image_preprocessing.md b/docs/image_preprocessing.md
index 1ca6460..8ec236e 100644
--- a/docs/image_preprocessing.md
+++ b/docs/image_preprocessing.md
@@ -149,12 +149,12 @@ an array of rule strings:
OpenAI-compatible requests accept it through
`<sd_cpp_extra_args>{...}</sd_cpp_extra_args>` in the prompt.
Request rules replace server-default rules. Generation metadata records the
-user rules; image encodings and channel conventions are unchanged.
+user rules for replay with the same input images; image encodings and channel conventions are unchanged.
## C API
Set `image_preprocess` on the existing image/video generation parameters.
-The `generate_image()` and `generate_video()` signatures are unchanged:
+Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md):
```c
sd_img_gen_params_t params;
@@ -162,7 +162,10 @@ sd_img_gen_params_init(¶ms);
/* Set prompt, original-resolution input images, and generation options. */
params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;"
"target=mask,filter=nearest-exact";
-bool ok = generate_image(ctx, ¶ms, &images, &count);
+sd_image_generation_result_t* result = NULL;
+bool ok = generate_image(ctx, ¶ms, &result);
+/* Consume result->items and their generation_info on success. */
+free_sd_image_generation_result(result);
```
Both generation parameter initializers set `image_preprocess.rules` to `NULL`,
diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt
index 6b1266c..90164cf 100644
--- a/examples/cli/CMakeLists.txt
+++ b/examples/cli/CMakeLists.txt
@@ -2,8 +2,10 @@ set(TARGET sd-cli)
add_executable(${TARGET}
../common/common.cpp
+ ../common/generation_metadata.cpp
../common/log.cpp
../common/media_io.cpp
+ ../common/media_metadata.cpp
image_metadata.cpp
main.cpp
)
diff --git a/examples/cli/README.md b/examples/cli/README.md
index bf9de18..5789455 100644
--- a/examples/cli/README.md
+++ b/examples/cli/README.md
@@ -31,14 +31,21 @@ rule for edge detection. See
[Image preprocessing](../../docs/image_preprocessing.md) for input selectors,
input defaults, downstream model processing, mask alignment, and examples.
-Metadata mode inspects PNG/JPEG container metadata without loading any model:
+Metadata mode inspects PNG, JPEG, WebP (including animated WebP), AVI, WebM, and Matroska
+container metadata without loading any model. Use `--image` for both images and videos:
```bash
./bin/sd-cli -M metadata --image ./output.png
./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json
+./bin/sd-cli -M metadata --image ./output.webm
+./bin/sd-cli -M metadata --image ./output.avi --metadata-format json
./bin/sd-cli -M metadata --image ./output.png --metadata-raw
./bin/sd-cli -M metadata --image ./output.png --metadata-all
```
For completely black or white images or videos, NaNs, and the `--linear-scale` /
`--attn-scale` workaround, see [Troubleshooting](../../docs/troubleshooting.md).
+
+Generation outputs embed the core [generation record](../../docs/generation_metadata.md).
+Images retain the native JSON and an additional WebUI-compatible infotext.
+`--disable-image-metadata` omits both from files.
diff --git a/examples/cli/image_metadata.cpp b/examples/cli/image_metadata.cpp
index ba0d6da..bb5ab7c 100644
--- a/examples/cli/image_metadata.cpp
+++ b/examples/cli/image_metadata.cpp
@@ -2,12 +2,14 @@
#include <algorithm>
#include <cctype>
+#include <codecvt>
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
+#include <locale>
#include <map>
#include <set>
#include <sstream>
@@ -391,8 +393,39 @@ namespace {
json& report,
std::string& error);
+ json parse_exif_user_comment(const uint8_t* data, size_t size) {
+ if (size >= 8 && memcmp(data, "ASCII\0\0\0", 8) == 0) {
+ return trim_trailing_nuls(bytes_to_string(data + 8, data + size));
+ }
+ if (size < 8 || memcmp(data, "UNICODE\0", 8) != 0) {
+ return json{{"raw_hex_preview", hex_preview(data, size)}};
+ }
+ size_t offset = 8;
+ // WebUI's piexif writer uses big-endian UTF-16 without a BOM.
+ bool little_endian = false;
+ if (size >= 10 && ((data[8] == 0xfe && data[9] == 0xff) || (data[8] == 0xff && data[9] == 0xfe))) {
+ little_endian = data[8] == 0xff;
+ offset = 10;
+ }
+ if ((size - offset) % 2 != 0) {
+ return json{{"error", "truncated UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}};
+ }
+ std::u16string text;
+ for (; offset < size; offset += 2) {
+ text.push_back(static_cast<char16_t>(little_endian ? data[offset] | (uint16_t(data[offset + 1]) << 8)
+ : (uint16_t(data[offset]) << 8) | data[offset + 1]));
+ }
+ try {
+ std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+ return trim_trailing_nuls(converter.to_bytes(text));
+ } catch (const std::range_error&) {
+ return json{{"error", "invalid UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}};
+ }
+ }
+
json parse_exif_value(const std::vector<uint8_t>& data,
size_t value_offset,
+ uint16_t tag,
uint16_t type,
uint32_t count,
bool little_endian,
@@ -466,8 +499,9 @@ namespace {
break;
}
case 7:
- value = bytes_to_string(data.data() + value_offset,
- data.data() + value_offset + count);
+ value = tag == 0x9286 ? parse_exif_user_comment(data.data() + value_offset, count)
+ : json(bytes_to_string(data.data() + value_offset,
+ data.data() + value_offset + count));
break;
case 9: {
if (count == 1) {
@@ -591,6 +625,7 @@ namespace {
tag_json["value"] = parse_exif_value(data,
value_offset,
+ tag,
type,
value_count,
little_endian,
@@ -1098,6 +1133,178 @@ namespace {
return true;
}
+ static bool parse_avi_chunks(const std::vector<uint8_t>& data, size_t offset, size_t end, bool info, bool include_raw, json& result, std::string& error) {
+ while (offset < end) {
+ if (end - offset < 8) {
+ error = "truncated AVI chunk header";
+ return false;
+ }
+ const auto name = bytes_to_string(data.data() + offset, data.data() + offset + 4);
+ const size_t size = read_u32_le(data, offset + 4);
+ offset += 8;
+ if (size > end - offset || (size & 1u) > end - offset - size) {
+ error = "AVI chunk exceeds its container";
+ return false;
+ }
+ json entry = {{"entry_type", "chunk"}, {"name", name}, {"length", size}, {"metadata_like", info}};
+ if (name == "LIST") {
+ if (size < 4) {
+ error = "truncated AVI LIST type";
+ return false;
+ }
+ const auto type = bytes_to_string(data.data() + offset, data.data() + offset + 4);
+ entry["data"]["list_type"] = type;
+ if (type == "INFO" && !info &&
+ !parse_avi_chunks(data, offset + 4, offset + size, true, include_raw, result, error))
+ return false;
+ } else if (info) {
+ entry["data"]["text"] = trim_trailing_nuls(bytes_to_string(data.data() + offset, data.data() + offset + size));
+ } else {
+ append_raw_preview(entry, data.data() + offset, size, include_raw);
+ }
+ result["entries"].push_back(std::move(entry));
+ offset += size + (size & 1u);
+ }
+ return true;
+ }
+
+ static bool parse_avi(const std::vector<uint8_t>& data, bool include_raw, json& result, std::string& error) {
+ result = {{"format", "AVI"}, {"entries", json::array()}};
+ size_t offset = 0;
+ while (offset < data.size()) {
+ if (data.size() - offset < 12 || memcmp(data.data() + offset, "RIFF", 4) != 0 ||
+ memcmp(data.data() + offset + 8, offset == 0 ? "AVI " : "AVIX", 4) != 0) {
+ error = "invalid AVI RIFF header";
+ return false;
+ }
+ const size_t size = read_u32_le(data, offset + 4);
+ if (size < 4 || size > data.size() - offset - 8 ||
+ (size & 1u) > data.size() - offset - 8 - size) {
+ error = "AVI RIFF size exceeds file size";
+ return false;
+ }
+ if (!parse_avi_chunks(data, offset + 12, offset + 8 + size, false, include_raw, result, error))
+ return false;
+ offset += 8 + size + (size & 1u);
+ }
+ return true;
+ }
+
+ static bool read_ebml_uint(const std::vector<uint8_t>& data, size_t& offset, size_t end, bool id, uint64_t& value, bool& unknown) {
+ if (offset == end || data[offset] == 0)
+ return false;
+ uint8_t mask = 0x80;
+ size_t length = 1;
+ while (!(data[offset] & mask)) {
+ mask >>= 1;
+ ++length;
+ }
+ if (length > (id ? 4u : 8u) || length > end - offset)
+ return false;
+ value = data[offset++] & (id ? 0xff : mask - 1);
+ for (size_t i = 1; i < length; ++i)
+ value = (value << 8) | data[offset++];
+ unknown = !id && value == (uint64_t(1) << (7 * length)) - 1;
+ return true;
+ }
+
+ static bool is_webm_segment_child(uint64_t id) {
+ return id == 0x114D9B74 || id == 0x1549A966 || id == 0x1654AE6B ||
+ id == 0x1F43B675 || id == 0x1C53BB6B || id == 0x1941A469 ||
+ id == 0x1043A770 || id == 0x1254C367;
+ }
+
+ static bool parse_webm_elements(const std::vector<uint8_t>& data, size_t& offset, size_t end, uint64_t parent, bool unknown_cluster, int depth, bool include_raw, json& result, std::string& error) {
+ if (depth > 16) {
+ error = "WebM tag nesting exceeds limit";
+ return false;
+ }
+ std::string tag_name;
+ std::string tag_text;
+ bool has_text = false;
+ while (offset < end) {
+ const auto start = offset;
+ uint64_t id, size;
+ bool unknown;
+ if (!read_ebml_uint(data, offset, end, true, id, unknown)) {
+ error = "invalid or truncated WebM element ID";
+ return false;
+ }
+ // An unknown-size Cluster ends at the next Segment-level element.
+ if (unknown_cluster && is_webm_segment_child(id)) {
+ offset = start;
+ return true;
+ }
+ if (!read_ebml_uint(data, offset, end, false, size, unknown) ||
+ (!unknown && size > end - offset)) {
+ error = "invalid or truncated WebM element size";
+ return false;
+ }
+ if (unknown && id != 0x18538067 && id != 0x1F43B675) {
+ error = "unsupported unknown-size WebM element";
+ return false;
+ }
+ const size_t stop = unknown ? end : offset + static_cast<size_t>(size);
+ const bool container = (parent == 0 && (id == 0x1A45DFA3 || id == 0x18538067)) ||
+ (parent == 0x18538067 && (id == 0x1254C367 || (id == 0x1F43B675 && unknown))) ||
+ (parent == 0x1254C367 && id == 0x7373) ||
+ ((parent == 0x7373 || parent == 0x67C8) && id == 0x67C8);
+ if (parent == 0 && id == 0x18538067)
+ result["has_segment"] = true;
+ if (parent == 0x1A45DFA3 && id == 0x4282) {
+ const auto type = bytes_to_string(data.data() + offset, data.data() + stop);
+ if (type != "webm" && type != "matroska") {
+ error = "EBML document is not WebM or Matroska";
+ return false;
+ }
+ result["doc_type"] = type;
+ result["format"] = type == "webm" ? "WEBM" : "MATROSKA";
+ }
+ if (parent == 0x67C8 && (id == 0x45A3 || id == 0x4487)) {
+ auto value = bytes_to_string(data.data() + offset, data.data() + stop);
+ if (id == 0x45A3)
+ tag_name = std::move(value);
+ else {
+ tag_text = std::move(value);
+ has_text = true;
+ }
+ }
+ if (parent == 0 || parent == 0x18538067) {
+ std::ostringstream name;
+ name << "0x" << std::hex << std::uppercase << id;
+ json entry = {{"entry_type", "element"}, {"name", name.str()}, {"length", stop - offset}, {"metadata_like", false}};
+ if (unknown)
+ entry["unknown_size"] = true;
+ append_raw_preview(entry, data.data() + offset, stop - offset, include_raw);
+ result["entries"].push_back(std::move(entry));
+ }
+ if (container) {
+ if (!parse_webm_elements(data, offset, stop, id, id == 0x1F43B675 && unknown,
+ depth + 1, include_raw, result, error))
+ return false;
+ } else {
+ offset = stop;
+ }
+ }
+ if (parent == 0x67C8 && has_text) {
+ result["entries"].push_back({{"entry_type", "tag"}, {"name", tag_name}, {"metadata_like", true}, {"data", {{"text", tag_text}}}});
+ }
+ return true;
+ }
+
+ static bool parse_webm(const std::vector<uint8_t>& data, bool include_raw, json& result, std::string& error) {
+ result = {{"format", "WEBM"}, {"entries", json::array()}};
+ size_t offset = 0;
+ if (!parse_webm_elements(data, offset, data.size(), 0, false, 0, include_raw, result, error))
+ return false;
+ if (!result.contains("doc_type") || !result.value("has_segment", false)) {
+ error = "missing WebM document type or Segment";
+ return false;
+ }
+ result.erase("has_segment");
+ return true;
+ }
+
std::string abbreviate(const std::string& value, bool brief) {
if (!brief || value.size() <= 240) {
return value;
@@ -1116,7 +1323,7 @@ namespace {
return;
}
if (value.is_primitive()) {
- out << prefix << key << ": " << value.dump() << "\n";
+ out << prefix << key << ": " << value.dump(-1, ' ', false, json::error_handler_t::replace) << "\n";
return;
}
if (value.is_array()) {
@@ -1125,7 +1332,7 @@ namespace {
if (item.is_string()) {
out << prefix << " - " << abbreviate(item.get<std::string>(), brief) << "\n";
} else if (item.is_primitive()) {
- out << prefix << " - " << item.dump() << "\n";
+ out << prefix << " - " << item.dump(-1, ' ', false, json::error_handler_t::replace) << "\n";
} else {
out << prefix << " -\n";
for (auto it = item.begin(); it != item.end(); ++it) {
@@ -1158,10 +1365,12 @@ namespace {
}
for (const auto& entry : entries) {
- const bool is_chunk = entry.value("entry_type", "") == "chunk";
- const std::string name = is_chunk ? entry.value("name", "unknown")
- : entry.value("marker", "unknown");
- out << (is_chunk ? "Chunk: " : "Segment: ") << name << "\n";
+ const auto type = entry.value("entry_type", "");
+ const auto name = entry.value("name", entry.value("marker", "unknown"));
+ const char* label = type == "chunk" ? "Chunk" : type == "tag" ? "Tag"
+ : type == "element" ? "Element"
+ : "Segment";
+ out << label << ": " << name << "\n";
for (auto it = entry.begin(); it != entry.end(); ++it) {
if (it.key() == "entry_type" || it.key() == "name" || it.key() == "marker" ||
it.key() == "metadata_like") {
@@ -1211,7 +1420,13 @@ namespace {
return parse_webp(data, include_raw, report, error);
}
- error = "unsupported image format; only PNG, JPEG, and WebP are supported";
+ if (data.size() >= 12 && memcmp(data.data(), "RIFF", 4) == 0 &&
+ memcmp(data.data() + 8, "AVI ", 4) == 0)
+ return parse_avi(data, include_raw, report, error);
+ if (data.size() >= 4 && read_u32_be(data, 0) == 0x1A45DFA3)
+ return parse_webm(data, include_raw, report, error);
+
+ error = "unsupported media format; only PNG, JPEG, WebP, AVI, WebM, and Matroska are supported";
return false;
}
@@ -1230,7 +1445,7 @@ bool print_image_metadata(const std::string& image_path,
if (options.output_format == MetadataOutputFormat::JSON) {
visible_report["file"] = image_path;
- out << visible_report.dump(2) << "\n";
+ out << visible_report.dump(2, ' ', false, json::error_handler_t::replace) << "\n";
} else {
print_text_report(image_path, visible_report, options.include_structural, options.brief, out);
}
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index ee1e56f..591a151 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -69,7 +69,7 @@ struct SDCliParams {
&output_path},
{"",
"--image",
- "path to the image to inspect (for metadata mode)",
+ "path to the image or video to inspect (for metadata mode)",
0,
&image_path},
{"",
@@ -227,7 +227,7 @@ struct SDCliParams {
}
} else {
if (image_path.empty()) {
- LOG_ERROR("error: metadata mode needs an image path (--image)");
+ LOG_ERROR("error: metadata mode needs an image or video path (--image)");
return false;
}
if (metadata_format != "text" && metadata_format != "json") {
@@ -441,10 +441,11 @@ static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
bool save_results(const SDCliParams& cli_params,
const SDContextParams& ctx_params,
const SDGenerationParams& gen_params,
- sd_image_t* results,
+ SDImageVec& results,
int num_results,
- const sd_audio_t* generated_audio = nullptr) {
- if (results == nullptr || num_results <= 0) {
+ const sd_audio_t* generated_audio,
+ const std::string& video_info) {
+ if (results.empty() || num_results <= 0) {
return false;
}
@@ -487,12 +488,10 @@ bool save_results(const SDCliParams& cli_params,
if (!img.data)
return false;
- int images_per_batch = gen_params.batch_count > 0 ? std::max(1, num_results / gen_params.batch_count) : 1;
- const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx / images_per_batch;
- std::string params = gen_params.embed_image_metadata
- ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
- : "";
- const bool ok = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
+ std::string params = gen_params.embed_image_metadata
+ ? results.generation_info(idx)
+ : "";
+ const bool ok = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
LOG_INFO("save result image %d to '%s' (%s)", idx, path.u8string().c_str(), ok ? "success" : "failure");
return ok;
};
@@ -534,7 +533,7 @@ bool save_results(const SDCliParams& cli_params,
if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
ext = ".avi";
std::string params = gen_params.embed_image_metadata
- ? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode)
+ ? video_info
: "";
fs::path video_path = base_path;
@@ -542,7 +541,7 @@ bool save_results(const SDCliParams& cli_params,
std::string final_ext_lower = ext.u8string();
std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
- if (create_video_from_sd_images(video_path.u8string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
+ if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
LOG_INFO("save result video to '%s'", video_path.u8string().c_str());
if (generated_audio != nullptr && !mux_audio) {
fs::path wav_path = video_path;
@@ -607,30 +606,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx,
continue;
}
sd_img_gen_params_t ad_generation_params = img_gen_params;
- ad_generation_params.seed = img_gen_params.seed + i;
+ ad_generation_params.seed = generation_seed(results.generation_info(i), img_gen_params.seed);
if (mode == IMG_GEN) {
ad_generation_params.width = 512;
ad_generation_params.height = 512;
ad_generation_params.strength = 0.4f;
}
- sd_image_t* detailed_images = nullptr;
- int detailed_count = 0;
+ sd_image_generation_result_t* detailed = nullptr;
if (!adetail_image(ad_ctx.get(),
sd_ctx,
results[i],
&ad_params,
&ad_generation_params,
- &detailed_images,
- &detailed_count) ||
- detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) {
- free_sd_images(detailed_images, detailed_count);
+ &detailed) ||
+ detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
+ free_sd_image_generation_result(detailed);
LOG_ERROR("ADetailer failed for image %d", i + 1);
return false;
}
+ results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir);
free(results[i].data);
- results[i] = detailed_images[0];
- detailed_images[0] = {0, 0, 0, nullptr};
- free_sd_images(detailed_images, detailed_count);
+ results[i] = detailed->items[0].image;
+ detailed->items[0].image = {};
+ free_sd_image_generation_result(detailed);
}
return true;
}
@@ -884,6 +882,7 @@ int main(int argc, const char* argv[]) {
SDImageVec results;
int num_results = 0;
sd_audio_t* generated_audio = nullptr;
+ std::string video_info;
if (cli_params.mode == UPSCALE) {
num_results = 1;
@@ -896,18 +895,6 @@ int main(int argc, const char* argv[]) {
return 1;
}
- if (gen_params.sample_params.sample_method == SAMPLE_METHOD_COUNT) {
- gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
- }
-
- if (gen_params.high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) {
- gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
- }
-
- if (gen_params.sample_params.scheduler == SCHEDULER_COUNT) {
- gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx.get(), gen_params.sample_params.sample_method);
- }
-
sd_img_gen_params_t img_gen_params{};
const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
if (use_img_gen_params) {
@@ -915,23 +902,24 @@ int main(int argc, const char* argv[]) {
}
if (cli_params.mode == IMG_GEN) {
- sd_image_t* generated_images = nullptr;
- if (!generate_image(sd_ctx.get(), &img_gen_params, &generated_images, &num_results)) {
- generated_images = nullptr;
- num_results = 0;
- }
- results.adopt(generated_images, num_results);
+ sd_image_generation_result_t* generated = nullptr;
+ generate_image(sd_ctx.get(), &img_gen_params, &generated);
+ results.adopt(generated, ctx_params.lora_model_dir);
+ num_results = results.count();
} else if (cli_params.mode == ADETAILER) {
num_results = 1;
results.push_back(gen_params.init_image.release());
} else if (cli_params.mode == VID_GEN) {
- sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
- sd_image_t* generated_video = nullptr;
- if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio, &cli_params.preview_fps)) {
- generated_video = nullptr;
+ sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
+ sd_video_generation_result_t* generated = nullptr;
+ if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) {
+ gen_params.fps = cli_params.preview_fps = generated->fps;
+ generated_audio = generated->audio;
+ generated->audio = nullptr;
+ num_results = generated->frame_count;
+ results.adopt_video(generated, ctx_params.lora_model_dir);
+ video_info = results.generation_info(0);
}
- gen_params.fps = cli_params.preview_fps;
- results.adopt(generated_video, num_results);
}
if (!results) {
@@ -971,21 +959,20 @@ int main(int argc, const char* argv[]) {
SDImageOwner current_image(results[i]);
results[i] = {0, 0, 0, nullptr};
for (int u = 0; u < gen_params.upscale_repeats; ++u) {
- sd_image_t* upscaled_images = nullptr;
- int upscaled_count = 0;
- bool upscale_ok = upscale(upscaler_ctx.get(),
- current_image.get(),
- upscale_factor,
- &upscaled_images,
- &upscaled_count);
- if (!upscale_ok || upscaled_count <= 0 || upscaled_images[0].data == nullptr) {
- free_sd_images(upscaled_images, upscaled_count);
+ sd_image_generation_result_t* upscaled = nullptr;
+ bool upscale_ok = upscale(upscaler_ctx.get(),
+ current_image.get(),
+ upscale_factor,
+ &upscaled);
+ if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) {
+ free_sd_image_generation_result(upscaled);
LOG_ERROR("upscale failed");
break;
}
- sd_image_t upscaled_image = upscaled_images[0];
- upscaled_images[0] = {0, 0, 0, nullptr};
- free_sd_images(upscaled_images, upscaled_count);
+ results.append_generation_info(i, upscaled->items[0].generation_info);
+ sd_image_t upscaled_image = upscaled->items[0].image;
+ upscaled->items[0].image = {};
+ free_sd_image_generation_result(upscaled);
current_image.reset(upscaled_image);
}
results[i] = current_image.release(); // Set the final upscaled image as the result
@@ -993,7 +980,9 @@ int main(int argc, const char* argv[]) {
}
}
- if (!save_results(cli_params, ctx_params, gen_params, results.data(), num_results, generated_audio)) {
+ if (!video_info.empty() && results.count() > 0)
+ video_info = results.generation_info(0);
+ if (!save_results(cli_params, ctx_params, gen_params, results, num_results, generated_audio, video_info)) {
free_sd_audio(generated_audio);
return 1;
}
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index bd25f90..00301dc 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -2548,11 +2548,6 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
return false;
}
- if (seed < 0) {
- srand((int)time(nullptr));
- seed = rand();
- }
-
if (strict) {
batch_count = std::clamp(batch_count, 1, 8);
sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100);
@@ -3029,301 +3024,3 @@ std::string SDGenerationParams::to_string() const {
std::string version_string() {
return std::string("stable-diffusion.cpp version ") + sd_version() + ", commit " + sd_commit();
}
-
-static std::string safe_json_string(const char* value) {
- return value ? value : "";
-}
-
-static void set_json_basename_if_not_empty(json& target, const char* key, const std::string& path) {
- if (!path.empty()) {
- target[key] = sd_basename(path);
- }
-}
-
-static json build_sampling_metadata_json(const sd_sample_params_t& sample_params,
- const std::vector<int>& skip_layers,
- const std::vector<float>* custom_sigmas = nullptr) {
- json sampling = {
- {"steps", sample_params.sample_steps},
- {"eta", sample_params.eta},
- {"shifted_timestep", sample_params.shifted_timestep},
- {"flow_shift", sample_params.flow_shift},
- {"extra_sample_args", safe_json_string(sample_params.extra_sample_args)},
- {"guidance",
- {
- {"txt_cfg", sample_params.guidance.txt_cfg},
- {"img_cfg", sample_params.guidance.img_cfg},
- {"distilled_guidance", sample_params.guidance.distilled_guidance},
- {"slg",
- {
- {"scale", sample_params.guidance.slg.scale},
- {"layers", skip_layers},
- {"start", sample_params.guidance.slg.layer_start},
- {"end", sample_params.guidance.slg.layer_end},
- }},
- }},
- };
- if (sample_params.sample_method != SAMPLE_METHOD_COUNT) {
- sampling["method"] = safe_json_string(sd_sample_method_name(sample_params.sample_method));
- }
- if (sample_params.scheduler != SCHEDULER_COUNT) {
- sampling["scheduler"] = safe_json_string(sd_scheduler_name(sample_params.scheduler));
- }
- if (custom_sigmas != nullptr) {
- sampling["custom_sigmas"] = *custom_sigmas;
- }
- return sampling;
-}
-
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- int64_t seed,
- SDMode mode) {
- json root;
- root["schema"] = "sdcpp.image.params/v1";
- root["mode"] = "img_gen";
- if (mode == VID_GEN) {
- root["mode"] = "vid_gen";
- } else if (mode == ADETAILER) {
- root["mode"] = "adetailer";
- }
- root["generator"] = {
- {"name", "stable-diffusion.cpp"},
- {"version", safe_json_string(sd_version())},
- {"commit", safe_json_string(sd_commit())},
- };
- root["seed"] = seed;
- root["width"] = gen_params.get_resolved_width();
- root["height"] = gen_params.get_resolved_height();
-
- root["prompt"] = {
- {"positive", gen_params.prompt},
- {"negative", gen_params.negative_prompt},
- };
- if (!gen_params.ad_model_path.empty()) {
- root["adetailer"] = {
- {"model", sd_basename(gen_params.ad_model_path)},
- {"prompt", gen_params.ad_prompt},
- {"negative_prompt", gen_params.ad_negative_prompt},
- {"extra_args", gen_params.extra_ad_args},
- };
- }
- root["sampling"] = build_sampling_metadata_json(gen_params.sample_params,
- gen_params.skip_layers,
- &gen_params.custom_sigmas);
-
- json models;
- set_json_basename_if_not_empty(models, "model", ctx_params.model_path);
- set_json_basename_if_not_empty(models, "clip_l", ctx_params.clip_l_path);
- set_json_basename_if_not_empty(models, "clip_g", ctx_params.clip_g_path);
- set_json_basename_if_not_empty(models, "clip_vision", ctx_params.clip_vision_path);
- set_json_basename_if_not_empty(models, "t5xxl", ctx_params.t5xxl_path);
- set_json_basename_if_not_empty(models, "llm", ctx_params.llm_path);
- set_json_basename_if_not_empty(models, "llm_vision", ctx_params.llm_vision_path);
- set_json_basename_if_not_empty(models, "diffusion_model", ctx_params.diffusion_model_path);
- set_json_basename_if_not_empty(models, "high_noise_diffusion_model", ctx_params.high_noise_diffusion_model_path);
- set_json_basename_if_not_empty(models, "uncond_diffusion_model", ctx_params.uncond_diffusion_model_path);
- set_json_basename_if_not_empty(models, "vae", ctx_params.vae_path);
- set_json_basename_if_not_empty(models, "taesd", ctx_params.taesd_path);
- set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path);
- root["models"] = std::move(models);
-
- root["clip_skip"] = gen_params.clip_skip;
- root["strength"] = gen_params.strength;
- root["control_strength"] = gen_params.control_strength;
- root["ip_adapter_strength"] = gen_params.ip_adapter_strength;
- root["ref_image_args"] = gen_params.ref_image_args;
- root["image_preprocess"] = gen_params.image_preprocess;
- root["increase_ref_index"] = gen_params.increase_ref_index;
- if (mode == VID_GEN) {
- root["video"] = {
- {"frame_count", gen_params.video_frames},
- {"fps", gen_params.fps},
- };
- root["moe_boundary"] = gen_params.moe_boundary;
- root["vace_strength"] = gen_params.vace_strength;
- root["high_noise_sampling"] = build_sampling_metadata_json(gen_params.high_noise_sample_params,
- gen_params.high_noise_skip_layers);
- }
-
- root["rng"] = safe_json_string(sd_rng_type_name(ctx_params.rng_type));
- if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
- root["sampler_rng"] = safe_json_string(sd_rng_type_name(ctx_params.sampler_rng_type));
- }
-
- json loras = json::array();
- for (const auto& entry : gen_params.lora_map) {
- loras.push_back({
- {"name", sd_basename(entry.first)},
- {"multiplier", entry.second},
- {"is_high_noise", false},
- });
- }
- for (const auto& entry : gen_params.high_noise_lora_map) {
- loras.push_back({
- {"name", sd_basename(entry.first)},
- {"multiplier", entry.second},
- {"is_high_noise", true},
- });
- }
- if (!loras.empty()) {
- root["loras"] = std::move(loras);
- }
-
- if (gen_params.hires_enabled) {
- root["hires"] = {
- {"enabled", gen_params.hires_enabled},
- {"upscaler", gen_params.hires_upscaler},
- {"model", gen_params.hires_upscaler_model_path.empty() ? "" : sd_basename(gen_params.hires_upscaler_model_path)},
- {"scale", gen_params.hires_scale},
- {"target_width", gen_params.hires_width},
- {"target_height", gen_params.hires_height},
- {"steps", gen_params.hires_steps},
- {"denoising_strength", gen_params.hires_denoising_strength},
- {"custom_sigmas", gen_params.hires_custom_sigmas},
- {"upscale_tile_size", gen_params.hires_upscale_tile_size},
- };
- }
-
- if (gen_params.cache_params.mode != SD_CACHE_DISABLED) {
- root["cache"] = {
- {"requested_mode", gen_params.cache_mode},
- {"requested_option", gen_params.cache_option},
- {"mode", gen_params.cache_params.mode},
- {"scm_mask", gen_params.scm_mask},
- {"scm_policy_dynamic", gen_params.scm_policy_dynamic},
- {"reuse_threshold", gen_params.cache_params.reuse_threshold},
- {"start_percent", gen_params.cache_params.start_percent},
- {"end_percent", gen_params.cache_params.end_percent},
- {"error_decay_rate", gen_params.cache_params.error_decay_rate},
- {"use_relative_threshold", gen_params.cache_params.use_relative_threshold},
- {"reset_error_on_compute", gen_params.cache_params.reset_error_on_compute},
- {"Fn_compute_blocks", gen_params.cache_params.Fn_compute_blocks},
- {"Bn_compute_blocks", gen_params.cache_params.Bn_compute_blocks},
- {"residual_diff_threshold", gen_params.cache_params.residual_diff_threshold},
- {"max_warmup_steps", gen_params.cache_params.max_warmup_steps},
- {"max_cached_steps", gen_params.cache_params.max_cached_steps},
- {"max_continuous_cached_steps", gen_params.cache_params.max_continuous_cached_steps},
- {"taylorseer_n_derivatives", gen_params.cache_params.taylorseer_n_derivatives},
- {"taylorseer_skip_interval", gen_params.cache_params.taylorseer_skip_interval},
- {"spectrum_w", gen_params.cache_params.spectrum_w},
- {"spectrum_m", gen_params.cache_params.spectrum_m},
- {"spectrum_lam", gen_params.cache_params.spectrum_lam},
- {"spectrum_window_size", gen_params.cache_params.spectrum_window_size},
- {"spectrum_flex_window", gen_params.cache_params.spectrum_flex_window},
- {"spectrum_warmup_steps", gen_params.cache_params.spectrum_warmup_steps},
- {"spectrum_stop_percent", gen_params.cache_params.spectrum_stop_percent},
- };
- }
-
- if (gen_params.vae_tiling_params.enabled ||
- gen_params.vae_tiling_params.temporal_tiling ||
- !gen_params.extra_tiling_args.empty()) {
- root["vae_tiling"] = {
- {"enabled", gen_params.vae_tiling_params.enabled},
- {"temporal_tiling", gen_params.vae_tiling_params.temporal_tiling},
- {"tile_size_w", gen_params.vae_tiling_params.tile_size_w},
- {"tile_size_h", gen_params.vae_tiling_params.tile_size_h},
- {"target_overlap", gen_params.vae_tiling_params.target_overlap},
- {"rel_size_w", gen_params.vae_tiling_params.rel_size_w},
- {"rel_size_h", gen_params.vae_tiling_params.rel_size_h},
- {"extra_tiling_args", gen_params.extra_tiling_args},
- };
- }
-
- return root.dump();
-}
-
-std::string get_image_params(const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- int64_t seed,
- SDMode mode) {
- std::string parameter_string;
- if (gen_params.prompt_with_lora.size() != 0) {
- parameter_string += gen_params.prompt_with_lora + "\n";
- } else {
- parameter_string += gen_params.prompt + "\n";
- }
- if (gen_params.negative_prompt.size() != 0) {
- parameter_string += "Negative prompt: " + gen_params.negative_prompt + "\n";
- }
- parameter_string += "Steps: " + std::to_string(gen_params.sample_params.sample_steps) + ", ";
- parameter_string += "CFG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
- if (gen_params.sample_params.guidance.slg.scale != 0 && gen_params.skip_layers.size() != 0) {
- parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.slg.scale) + ", ";
- parameter_string += "Skip layers: [";
- for (const auto& layer : gen_params.skip_layers) {
- parameter_string += std::to_string(layer) + ", ";
- }
- parameter_string += "], ";
- parameter_string += "Skip layer start: " + std::to_string(gen_params.sample_params.guidance.slg.layer_start) + ", ";
- parameter_string += "Skip layer end: " + std::to_string(gen_params.sample_params.guidance.slg.layer_end) + ", ";
- }
- parameter_string += "Guidance: " + std::to_string(gen_params.sample_params.guidance.distilled_guidance) + ", ";
- parameter_string += "Eta: " + std::to_string(gen_params.sample_params.eta) + ", ";
- if (!gen_params.extra_sample_args.empty()) {
- parameter_string += "Extra sample args: " + gen_params.extra_sample_args + ", ";
- }
- if (!gen_params.ad_model_path.empty()) {
- parameter_string += "ADetailer model: " + sd_basename(gen_params.ad_model_path) + ", ";
- if (!gen_params.ad_prompt.empty()) {
- parameter_string += "ADetailer prompt: " + gen_params.ad_prompt + ", ";
- }
- if (!gen_params.ad_negative_prompt.empty()) {
- parameter_string += "ADetailer negative prompt: " + gen_params.ad_negative_prompt + ", ";
- }
- if (!gen_params.extra_ad_args.empty()) {
- parameter_string += "ADetailer args: " + gen_params.extra_ad_args + ", ";
- }
- }
- parameter_string += "Seed: " + std::to_string(seed) + ", ";
- parameter_string += "Size: " + std::to_string(gen_params.get_resolved_width()) + "x" + std::to_string(gen_params.get_resolved_height()) + ", ";
- parameter_string += "Model: " + sd_basename(ctx_params.model_path) + ", ";
- parameter_string += "RNG: " + std::string(sd_rng_type_name(ctx_params.rng_type)) + ", ";
- if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
- parameter_string += "Sampler RNG: " + std::string(sd_rng_type_name(ctx_params.sampler_rng_type)) + ", ";
- }
- parameter_string += "Sampler: " + std::string(sd_sample_method_name(gen_params.sample_params.sample_method));
- if (!gen_params.custom_sigmas.empty()) {
- parameter_string += ", Custom Sigmas: [";
- for (size_t i = 0; i < gen_params.custom_sigmas.size(); ++i) {
- std::ostringstream oss;
- oss << std::fixed << std::setprecision(4) << gen_params.custom_sigmas[i];
- parameter_string += oss.str() + (i == gen_params.custom_sigmas.size() - 1 ? "" : ", ");
- }
- parameter_string += "]";
- } else if (gen_params.sample_params.scheduler != SCHEDULER_COUNT) { // Only show schedule if not using custom sigmas
- parameter_string += " " + std::string(sd_scheduler_name(gen_params.sample_params.scheduler));
- }
- parameter_string += ", ";
- for (const auto& te : {ctx_params.clip_l_path, ctx_params.clip_g_path, ctx_params.t5xxl_path, ctx_params.llm_path, ctx_params.llm_vision_path}) {
- if (!te.empty()) {
- parameter_string += "TE: " + sd_basename(te) + ", ";
- }
- }
- if (!ctx_params.diffusion_model_path.empty()) {
- parameter_string += "Unet: " + sd_basename(ctx_params.diffusion_model_path) + ", ";
- }
- if (!ctx_params.uncond_diffusion_model_path.empty()) {
- parameter_string += "Uncond Unet: " + sd_basename(ctx_params.uncond_diffusion_model_path) + ", ";
- }
- if (!ctx_params.vae_path.empty()) {
- parameter_string += "VAE: " + sd_basename(ctx_params.vae_path) + ", ";
- }
- if (gen_params.clip_skip != -1) {
- parameter_string += "Clip skip: " + std::to_string(gen_params.clip_skip) + ", ";
- }
- if (gen_params.hires_enabled) {
- parameter_string += "Hires upscale: " + gen_params.hires_upscaler + ", ";
- parameter_string += "Hires scale: " + std::to_string(gen_params.hires_scale) + ", ";
- parameter_string += "Hires resize: " + std::to_string(gen_params.hires_width) + "x" + std::to_string(gen_params.hires_height) + ", ";
- parameter_string += "Hires steps: " + std::to_string(gen_params.hires_steps) + ", ";
- parameter_string += "Denoising strength: " + std::to_string(gen_params.hires_denoising_strength) + ", ";
- if (!gen_params.hires_custom_sigmas.empty()) {
- parameter_string += "Hires custom sigmas: " + vec_to_string(gen_params.hires_custom_sigmas) + ", ";
- }
- }
- parameter_string += "Version: stable-diffusion.cpp";
- parameter_string += ", SDCPP: " + build_sdcpp_image_metadata_json(ctx_params, gen_params, seed, mode);
- return parameter_string;
-}
diff --git a/examples/common/common.h b/examples/common/common.h
index 7b9f691..39d6011 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -1,6 +1,8 @@
#ifndef __EXAMPLES_COMMON_COMMON_H__
#define __EXAMPLES_COMMON_COMMON_H__
+#include "generation_metadata.h"
+
#include <cmath>
#include <cstdint>
#include <functional>
@@ -338,13 +340,4 @@ struct SDGenerationParams {
};
std::string version_string();
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- int64_t seed,
- SDMode mode = IMG_GEN);
-std::string get_image_params(const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- int64_t seed,
- SDMode mode = IMG_GEN);
-
#endif // __EXAMPLES_COMMON_COMMON_H__
diff --git a/examples/common/generation_metadata.cpp b/examples/common/generation_metadata.cpp
new file mode 100644
index 0000000..989e914
--- /dev/null
+++ b/examples/common/generation_metadata.cpp
@@ -0,0 +1,155 @@
+#include "generation_metadata.h"
+
+#include <filesystem>
+#include <map>
+
+#include "json.hpp"
+
+static void relativize_lora_paths(nlohmann::json& info, const std::filesystem::path& root) {
+ if (info.contains("loras")) {
+ for (auto& lora : info["loras"]) {
+ const auto path = lora.value("path", "");
+ if (path.empty())
+ continue;
+ std::error_code error;
+ const auto absolute = std::filesystem::absolute(std::filesystem::u8path(path), error);
+ if (error)
+ continue;
+ const auto relative = absolute.lexically_normal().lexically_relative(root);
+ if (!relative.empty())
+ lora["path"] = relative.generic_u8string();
+ }
+ }
+ if (info.contains("inpaint_params"))
+ relativize_lora_paths(info["inpaint_params"], root);
+ if (info.contains("postprocess")) {
+ for (auto& operation : info["postprocess"])
+ relativize_lora_paths(operation, root);
+ }
+}
+
+std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir) {
+ if (info.empty() || lora_model_dir.empty())
+ return info;
+ std::error_code error;
+ const auto root = std::filesystem::absolute(std::filesystem::u8path(lora_model_dir), error);
+ if (error)
+ return info;
+ auto result = nlohmann::json::parse(info);
+ relativize_lora_paths(result, root.lexically_normal());
+ return result.dump();
+}
+
+std::string merge_generation_info(const std::string& source, const std::string& operation) {
+ if (source.empty())
+ return operation;
+ auto result = nlohmann::json::parse(source);
+ const auto next = nlohmann::json::parse(operation);
+ result["postprocess"].push_back(next);
+ return result.dump();
+}
+
+int64_t generation_seed(const std::string& info, int64_t fallback) {
+ if (info.empty())
+ return fallback;
+ const auto value = nlohmann::json::parse(info);
+ const auto& params = value.contains("inpaint_params") ? value.at("inpaint_params") : value;
+ return params.contains("seed") ? std::stoll(params.at("seed").get<std::string>()) : fallback;
+}
+
+std::string generation_info_to_webui(const std::string& text) {
+ using nlohmann::json;
+ if (text.empty())
+ return "";
+ const auto info = json::parse(text);
+ if (info.value("schema", "") != "sdcpp.generation/v1")
+ return "";
+ // Standalone postprocessing has no source diffusion settings to import.
+ if (!info.contains("sample_params"))
+ return "";
+ const auto& sampling = info.at("sample_params");
+ std::string result = info.value("prompt", "");
+ if (info.contains("loras")) {
+ for (const auto& lora : info.at("loras")) {
+ // WebUI tags cannot restrict a LoRA to the high-noise model.
+ if (lora.value("is_high_noise", false))
+ continue;
+ auto name = lora.at("path").get<std::string>();
+ const auto separator = name.find_last_of("/\\");
+ if (separator != std::string::npos)
+ name.erase(0, separator + 1);
+ const auto extension = name.find_last_of('.');
+ if (extension != std::string::npos && extension > 0)
+ name.erase(extension);
+ if (name.empty() || name.find_first_of(":<>\r\n") != std::string::npos)
+ continue;
+ if (!result.empty())
+ result += " ";
+ result += "<lora:" + name + ":" + lora.at("multiplier").dump() + ">";
+ }
+ }
+ const auto negative = info.value("negative_prompt", "");
+ if (!negative.empty())
+ result += "\nNegative prompt: " + negative;
+ result += "\n";
+ bool first = true;
+ const auto field = [&](const char* key, const json& value) {
+ std::string v = value.is_string() ? value.get<std::string>() : value.dump();
+ if (v.find_first_of(",\n\r:\"") != std::string::npos)
+ v = json(v).dump(-1, ' ', false);
+ if (!first)
+ result += ", ";
+ first = false;
+ result += std::string(key) + ": " + v;
+ };
+ field("Steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps"));
+ static const std::map<std::string, std::string> samplers = {
+ {"euler", "Euler"}, {"euler_a", "Euler a"}, {"heun", "Heun"}, {"dpm2", "DPM2"}, {"dpm++2s_a", "DPM++ 2S a"}, {"dpm++2m", "DPM++ 2M"}, {"dpm++2m_sde", "DPM++ 2M SDE"}, {"lcm", "LCM"}};
+ const auto method = sampling.at("sample_method").get<std::string>();
+ auto mapped = samplers.find(method);
+ field(mapped != samplers.end() ? "Sampler" : "SDCPP Sampler", mapped != samplers.end() ? mapped->second : method);
+ static const std::map<std::string, std::string> schedules = {
+ {"discrete", "Normal"}, {"karras", "Karras"}, {"exponential", "Exponential"}, {"sgm_uniform", "SGM Uniform"}, {"simple", "Simple"}, {"ddim_uniform", "DDIM"}, {"beta", "Beta"}};
+ if (sampling.contains("scheduler")) {
+ const auto scheduler = sampling.at("scheduler").get<std::string>();
+ auto mapped_schedule = schedules.find(scheduler);
+ field(mapped_schedule != schedules.end() ? "Schedule type" : "SDCPP Schedule", mapped_schedule != schedules.end() ? mapped_schedule->second : scheduler);
+ }
+ field("CFG scale", sampling.at("guidance").at("txt_cfg"));
+ field("Seed", info.at("seed"));
+ field("Size", std::to_string(info.at("width").get<int>()) + "x" + std::to_string(info.at("height").get<int>()));
+ if (info.contains("models")) {
+ const auto& models = info.at("models");
+ auto model = models.value("diffusion_model", models.value("model", ""));
+ const auto extension = model.find_last_of('.');
+ if (extension != std::string::npos && extension > 0)
+ model.erase(extension);
+ if (!model.empty())
+ field("Model", model);
+ if (models.contains("vae"))
+ field("VAE", models.at("vae"));
+ }
+ if (sampling.contains("eta"))
+ field("Eta", sampling.at("eta"));
+ if (info.value("clip_skip", 0) > 0)
+ field("Clip skip", info.at("clip_skip"));
+ if (info.contains("strength"))
+ field("Denoising strength", info.at("strength"));
+ if (info.contains("hires") && info.at("hires").value("enabled", false)) {
+ const auto& hires = info.at("hires");
+ field("Hires resize", std::to_string(hires.at("target_width").get<int>()) + "x" + std::to_string(hires.at("target_height").get<int>()));
+ if (hires.value("steps", 0) > 0)
+ field("Hires steps", hires.at("steps"));
+ field("Denoising strength", hires.at("denoising_strength"));
+ field("Hires upscaler", hires.at("upscaler"));
+ }
+ field("Version", "stable-diffusion.cpp");
+ return result;
+}
+
+std::string generation_info_to_image_metadata(const std::string& info) {
+ if (info.empty())
+ return "";
+ const auto parameters = generation_info_to_webui(info);
+ return parameters.empty() ? info : parameters + ", SDCPP: " + info;
+}
diff --git a/examples/common/generation_metadata.h b/examples/common/generation_metadata.h
new file mode 100644
index 0000000..7c9e9ff
--- /dev/null
+++ b/examples/common/generation_metadata.h
@@ -0,0 +1,13 @@
+#ifndef __EXAMPLES_COMMON_GENERATION_METADATA_H__
+#define __EXAMPLES_COMMON_GENERATION_METADATA_H__
+
+#include <cstdint>
+#include <string>
+
+std::string merge_generation_info(const std::string& source, const std::string& operation);
+std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir);
+int64_t generation_seed(const std::string& info, int64_t fallback = -1);
+std::string generation_info_to_webui(const std::string& text);
+std::string generation_info_to_image_metadata(const std::string& info);
+
+#endif // __EXAMPLES_COMMON_GENERATION_METADATA_H__
diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp
index baac6d2..f09c216 100644
--- a/examples/common/media_io.cpp
+++ b/examples/common/media_io.cpp
@@ -1,5 +1,7 @@
#include "media_io.h"
+#include "generation_metadata.h"
#include "log.h"
+#include "media_metadata.h"
#include "resource_owners.hpp"
#include <algorithm>
@@ -316,23 +318,22 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data,
return output.release();
}
-std::string build_webp_xmp_packet(const std::string& parameters) {
- if (parameters.empty()) {
- return "";
- }
-
- const std::string escaped_parameters = xml_escape(parameters);
- return "<?xpacket begin=\"\" id=\"W5M0MpCehiHzreSzNTczkc9d\"?>\n"
- "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\">\n"
- " <rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n"
- " <rdf:Description xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">\n"
- " <sdcpp:parameters>" +
- escaped_parameters +
- "</sdcpp:parameters>\n"
- " </rdf:Description>\n"
- " </rdf:RDF>\n"
- "</x:xmpmeta>\n"
- "<?xpacket end=\"w\"?>";
+static bool embed_webp_metadata(std::vector<uint8_t>& image, const std::string& parameters) {
+ if (parameters.empty())
+ return true;
+ WebPData data{image.data(), image.size()};
+ WebPMuxPtr mux(WebPMuxCreate(&data, 1));
+ if (!mux)
+ return false;
+ const auto xmp = generation_xmp(parameters);
+ WebPData xmp_data{reinterpret_cast<const uint8_t*>(xmp.data()), xmp.size()};
+ if (WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) != WEBP_MUX_OK)
+ return false;
+ WebPDataGuard assembled;
+ if (WebPMuxAssemble(mux.get(), &assembled.data) != WEBP_MUX_OK)
+ return false;
+ image.assign(assembled.data.bytes, assembled.data.bytes + assembled.data.size);
+ return true;
}
bool encode_webp_image_to_vector(const uint8_t* image,
@@ -376,37 +377,7 @@ bool encode_webp_image_to_vector(const uint8_t* image,
out.assign(encoded.get(), encoded.get() + encoded_size);
- if (parameters.empty()) {
- return true;
- }
-
- WebPData image_data;
- WebPDataInit(&image_data);
- WebPDataGuard assembled_data;
-
- image_data.bytes = out.data();
- image_data.size = out.size();
-
- WebPMuxPtr mux(WebPMuxNew());
- if (mux == nullptr) {
- return false;
- }
-
- const std::string xmp_packet = build_webp_xmp_packet(parameters);
- WebPData xmp_data;
- WebPDataInit(&xmp_data);
- xmp_data.bytes = reinterpret_cast<const uint8_t*>(xmp_packet.data());
- xmp_data.size = xmp_packet.size();
-
- const bool ok = WebPMuxSetImage(mux.get(), &image_data, 1) == WEBP_MUX_OK &&
- WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) == WEBP_MUX_OK &&
- WebPMuxAssemble(mux.get(), &assembled_data.data) == WEBP_MUX_OK;
-
- if (ok) {
- out.assign(assembled_data.data.bytes, assembled_data.data.bytes + assembled_data.data.size);
- }
-
- return ok;
+ return embed_webp_metadata(out, generation_info_to_image_metadata(parameters));
}
#ifdef SD_USE_WEBM
@@ -739,6 +710,7 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
const std::string& parameters,
int quality) {
std::vector<uint8_t> buffer;
+ const auto metadata = format == EncodedImageFormat::WEBP ? std::string() : generation_info_to_image_metadata(parameters);
auto write_func = [&buffer](void* context, void* data, int size) {
(void)context;
@@ -761,7 +733,7 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
result = stbi_write_jpg_to_func(c_func, &ctx, width, height, channels, image, quality);
break;
case EncodedImageFormat::PNG:
- result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, parameters.empty() ? nullptr : parameters.c_str());
+ result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, metadata.empty() ? nullptr : metadata.c_str());
break;
case EncodedImageFormat::WEBP:
#ifdef SD_USE_WEBP
@@ -779,9 +751,12 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
break;
}
- if (!result) {
- buffer.clear();
+ if (result && format == EncodedImageFormat::JPEG && !embed_jpeg_metadata(buffer, metadata)) {
+ LOG_ERROR("generation metadata exceeds the container capacity or the encoded image is invalid");
+ result = 0;
}
+ if (!result)
+ buffer.clear();
return buffer;
}
@@ -794,18 +769,8 @@ bool write_image_to_file(const std::string& path,
int quality) {
const EncodedImageFormat format = encoded_image_format_from_path(path);
- switch (format) {
- case EncodedImageFormat::JPEG:
- return stbi_write_jpg(path.c_str(), width, height, channels, image, quality, parameters.empty() ? nullptr : parameters.c_str()) != 0;
- case EncodedImageFormat::PNG:
- return stbi_write_png(path.c_str(), width, height, channels, image, 0, parameters.empty() ? nullptr : parameters.c_str()) != 0;
- case EncodedImageFormat::WEBP: {
- const std::vector<uint8_t> encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality);
- return !encoded.empty() && write_binary_file_bytes(path, encoded);
- }
- default:
- return false;
- }
+ const auto encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality);
+ return !encoded.empty() && write_binary_file_bytes(path, encoded);
}
uint8_t* load_image_from_file(const char* image_path,
@@ -1384,7 +1349,10 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
#ifdef SD_USE_WEBP
if (format == "webp") {
- return create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
+ auto data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
+ if (!data.empty() && !embed_webp_metadata(data, parameters))
+ return {};
+ return data;
}
#endif
diff --git a/examples/common/media_io.h b/examples/common/media_io.h
index b2b56ec..18042a8 100644
--- a/examples/common/media_io.h
+++ b/examples/common/media_io.h
@@ -16,6 +16,7 @@ enum class EncodedImageFormat {
EncodedImageFormat encoded_image_format_from_path(const std::string& path);
+// parameters is the core sdcpp.generation/v1 JSON; empty disables both metadata formats.
std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
const uint8_t* image,
int width,
diff --git a/examples/common/media_metadata.cpp b/examples/common/media_metadata.cpp
new file mode 100644
index 0000000..d5e9af3
--- /dev/null
+++ b/examples/common/media_metadata.cpp
@@ -0,0 +1,47 @@
+#include "media_metadata.h"
+
+std::string generation_xmp(const std::string& info) {
+ std::string escaped;
+ for (char c : info) {
+ switch (c) {
+ case '&':
+ escaped += "&";
+ break;
+ case '<':
+ escaped += "<";
+ break;
+ case '>':
+ escaped += ">";
+ break;
+ case '"':
+ escaped += """;
+ break;
+ case '\'':
+ escaped += "'";
+ break;
+ default:
+ escaped += c;
+ break;
+ }
+ }
+ return "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">"
+ "<rdf:Description rdf:about=\"\" xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">"
+ "<sdcpp:parameters>" +
+ escaped + "</sdcpp:parameters></rdf:Description></rdf:RDF></x:xmpmeta>";
+}
+
+bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters) {
+ if (parameters.empty())
+ return true;
+ const char label[] = "parameters";
+ const size_t overhead = sizeof(label) + 1;
+ if (image.size() < 2 || parameters.size() > 65533 - overhead)
+ return false;
+ const size_t length = overhead + parameters.size() + 2;
+ std::vector<uint8_t> segment = {0xff, 0xfe, static_cast<uint8_t>(length >> 8), static_cast<uint8_t>(length)};
+ segment.insert(segment.end(), label, label + sizeof(label));
+ segment.insert(segment.end(), parameters.begin(), parameters.end());
+ segment.push_back(0);
+ image.insert(image.begin() + 2, segment.begin(), segment.end());
+ return true;
+}
diff --git a/examples/common/media_metadata.h b/examples/common/media_metadata.h
new file mode 100644
index 0000000..1843dee
--- /dev/null
+++ b/examples/common/media_metadata.h
@@ -0,0 +1,11 @@
+#ifndef __EXAMPLES_COMMON_MEDIA_METADATA_H__
+#define __EXAMPLES_COMMON_MEDIA_METADATA_H__
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+std::string generation_xmp(const std::string& parameters);
+bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters);
+
+#endif // __EXAMPLES_COMMON_MEDIA_METADATA_H__
diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp
index 73f8bdd..8a8f843 100644
--- a/examples/common/resource_owners.hpp
+++ b/examples/common/resource_owners.hpp
@@ -8,6 +8,7 @@
#include <utility>
#include <vector>
+#include "generation_metadata.h"
#include "stable-diffusion.h"
struct FreeDeleter {
@@ -175,6 +176,7 @@ public:
class SDImageVec {
private:
std::vector<sd_image_t> images_;
+ std::vector<std::string> generation_infos_;
public:
SDImageVec() = default;
@@ -183,13 +185,14 @@ public:
SDImageVec& operator=(const SDImageVec&) = delete;
SDImageVec(SDImageVec&& other) noexcept
- : images_(std::move(other.images_)) {
+ : images_(std::move(other.images_)), generation_infos_(std::move(other.generation_infos_)) {
}
SDImageVec& operator=(SDImageVec&& other) noexcept {
if (this != &other) {
clear();
- images_ = std::move(other.images_);
+ images_ = std::move(other.images_);
+ generation_infos_ = std::move(other.generation_infos_);
}
return *this;
}
@@ -210,6 +213,41 @@ public:
images_.reserve(count);
}
+ const std::string& generation_info(size_t index) const {
+ static const std::string empty;
+ return index < generation_infos_.size() ? generation_infos_[index] : empty;
+ }
+
+ void append_generation_info(size_t index, const std::string& info, const std::string& lora_model_dir = {}) {
+ generation_infos_.resize(images_.size());
+ generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir));
+ }
+
+ void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) {
+ clear();
+ if (!result)
+ return;
+ for (int i = 0; i < result->count; ++i) {
+ images_.push_back(result->items[i].image);
+ result->items[i].image = {};
+ generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir));
+ }
+ free_sd_image_generation_result(result);
+ }
+
+ void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) {
+ clear();
+ if (!result)
+ return;
+ const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir);
+ for (int i = 0; i < result->frame_count; ++i) {
+ images_.push_back(result->frames[i]);
+ result->frames[i] = {};
+ generation_infos_.push_back(info);
+ }
+ free_sd_video_generation_result(result);
+ }
+
void adopt(sd_image_t* images, int count) {
clear();
if (images == nullptr || count <= 0) {
@@ -270,6 +308,7 @@ public:
image.data = nullptr;
}
images_.clear();
+ generation_infos_.clear();
}
};
diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
index 58260be..d01f7a7 100644
--- a/examples/server/CMakeLists.txt
+++ b/examples/server/CMakeLists.txt
@@ -64,8 +64,10 @@ endif()
add_executable(${TARGET}
../common/common.cpp
+ ../common/generation_metadata.cpp
../common/log.cpp
../common/media_io.cpp
+ ../common/media_metadata.cpp
main.cpp
runtime.cpp
model_catalog.cpp
diff --git a/examples/server/api.md b/examples/server/api.md
index 0d83214..1258b1a 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -1776,3 +1776,14 @@ Example cancelled job:
- `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, invalid generation parameters, or an unsupported output format
- `429 Too Many Requests` when the job queue is full
- `500 Internal Server Error` for unexpected server exceptions during submission
+
+## Actual generation parameters
+
+Generation responses expose the core `sdcpp.generation/v1` record as
+`generation_info`, independently of `embed_image_metadata`. Native async image
+result items and OpenAI image items contain the parameter snapshot with each
+image's actual seed and without `batch_count`. Native video
+results contain the video request snapshot. SD API `info.generation_info` is an array
+aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field
+remains the request echo. See [generation metadata](../../docs/generation_metadata.md)
+for field semantics, postprocessing and container storage.
diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp
index 943316e..7a80107 100644
--- a/examples/server/async_jobs.cpp
+++ b/examples/server/async_jobs.cpp
@@ -179,6 +179,7 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
job.status = AsyncJobStatus::Cancelled;
job.completed_at = unix_timestamp_now_ms();
job.result_images_b64.clear();
+ job.result_generation_info = nullptr;
job.result_media_b64.clear();
job.result_media_mime_type.clear();
job.result_frame_count = 0;
@@ -218,11 +219,12 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
{"fps", job.result_fps},
{"frame_count", job.result_frame_count},
{"b64_json", job.result_media_b64},
+ {"generation_info", job.result_generation_info},
};
} else {
json images = json::array();
for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
- images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+ images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}, {"generation_info", job.result_generation_info.at(i)}});
}
result["result"] = {
{"output_format", job.img_gen.output_format},
@@ -263,7 +265,9 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
bool execute_img_gen_job(ServerRuntime& runtime,
AsyncGenerationJob& job,
std::vector<std::string>& output_images,
+ json& output_info,
std::string& error_message) {
+ output_info = json::array();
sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
SDImageVec results;
@@ -275,14 +279,10 @@ bool execute_img_gen_job(ServerRuntime& runtime,
}
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
set_preview_callback(job);
- sd_image_t* raw_results = nullptr;
- int num_results = 0;
- if (!generate_image(runtime.sd_ctx, ¶ms, &raw_results, &num_results)) {
- raw_results = nullptr;
- num_results = 0;
- }
+ sd_image_generation_result_t* generated = nullptr;
+ generate_image(runtime.sd_ctx, ¶ms, &generated);
+ results.adopt(generated, runtime.ctx_params->lora_model_dir);
clear_preview_callback();
- results.adopt(raw_results, num_results);
if (job.cancel_requested.load()) {
return false;
}
@@ -305,17 +305,13 @@ bool execute_img_gen_job(ServerRuntime& runtime,
encoded_format = EncodedImageFormat::WEBP;
}
- int batch_count = job.img_gen.gen_params.batch_count;
- int images_per_batch = batch_count > 0 ? std::max(1, num_results / batch_count) : 1;
for (int i = 0; i < num_results; ++i) {
if (results[i].data == nullptr) {
continue;
}
const std::string metadata = job.img_gen.gen_params.embed_image_metadata
- ? get_image_params(*runtime.ctx_params,
- job.img_gen.gen_params,
- job.img_gen.gen_params.seed + i / images_per_batch)
+ ? results.generation_info(i)
: "";
auto image_bytes = encode_image_to_vector(encoded_format,
results[i].data,
@@ -328,6 +324,7 @@ bool execute_img_gen_job(ServerRuntime& runtime,
continue;
}
output_images.push_back(base64_encode(image_bytes));
+ output_info.push_back(json::parse(results.generation_info(i)));
}
if (output_images.empty()) {
@@ -344,11 +341,10 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
std::string& output_media_mime_type,
int& output_frame_count,
int& output_fps,
+ json& output_info,
std::string& error_message) {
sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
- std::string str_params = job.vid_gen.gen_params.embed_image_metadata
- ? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN)
- : "";
+ std::string str_params;
SDImageVec results;
int num_results = 0;
@@ -361,12 +357,17 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
}
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
set_preview_callback(job);
- sd_image_t* raw_results = nullptr;
- if (!generate_video(runtime.sd_ctx, ¶ms, &raw_results, &num_results, &generated_audio, &output_fps)) {
- raw_results = nullptr;
+ sd_video_generation_result_t* generated = nullptr;
+ if (generate_video(runtime.sd_ctx, ¶ms, &generated)) {
+ output_fps = generated->fps;
+ generated_audio = generated->audio;
+ generated->audio = nullptr;
+ results.adopt_video(generated, runtime.ctx_params->lora_model_dir);
+ output_info = json::parse(results.generation_info(0));
+ if (job.vid_gen.gen_params.embed_image_metadata)
+ str_params = results.generation_info(0);
}
clear_preview_callback();
- results.adopt(raw_results, num_results);
}
num_results = results.count();
@@ -427,6 +428,7 @@ void async_job_worker(ServerRuntime& runtime) {
}
std::vector<std::string> output_images;
+ json output_info;
std::string output_media_b64;
std::string output_media_mime_type;
int output_frame_count = 0;
@@ -435,7 +437,7 @@ void async_job_worker(ServerRuntime& runtime) {
bool ok = false;
if (job->kind == AsyncJobKind::ImgGen) {
- ok = execute_img_gen_job(runtime, *job, output_images, error_message);
+ ok = execute_img_gen_job(runtime, *job, output_images, output_info, error_message);
} else if (job->kind == AsyncJobKind::VidGen) {
ok = execute_vid_gen_job(runtime,
*job,
@@ -443,6 +445,7 @@ void async_job_worker(ServerRuntime& runtime) {
output_media_mime_type,
output_frame_count,
output_fps,
+ output_info,
error_message);
} else {
error_message = "unsupported job kind";
@@ -463,6 +466,7 @@ void async_job_worker(ServerRuntime& runtime) {
} else if (ok) {
job->status = AsyncJobStatus::Completed;
job->result_images_b64 = std::move(output_images);
+ job->result_generation_info = std::move(output_info);
job->result_media_b64 = std::move(output_media_b64);
job->result_media_mime_type = std::move(output_media_mime_type);
job->result_frame_count = output_frame_count;
@@ -474,6 +478,7 @@ void async_job_worker(ServerRuntime& runtime) {
job->error_code = "generation_failed";
job->error_message = error_message.empty() ? "unknown generation error" : error_message;
job->result_images_b64.clear();
+ job->result_generation_info = nullptr;
job->result_media_b64.clear();
job->result_media_mime_type.clear();
job->result_frame_count = 0;
diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h
index aa54370..6fa3cbd 100644
--- a/examples/server/async_jobs.h
+++ b/examples/server/async_jobs.h
@@ -40,6 +40,7 @@ struct AsyncGenerationJob {
ImgGenJobRequest img_gen;
VidGenJobRequest vid_gen;
std::vector<std::string> result_images_b64;
+ json result_generation_info;
std::string result_media_b64;
std::string result_media_mime_type;
int result_frame_count = 0;
@@ -74,6 +75,7 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
bool execute_img_gen_job(ServerRuntime& runtime,
AsyncGenerationJob& job,
std::vector<std::string>& output_images,
+ json& output_info,
std::string& error_message);
bool execute_vid_gen_job(ServerRuntime& runtime,
AsyncGenerationJob& job,
@@ -81,5 +83,6 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
std::string& output_media_mime_type,
int& output_frame_count,
int& output_fps,
+ json& output_info,
std::string& error_message);
void async_job_worker(ServerRuntime& runtime);
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index 409c88d..f721e14 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -223,12 +223,10 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
{
std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
- sd_image_t* raw_results = nullptr;
- if (!generate_image(runtime.sd_ctx, &img_gen_params, &raw_results, &num_results)) {
- raw_results = nullptr;
- num_results = 0;
- }
- results.adopt(raw_results, num_results);
+ sd_image_generation_result_t* generated = nullptr;
+ generate_image(runtime.sd_ctx, &img_gen_params, &generated);
+ results.adopt(generated, runtime.ctx_params->lora_model_dir);
+ num_results = results.count();
if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) {
error_message = "ADetailer failed";
return false;
@@ -285,16 +283,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
out["data"] = json::array();
out["output_format"] = request.output_format;
- int result_count = results.count();
- int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1;
+ int result_count = results.count();
+
for (int i = 0; i < result_count; ++i) {
if (results[i].data == nullptr) {
continue;
}
std::string params = request.gen_params.embed_image_metadata
- ? get_image_params(*runtime->ctx_params,
- request.gen_params,
- request.gen_params.seed + i / images_per_batch)
+ ? results.generation_info(i)
: "";
auto image_bytes = encode_image_to_vector(request.output_format == "jpeg"
? EncodedImageFormat::JPEG
@@ -313,7 +309,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
}
json item;
- item["b64_json"] = base64_encode(image_bytes);
+ item["b64_json"] = base64_encode(image_bytes);
+ item["generation_info"] = json::parse(results.generation_info(i));
out["data"].push_back(item);
}
@@ -359,16 +356,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
out["data"] = json::array();
out["output_format"] = request.output_format;
- int result_count = results.count();
- int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1;
+ int result_count = results.count();
+
for (int i = 0; i < result_count; ++i) {
if (results[i].data == nullptr) {
continue;
}
std::string params = request.gen_params.embed_image_metadata
- ? get_image_params(*runtime->ctx_params,
- request.gen_params,
- request.gen_params.seed + i / images_per_batch)
+ ? results.generation_info(i)
: "";
auto image_bytes = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG,
results[i].data,
@@ -378,7 +373,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
params,
request.output_compression);
json item;
- item["b64_json"] = base64_encode(image_bytes);
+ item["b64_json"] = base64_encode(image_bytes);
+ item["generation_info"] = json::parse(results.generation_info(i));
out["data"].push_back(item);
}
diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp
index ef9a6fe..0d37898 100644
--- a/examples/server/routes_sdapi.cpp
+++ b/examples/server/routes_sdapi.cpp
@@ -462,46 +462,27 @@ static bool build_sdapi_img_gen_request(const json& j,
validate_server_img_gen_request(runtime, request, error_message);
}
-static nlohmann::json prepare_info_field(const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- bool img2img) {
- nlohmann::json jsoninfo = nlohmann::json::object();
- jsoninfo["prompt"] = gen_params.prompt;
- if (!gen_params.negative_prompt.empty()) {
- jsoninfo["negative_prompt"] = gen_params.negative_prompt;
- }
- jsoninfo["seed"] = gen_params.seed;
- jsoninfo["cfg_scale"] = gen_params.sample_params.guidance.txt_cfg;
- jsoninfo["width"] = gen_params.get_resolved_width();
- jsoninfo["height"] = gen_params.get_resolved_height();
- jsoninfo["steps"] = gen_params.sample_params.sample_steps;
- jsoninfo["sampler_name"] = sd_sample_method_name(gen_params.sample_params.sample_method);
- if (gen_params.clip_skip != -1) {
- jsoninfo["clip_skip"] = gen_params.clip_skip;
- }
- if (gen_params.sample_params.scheduler != scheduler_t::SCHEDULER_COUNT) {
- jsoninfo["extra_generation_params"] = nlohmann::json::object();
- jsoninfo["extra_generation_params"]["Schedule type"] = sd_scheduler_name(gen_params.sample_params.scheduler);
- }
- if (img2img) {
- jsoninfo["denoising_strength"] = gen_params.strength;
- }
- // not clear what should happen if we have both model and diffusion_model
- if (!ctx_params.diffusion_model_path.empty()) {
- jsoninfo["sd_model_name"] = sd_basename(ctx_params.diffusion_model_path);
- } else if (!ctx_params.model_path.empty()) {
- jsoninfo["sd_model_name"] = sd_basename(ctx_params.model_path);
- }
- if (!ctx_params.vae_path.empty()) {
- jsoninfo["sd_vae_name"] = sd_basename(ctx_params.vae_path);
- }
- jsoninfo["version"] = "stable-diffusion.cpp";
-
- jsoninfo["infotexts"] = nlohmann::json::array();
- jsoninfo["all_prompts"] = nlohmann::json::array();
- jsoninfo["all_negative_prompts"] = nlohmann::json::array();
- jsoninfo["all_seeds"] = nlohmann::json::array();
- return jsoninfo;
+static nlohmann::json prepare_info_field(const std::string& generation_info) {
+ const auto info = nlohmann::json::parse(generation_info);
+ const auto& sampling = info.at("sample_params");
+ nlohmann::json result = {{"prompt", info.at("prompt")},
+ {"negative_prompt", info.at("negative_prompt")},
+ {"seed", generation_seed(generation_info)},
+ {"cfg_scale", sampling.at("guidance").at("txt_cfg")},
+ {"width", info.at("width")},
+ {"height", info.at("height")},
+ {"steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps")},
+ {"sampler_name", sampling.at("sample_method")},
+ {"version", "stable-diffusion.cpp"}};
+ if (info.contains("clip_skip"))
+ result["clip_skip"] = info.at("clip_skip");
+ if (info.contains("strength"))
+ result["denoising_strength"] = info.at("strength");
+ if (sampling.contains("scheduler"))
+ result["extra_generation_params"]["Schedule type"] = sampling.at("scheduler");
+ for (const auto* key : {"infotexts", "all_prompts", "all_negative_prompts", "all_seeds", "generation_info"})
+ result[key] = nlohmann::json::array();
+ return result;
}
struct SDAPIPreviewState {
@@ -621,12 +602,10 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty();
SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing);
- sd_image_t* raw_results = nullptr;
- if (!generate_image(runtime->sd_ctx, &img_gen_params, &raw_results, &num_results)) {
- raw_results = nullptr;
- num_results = 0;
- }
- results.adopt(raw_results, num_results);
+ sd_image_generation_result_t* generated = nullptr;
+ generate_image(runtime->sd_ctx, &img_gen_params, &generated);
+ results.adopt(generated, runtime->ctx_params->lora_model_dir);
+ num_results = results.count();
if (detailing && !results.empty())
preview_guard.begin_detailing();
if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) {
@@ -645,9 +624,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
json out;
out["images"] = json::array();
out["parameters"] = j;
- json jsoninfo = prepare_info_field(*runtime->ctx_params, request.gen_params, img2img);
+ json jsoninfo = prepare_info_field(results.generation_info(0));
- int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, num_results / request.gen_params.batch_count) : 1;
for (int i = 0; i < num_results; ++i) {
if (results[i].data == nullptr) {
continue;
@@ -655,16 +633,14 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
bool embed_meta = request.gen_params.embed_image_metadata;
- std::string params = get_image_params(*runtime->ctx_params,
- request.gen_params,
- request.gen_params.seed + i / images_per_batch);
+ std::string params = generation_info_to_webui(results.generation_info(i));
auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG,
results[i].data,
results[i].width,
results[i].height,
results[i].channel,
- embed_meta ? params : "");
+ embed_meta ? results.generation_info(i) : "");
if (image_bytes.empty()) {
LOG_ERROR("write image to mem failed");
@@ -673,10 +649,12 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
out["images"].push_back(base64_encode(image_bytes));
- jsoninfo["infotexts"][i] = params;
- jsoninfo["all_seeds"][i] = request.gen_params.seed + i;
- jsoninfo["all_prompts"][i] = request.gen_params.prompt;
- jsoninfo["all_negative_prompts"][i] = request.gen_params.negative_prompt;
+ const auto actual = json::parse(results.generation_info(i));
+ jsoninfo["infotexts"].push_back(params);
+ jsoninfo["all_seeds"].push_back(generation_seed(results.generation_info(i)));
+ jsoninfo["generation_info"].push_back(actual);
+ jsoninfo["all_prompts"].push_back(actual["prompt"]);
+ jsoninfo["all_negative_prompts"].push_back(actual["negative_prompt"]);
}
// not a mistake: it is supposed to be a string in json format
diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp
index 29eed7e..b972768 100644
--- a/examples/server/routes_sdcpp.cpp
+++ b/examples/server/routes_sdcpp.cpp
@@ -609,19 +609,20 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
}
SDImageOwner current(input.release());
+ std::string generation_info;
for (int i = 0; i < repeats; ++i) {
- sd_image_t* out_images = nullptr;
- int out_count = 0;
- if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) ||
- out_count <= 0 || out_images[0].data == nullptr) {
- free_sd_images(out_images, out_count);
+ sd_image_generation_result_t* generated = nullptr;
+ if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) ||
+ !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) {
+ free_sd_image_generation_result(generated);
res.status = 500;
res.set_content(R"({"error":"upscale failed"})", "application/json");
return;
}
- sd_image_t produced = out_images[0];
- out_images[0] = {0, 0, 0, nullptr};
- free_sd_images(out_images, out_count);
+ generation_info = merge_generation_info(generation_info, generated->items[0].generation_info);
+ sd_image_t produced = generated->items[0].image;
+ generated->items[0].image = {};
+ free_sd_image_generation_result(generated);
current.reset(produced);
}
@@ -635,7 +636,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
result.width,
result.height,
result.channel,
- "",
+ generation_info,
compression);
if (image_bytes.empty()) {
res.status = 500;
@@ -644,13 +645,14 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
}
json out;
- out["upscaler"] = used_name;
- out["scale"] = factor;
- out["repeats"] = repeats;
- out["width"] = result.width;
- out["height"] = result.height;
- out["output_format"] = format;
- json images = json::array();
+ out["generation_info"] = json::parse(generation_info);
+ out["upscaler"] = used_name;
+ out["scale"] = factor;
+ out["repeats"] = repeats;
+ out["width"] = result.width;
+ out["height"] = result.height;
+ out["output_format"] = format;
+ json images = json::array();
images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}});
out["images"] = std::move(images);
res.set_content(out.dump(), "application/json");
diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp
index 5b33b33..b187ff8 100644
--- a/examples/server/runtime.cpp
+++ b/examples/server/runtime.cpp
@@ -136,7 +136,7 @@ bool apply_server_adetailer(ServerRuntime& runtime,
continue;
}
sd_img_gen_params_t ad_generation_params = params;
- ad_generation_params.seed = params.seed + i;
+ ad_generation_params.seed = generation_seed(results.generation_info(i));
ad_generation_params.width = pass.width;
ad_generation_params.height = pass.height;
ad_generation_params.strength = pass.strength;
@@ -147,24 +147,23 @@ bool apply_server_adetailer(ServerRuntime& runtime,
ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT;
}
- sd_image_t* detailed_images = nullptr;
- int detailed_count = 0;
+ sd_image_generation_result_t* detailed = nullptr;
if (!adetail_image(ad_ctx.get(),
runtime.sd_ctx,
results[i],
&ad_params,
&ad_generation_params,
- &detailed_images,
- &detailed_count) ||
- detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) {
- free_sd_images(detailed_images, detailed_count);
+ &detailed) ||
+ detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
+ free_sd_image_generation_result(detailed);
LOG_ERROR("ADetailer failed for image %d", i + 1);
return false;
}
+ results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir);
free(results[i].data);
- results[i] = detailed_images[0];
- detailed_images[0] = {0, 0, 0, nullptr};
- free_sd_images(detailed_images, detailed_count);
+ results[i] = detailed->items[0].image;
+ detailed->items[0].image = {};
+ free_sd_image_generation_result(detailed);
}
}
return true;
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index d029261..f4ea0ea 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -272,6 +272,25 @@ typedef struct {
uint8_t* data;
} sd_image_t;
+typedef struct {
+ sd_image_t image;
+ // Owned UTF-8 JSON using the sdcpp.generation/v1 schema.
+ char* generation_info;
+} sd_image_result_t;
+
+typedef struct {
+ sd_image_result_t* items;
+ int count;
+} sd_image_generation_result_t;
+
+typedef struct {
+ sd_image_t* frames;
+ int frame_count;
+ sd_audio_t* audio;
+ int fps;
+ char* generation_info;
+} sd_video_generation_result_t;
+
typedef struct {
// Semicolon-separated target=...,key=value rules. NULL preserves defaults.
const char* rules;
@@ -536,8 +555,12 @@ SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params);
SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params);
SD_API bool generate_image(sd_ctx_t* sd_ctx,
const sd_img_gen_params_t* sd_img_gen_params,
- sd_image_t** images_out,
- int* num_images_out);
+ sd_image_generation_result_t** result_out);
+
+// Results own their pixels, audio and generation_info, independently of the context.
+// Generation calls require result_out and set it to NULL on failure.
+SD_API void free_sd_image_generation_result(sd_image_generation_result_t* result);
+SD_API void free_sd_video_generation_result(sd_video_generation_result_t* result);
enum sd_cancel_mode_t {
// Stop the current generation as soon as possible.
@@ -551,13 +574,9 @@ enum sd_cancel_mode_t {
SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode);
SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params);
-// If non-NULL, fps_out receives the effective encoding frame rate before preview callbacks.
SD_API bool generate_video(sd_ctx_t* sd_ctx,
const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_image_t** frames_out,
- int* num_frames_out,
- sd_audio_t** audio_out,
- int* fps_out);
+ sd_video_generation_result_t** result_out);
typedef struct upscaler_ctx_t upscaler_ctx_t;
@@ -572,8 +591,7 @@ SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
SD_API bool upscale(upscaler_ctx_t* upscaler_ctx,
sd_image_t input_image,
uint32_t upscale_factor,
- sd_image_t** images_out,
- int* num_images_out);
+ sd_image_generation_result_t** result_out);
SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx);
// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model.
@@ -597,8 +615,7 @@ SD_API bool adetail_image(adetailer_ctx_t* adetailer_ctx,
sd_image_t input_image,
const sd_adetailer_params_t* adetailer_params,
const sd_img_gen_params_t* inpaint_params,
- sd_image_t** images_out,
- int* num_images_out);
+ sd_image_generation_result_t** result_out);
SD_API bool convert(const char* input_path,
const char* vae_path,
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index c7ef70f..e3832a1 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -141,6 +141,7 @@ struct ConditionerParams {
};
struct Conditioner {
+ virtual int resolve_clip_skip(int) const { return 0; }
virtual ~Conditioner() = default;
public:
@@ -173,6 +174,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
std::vector<uint8_t> token_embed_custom2;
std::map<std::string, std::pair<int, int>> embedding_pos_map;
+ int resolve_clip_skip(int value) const override {
+ return value > 0 ? value : ((sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1);
+ }
+
FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map,
const std::map<std::string, std::string>& orig_embedding_map,
@@ -456,10 +461,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
sd::Tensor<float> hidden_states; // [n_token, hidden_size] or [n_token, hidden_size + hidden_size2]
sd::Tensor<float> pooled;
- if (clip_skip <= 0) {
- clip_skip = (sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1;
- }
-
size_t chunk_len = 77;
size_t chunk_count = tokens.size() / chunk_len;
for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
@@ -651,6 +652,8 @@ struct SD3CLIPEmbedder : public Conditioner {
std::shared_ptr<CLIPTextModelRunner> clip_g;
std::shared_ptr<T5Runner> t5;
+ int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; }
+
SD3CLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
@@ -898,10 +901,6 @@ struct SD3CLIPEmbedder : public Conditioner {
auto& t5_tokens = token_and_weights[2].first;
auto& t5_weights = token_and_weights[2].second;
- if (clip_skip <= 0) {
- clip_skip = 2;
- }
-
size_t chunk_len = 77;
int64_t t0 = ggml_time_ms();
sd::Tensor<float> hidden_states;
@@ -1097,6 +1096,8 @@ struct FluxCLIPEmbedder : public Conditioner {
std::shared_ptr<T5Runner> t5;
size_t chunk_len = 256;
+ int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; }
+
FluxCLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
@@ -1293,10 +1294,6 @@ struct FluxCLIPEmbedder : public Conditioner {
auto& t5_tokens = token_and_weights[1].first;
auto& t5_weights = token_and_weights[1].second;
- if (clip_skip <= 0) {
- clip_skip = 2;
- }
-
int64_t t0 = ggml_time_ms();
sd::Tensor<float> hidden_states; // [N, n_token, 4096]
sd::Tensor<float> pooled; // [768,]
diff --git a/src/detailer.cpp b/src/detailer.cpp
index ec4d606..9c48ca4 100644
--- a/src/detailer.cpp
+++ b/src/detailer.cpp
@@ -1,4 +1,6 @@
#include "detailer.h"
+#include "generation_info.h"
+#include "pipeline/request.h"
#include <algorithm>
#include <cctype>
@@ -915,14 +917,11 @@ bool adetail_image(adetailer_ctx_t* context,
sd_image_t input_image,
const sd_adetailer_params_t* public_params,
const sd_img_gen_params_t* inpaint_params,
- sd_image_t** images_out,
- int* num_images_out) {
- if (images_out != nullptr) {
- *images_out = nullptr;
- }
- if (num_images_out != nullptr) {
- *num_images_out = 0;
- }
+ sd_image_generation_result_t** result_out) try {
+ if (result_out)
+ *result_out = nullptr;
+ if (!result_out)
+ return false;
if (context == nullptr || context->detailer == nullptr || sd_ctx == nullptr || public_params == nullptr ||
inpaint_params == nullptr || input_image.data == nullptr) {
return false;
@@ -933,7 +932,10 @@ bool adetail_image(adetailer_ctx_t* context,
return false;
}
- auto detections = context->detailer->predict(input_image, params);
+ auto resolved_inpaint = *inpaint_params;
+ resolved_inpaint.seed = sd::pipeline::resolve_seed(resolved_inpaint.seed);
+ const auto info = sd::adetailer_generation_info(*public_params, resolved_inpaint);
+ auto detections = context->detailer->predict(input_image, params);
filter_and_sort_detections(&detections, input_image.width, input_image.height, params);
auto masks = make_masks(detections, input_image.width, input_image.height, params);
@@ -942,6 +944,7 @@ bool adetail_image(adetailer_ctx_t* context,
return false;
}
+ std::unique_ptr<uint8_t, decltype(&free)> current_owner(current.data, &free);
int applied = 0;
for (size_t i = 0; i < masks.size(); ++i) {
std::string prompt = resolve_prompt(params.prompt, inpaint_params->prompt, static_cast<int>(i));
@@ -968,11 +971,10 @@ bool adetail_image(adetailer_ctx_t* context,
if (local_image.data == nullptr || local_mask.data == nullptr) {
free(local_image.data);
free(local_mask.data);
- free(current.data);
return false;
}
- sd_img_gen_params_t generation = *inpaint_params;
+ sd_img_gen_params_t generation = resolved_inpaint;
generation.prompt = prompt.c_str();
generation.negative_prompt = negative_prompt.c_str();
generation.init_image = local_image;
@@ -980,7 +982,7 @@ bool adetail_image(adetailer_ctx_t* context,
generation.width = params.inpaint_width;
generation.height = params.inpaint_height;
generation.strength = params.denoising_strength;
- generation.seed = inpaint_params->seed + static_cast<int64_t>(i);
+ generation.seed = static_cast<int64_t>((static_cast<uint64_t>(resolved_inpaint.seed) + i) & INT64_MAX);
generation.batch_count = 1;
generation.ref_images = nullptr;
generation.ref_images_count = 0;
@@ -1004,37 +1006,32 @@ bool adetail_image(adetailer_ctx_t* context,
generation.sample_params.scheduler = params.scheduler;
}
- sd_image_t* generated = nullptr;
- int generated_count = 0;
- bool success = generate_image(sd_ctx, &generation, &generated, &generated_count);
+ sd_image_generation_result_t* generated = nullptr;
+ bool success = generate_image(sd_ctx, &generation, &generated);
free(local_image.data);
free(local_mask.data);
- if (!success || generated_count <= 0 || generated == nullptr || generated[0].data == nullptr) {
- free_sd_images(generated, generated_count);
- free(current.data);
+ if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) {
+ free_sd_image_generation_result(generated);
return false;
}
+ std::unique_ptr<sd_image_generation_result_t, decltype(&free_sd_image_generation_result)> generated_owner(generated, &free_sd_image_generation_result);
Mask feather = gaussian_blur_mask(masks[i], params.mask_blur);
- composite_crop(¤t, generated[0], feather, crop);
- free_sd_images(generated, generated_count);
+ composite_crop(¤t, generated->items[0].image, feather, crop);
++applied;
}
+ const std::vector<nlohmann::json> infos{info};
auto* outputs = static_cast<sd_image_t*>(calloc(1, sizeof(sd_image_t)));
- if (outputs == nullptr) {
- free(current.data);
+ if (!outputs)
return false;
- }
outputs[0] = current;
- if (images_out != nullptr) {
- *images_out = outputs;
- } else {
- free_sd_images(outputs, 1);
- }
- if (num_images_out != nullptr) {
- *num_images_out = 1;
- }
+ current_owner.release();
+ if (!sd::make_image_generation_result(outputs, 1, infos, result_out))
+ return false;
LOG_INFO("ADetailer applied %d mask(s)", applied);
return true;
+} catch (const std::exception& e) {
+ LOG_ERROR("ADetailer failed: %s", e.what());
+ return false;
}
diff --git a/src/generation_info.cpp b/src/generation_info.cpp
new file mode 100644
index 0000000..2f49054
--- /dev/null
+++ b/src/generation_info.cpp
@@ -0,0 +1,276 @@
+#include "generation_info.h"
+
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+
+#include "pipeline/request.h"
+#include "runtime/guidance.h"
+
+namespace sd {
+ using nlohmann::json;
+
+ static const char* text(const char* value) {
+ return value ? value : "";
+ }
+
+ static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) {
+ const auto& g = p.guidance;
+ json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}};
+ if (!high_noise && p.shifted_timestep > 0)
+ info["shifted_timestep"] = p.shifted_timestep;
+ if (std::isfinite(g.img_cfg) && g.img_cfg != 1.f)
+ info["guidance"]["img_cfg"] = g.img_cfg;
+ if (p.sample_method != SAMPLE_METHOD_COUNT)
+ info["sample_method"] = sd_sample_method_name(p.sample_method);
+ if (!high_noise && p.scheduler != SCHEDULER_COUNT)
+ info["scheduler"] = sd_scheduler_name(p.scheduler);
+ if (std::isfinite(p.eta))
+ info["eta"] = p.eta;
+ if (!high_noise && std::isfinite(p.flow_shift))
+ info["flow_shift"] = p.flow_shift;
+ if (!high_noise && p.custom_sigmas && p.custom_sigmas_count > 0)
+ info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
+ if (p.extra_sample_args && *p.extra_sample_args)
+ info["extra_sample_args"] = p.extra_sample_args;
+ if (g.slg.layers && g.slg.layer_count > 0 &&
+ (g.slg.scale != 0.f || guidance::parse_skip_layer_guidance_uncond_arg(p.extra_sample_args)))
+ info["guidance"]["slg"] = {{"scale", g.slg.scale}, {"layer_start", g.slg.layer_start}, {"layer_end", g.slg.layer_end}, {"layers", std::vector<int>(g.slg.layers, g.slg.layers + g.slg.layer_count)}};
+ return info;
+ }
+
+ static json hires_info(const sd_hires_params_t& p) {
+ json info = {{"enabled", p.enabled}};
+ if (!p.enabled)
+ return info;
+ info.update({{"upscaler", sd_hires_upscaler_name(p.upscaler)}, {"scale", p.scale}, {"target_width", p.target_width}, {"target_height", p.target_height}, {"steps", p.steps}, {"denoising_strength", p.denoising_strength}});
+ if (p.upscaler == SD_HIRES_UPSCALER_MODEL) {
+ info["model_path"] = text(p.model_path);
+ info["upscale_tile_size"] = p.upscale_tile_size;
+ }
+ if (p.custom_sigmas && p.custom_sigmas_count > 0)
+ info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
+ return info;
+ }
+
+ static json cache_info(const sd_cache_params_t& p) {
+ static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"};
+ if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM)
+ return {{"mode", static_cast<int>(p.mode)}};
+ json info = {{"mode", modes[p.mode]}};
+ if (p.mode == SD_CACHE_DISABLED)
+ return info;
+ if (p.mode == SD_CACHE_EASYCACHE || p.mode == SD_CACHE_UCACHE) {
+ if (std::isfinite(p.reuse_threshold))
+ info["reuse_threshold"] = p.reuse_threshold;
+ info.update({{"start_percent", p.start_percent}, {"end_percent", p.end_percent}});
+ if (p.mode == SD_CACHE_UCACHE)
+ info.update({{"error_decay_rate", p.error_decay_rate}, {"use_relative_threshold", p.use_relative_threshold}, {"reset_error_on_compute", p.reset_error_on_compute}});
+ } else if (p.mode == SD_CACHE_DBCACHE || p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT) {
+ // TaylorSeer also uses the shared DBCache thresholds and limits.
+ info.update({{"Fn_compute_blocks", p.Fn_compute_blocks},
+ {"Bn_compute_blocks", p.Bn_compute_blocks},
+ {"residual_diff_threshold", p.residual_diff_threshold},
+ {"max_warmup_steps", p.max_warmup_steps},
+ {"max_cached_steps", p.max_cached_steps},
+ {"max_continuous_cached_steps", p.max_continuous_cached_steps}});
+ if (p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT)
+ info.update({{"taylorseer_n_derivatives", p.taylorseer_n_derivatives}, {"taylorseer_skip_interval", p.taylorseer_skip_interval}});
+ if (p.scm_mask && *p.scm_mask)
+ info.update({{"scm_mask", p.scm_mask}, {"scm_policy_dynamic", p.scm_policy_dynamic}});
+ } else if (p.mode == SD_CACHE_SPECTRUM) {
+ info.update({{"spectrum_w", p.spectrum_w},
+ {"spectrum_m", p.spectrum_m},
+ {"spectrum_lam", p.spectrum_lam},
+ {"spectrum_window_size", p.spectrum_window_size},
+ {"spectrum_flex_window", p.spectrum_flex_window},
+ {"spectrum_warmup_steps", p.spectrum_warmup_steps},
+ {"spectrum_stop_percent", p.spectrum_stop_percent}});
+ }
+ return info;
+ }
+
+ template <typename Params>
+ static json common_info(const Params& p, const char* kind) {
+ json info = {{"schema", "sdcpp.generation/v1"}, {"kind", kind}, {"prompt", text(p.prompt)}, {"negative_prompt", text(p.negative_prompt)}, {"width", p.width}, {"height", p.height}, {"clip_skip", p.clip_skip}, {"seed", std::to_string(p.seed)}, {"sample_params", sampling_info(p.sample_params)}};
+ if (p.hires.enabled)
+ info["hires"] = hires_info(p.hires);
+ if (p.cache.mode != SD_CACHE_DISABLED)
+ info["cache"] = cache_info(p.cache);
+ if (p.circular_x)
+ info["circular_x"] = true;
+ if (p.circular_y)
+ info["circular_y"] = true;
+ if (p.ref_image_args && *p.ref_image_args)
+ info["ref_image_args"] = p.ref_image_args;
+ if (p.image_preprocess.rules && *p.image_preprocess.rules)
+ info["image_preprocess"] = {{"rules", p.image_preprocess.rules}};
+ const auto& t = p.vae_tiling_params;
+ if (t.enabled || t.temporal_tiling) {
+ auto& tiling = info["vae_tiling_params"];
+ if (t.enabled)
+ tiling.update({{"enabled", true}, {"tile_size_w", t.tile_size_w}, {"tile_size_h", t.tile_size_h}, {"target_overlap", t.target_overlap}, {"rel_size_w", t.rel_size_w}, {"rel_size_h", t.rel_size_h}});
+ if (t.temporal_tiling)
+ tiling["temporal_tiling"] = true;
+ if (t.extra_tiling_args && *t.extra_tiling_args)
+ tiling["extra_tiling_args"] = t.extra_tiling_args;
+ }
+ if (p.lora_count > 0) {
+ info["loras"] = json::array();
+ for (uint32_t i = 0; i < p.lora_count; ++i)
+ info["loras"].push_back({{"path", text(p.loras[i].path)}, {"multiplier", p.loras[i].multiplier}, {"is_high_noise", p.loras[i].is_high_noise}});
+ }
+ return info;
+ }
+
+ static json image_info(const sd_img_gen_params_t& p) {
+ auto info = common_info(p, "image");
+ if (p.init_image.data)
+ info["strength"] = p.strength;
+ if (p.control_image.data)
+ info["control_strength"] = p.control_strength;
+ if (p.ip_adapter_image.data)
+ info["ip_adapter_strength"] = p.ip_adapter_strength;
+ if (p.pm_params.id_images && p.pm_params.id_images_count > 0) {
+ info["pm_params"] = {{"style_strength", p.pm_params.style_strength}};
+ if (p.pm_params.id_embed_path && *p.pm_params.id_embed_path)
+ info["pm_params"]["id_embed_path"] = p.pm_params.id_embed_path;
+ }
+ if (p.pulid_params.id_embedding_path && *p.pulid_params.id_embedding_path)
+ info["pulid_params"] = {{"id_embedding_path", p.pulid_params.id_embedding_path}, {"id_weight", p.pulid_params.id_weight}};
+ return info;
+ }
+
+ template <typename Params>
+ static Params resolved_params(const pipeline::GenerationRequest& request, Params p) {
+ p.width = request.width;
+ p.height = request.height;
+ p.seed = request.seed;
+ p.clip_skip = request.clip_skip;
+ p.sample_params = request.sample_params;
+ p.hires = request.hires;
+ p.vae_tiling_params = request.vae_tiling_params;
+ return p;
+ }
+
+ json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) {
+ auto p = resolved_params(request, params);
+ p.qwen_image_layers = request.qwen_image_layers;
+ auto info = image_info(p);
+ if (!request.models.empty())
+ info["models"] = request.models;
+ if (request.is_layered_image)
+ info["qwen_image_layers"] = p.qwen_image_layers;
+ return info;
+ }
+
+ json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) {
+ const auto p = resolved_params(request, params);
+ auto info = common_info(p, "video");
+ if (!request.models.empty())
+ info["models"] = request.models;
+ info.update({{"video_frames", request.frames}, {"fps", request.fps}});
+ if (p.init_image.data || p.end_image.data)
+ info["strength"] = p.strength;
+ if (request.use_vace)
+ info["vace_strength"] = p.vace_strength;
+ if (request.use_high_noise)
+ info.update({{"moe_boundary", p.moe_boundary}, {"high_noise_sample_params", sampling_info(request.high_noise_sample_params, true)}});
+ if (p.ref_videos_count > 0) {
+ info["ref_videos"] = json::array();
+ for (int i = 0; i < p.ref_videos_count; ++i)
+ info["ref_videos"].push_back({{"fps", p.ref_videos[i].fps}});
+ }
+ return info;
+ }
+
+ json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) {
+ auto inpaint = image_info(inpaint_params);
+ inpaint["strength"] = inpaint_params.strength;
+ // ADetailer clears these inputs before generating each crop.
+ for (const auto* key : {"control_strength", "pm_params", "pulid_params", "hires", "image_preprocess"})
+ inpaint.erase(key);
+ sd_img_gen_params_t defaults;
+ sd_img_gen_params_init(&defaults);
+ if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers)
+ inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers;
+ json adetailer = json::object();
+ if (params.prompt && *params.prompt)
+ adetailer["prompt"] = params.prompt;
+ if (params.negative_prompt && *params.negative_prompt)
+ adetailer["negative_prompt"] = params.negative_prompt;
+ if (params.extra_ad_args && *params.extra_ad_args)
+ adetailer["extra_ad_args"] = params.extra_ad_args;
+ return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"adetailer", adetailer}, {"inpaint_params", inpaint}};
+ }
+
+ json upscale_generation_info(uint32_t upscale_factor) {
+ return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}};
+ }
+
+ char* copy_generation_info(const json& info) {
+ const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace);
+ char* result = static_cast<char*>(malloc(text.size() + 1));
+ if (result)
+ memcpy(result, text.c_str(), text.size() + 1);
+ return result;
+ }
+
+ bool make_image_generation_result(sd_image_t* images, int count, const std::vector<json>& infos, sd_image_generation_result_t** result_out) {
+ if (!result_out || !images || count <= 0 || infos.size() != static_cast<size_t>(count)) {
+ free_sd_images(images, count);
+ return false;
+ }
+ auto* result = static_cast<sd_image_generation_result_t*>(calloc(1, sizeof(sd_image_generation_result_t)));
+ if (!result) {
+ free_sd_images(images, count);
+ return false;
+ }
+ result->items = static_cast<sd_image_result_t*>(calloc(count, sizeof(sd_image_result_t)));
+ if (!result->items) {
+ free(result);
+ free_sd_images(images, count);
+ return false;
+ }
+ result->count = count;
+ try {
+ for (int i = 0; i < count; ++i) {
+ result->items[i].image = images[i];
+ images[i] = {};
+ result->items[i].generation_info = copy_generation_info(infos[i]);
+ if (!result->items[i].generation_info) {
+ free_sd_images(images, count);
+ free_sd_image_generation_result(result);
+ return false;
+ }
+ }
+ } catch (...) {
+ free_sd_images(images, count);
+ free_sd_image_generation_result(result);
+ return false;
+ }
+ free(images);
+ *result_out = result;
+ return true;
+ }
+}
+
+void free_sd_image_generation_result(sd_image_generation_result_t* result) {
+ if (!result)
+ return;
+ for (int i = 0; i < result->count; ++i) {
+ free(result->items[i].image.data);
+ free(result->items[i].generation_info);
+ }
+ free(result->items);
+ free(result);
+}
+
+void free_sd_video_generation_result(sd_video_generation_result_t* result) {
+ if (!result)
+ return;
+ free_sd_images(result->frames, result->frame_count);
+ free_sd_audio(result->audio);
+ free(result->generation_info);
+ free(result);
+}
diff --git a/src/generation_info.h b/src/generation_info.h
new file mode 100644
index 0000000..56688ee
--- /dev/null
+++ b/src/generation_info.h
@@ -0,0 +1,23 @@
+#ifndef __SD_GENERATION_INFO_H__
+#define __SD_GENERATION_INFO_H__
+
+#include <vector>
+
+#include "json.hpp"
+#include "stable-diffusion.h"
+
+namespace sd::pipeline {
+ struct GenerationRequest;
+}
+
+namespace sd {
+ nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params);
+ nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params);
+ nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params);
+ nlohmann::json upscale_generation_info(uint32_t upscale_factor);
+ char* copy_generation_info(const nlohmann::json& info);
+ // Consumes images even on failure.
+ bool make_image_generation_result(sd_image_t* images, int count, const std::vector<nlohmann::json>& infos, sd_image_generation_result_t** result_out);
+}
+
+#endif // __SD_GENERATION_INFO_H__
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
index 6a1ff35..f44580c 100644
--- a/src/pipeline/diffusion_engine.cpp
+++ b/src/pipeline/diffusion_engine.cpp
@@ -675,87 +675,93 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
const auto* sd_ctx_params = &configuration.params;
auto& use_tae = configuration.use_tae;
auto& use_audio_vae = configuration.use_audio_vae;
+ const auto load_model = [&](const char* name, const char* path, const std::string& prefix = "") {
+ if (!model_loader.init_from_file(path, prefix))
+ return false;
+ configuration.record_model(name, path);
+ return true;
+ };
if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
- if (!model_loader.init_from_file(sd_ctx_params->model_path)) {
+ if (!load_model("model", sd_ctx_params->model_path)) {
LOG_ERROR("init model loader from file failed: '%s'", sd_ctx_params->model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->diffusion_model_path)) > 0) {
LOG_INFO("loading diffusion model from '%s'", sd_ctx_params->diffusion_model_path);
- if (!model_loader.init_from_file(sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
+ if (!load_model("diffusion_model", sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
LOG_INFO("loading high noise diffusion model from '%s'", sd_ctx_params->high_noise_diffusion_model_path);
- if (!model_loader.init_from_file(sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
+ if (!load_model("high_noise_diffusion_model", sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->high_noise_diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->uncond_diffusion_model_path)) > 0) {
LOG_INFO("loading unconditional diffusion model from '%s'", sd_ctx_params->uncond_diffusion_model_path);
- if (!model_loader.init_from_file(sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
+ if (!load_model("uncond_diffusion_model", sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
LOG_WARN("loading unconditional diffusion model from '%s' failed", sd_ctx_params->uncond_diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) {
LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path);
- if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, "clip_l.")) {
+ if (!load_model("clip_l", sd_ctx_params->clip_l_path, "clip_l.")) {
LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) {
LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path);
- if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, "clip_g.")) {
+ if (!load_model("clip_g", sd_ctx_params->clip_g_path, "clip_g.")) {
LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) {
LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path);
- if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, "clip_vision.")) {
+ if (!load_model("clip_vision", sd_ctx_params->clip_vision_path, "clip_vision.")) {
LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->t5xxl_path)) > 0) {
LOG_INFO("loading t5xxl from '%s'", sd_ctx_params->t5xxl_path);
- if (!model_loader.init_from_file(sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
+ if (!load_model("t5xxl", sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
LOG_WARN("loading t5xxl from '%s' failed", sd_ctx_params->t5xxl_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) {
LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path);
- if (!model_loader.init_from_file(sd_ctx_params->pulid_weights_path,
- "model.diffusion_model.")) {
+ if (!load_model("pulid_weights", sd_ctx_params->pulid_weights_path,
+ "model.diffusion_model.")) {
LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) {
LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path);
- if (!model_loader.init_from_file(sd_ctx_params->llm_path, "text_encoders.llm.")) {
+ if (!load_model("llm", sd_ctx_params->llm_path, "text_encoders.llm.")) {
LOG_WARN("loading llm from '%s' failed", sd_ctx_params->llm_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->llm_vision_path)) > 0) {
LOG_INFO("loading llm vision from '%s'", sd_ctx_params->llm_vision_path);
- if (!model_loader.init_from_file(sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
+ if (!load_model("llm_vision", sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
LOG_WARN("loading llm vision from '%s' failed", sd_ctx_params->llm_vision_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->vae_path)) > 0) {
LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
- if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) {
+ if (!load_model("vae", sd_ctx_params->vae_path, "vae.")) {
LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
external_vae_is_invalid = true;
}
@@ -763,7 +769,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->taesd_path)) > 0) {
LOG_INFO("loading tae from '%s'", sd_ctx_params->taesd_path);
- if (!model_loader.init_from_file(sd_ctx_params->taesd_path, "tae.")) {
+ if (!load_model("taesd", sd_ctx_params->taesd_path, "tae.")) {
LOG_WARN("loading tae from '%s' failed", sd_ctx_params->taesd_path);
} else {
use_tae = true;
@@ -772,14 +778,14 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) {
LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path);
- if (!model_loader.init_from_file(sd_ctx_params->embeddings_connectors_path)) {
+ if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) {
LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->audio_vae_path)) > 0) {
LOG_INFO("loading audio VAE from '%s'", sd_ctx_params->audio_vae_path);
- if (!model_loader.init_from_file(sd_ctx_params->audio_vae_path)) {
+ if (!load_model("audio_vae", sd_ctx_params->audio_vae_path)) {
LOG_WARN("loading audio VAE weights from '%s' failed", sd_ctx_params->audio_vae_path);
} else {
use_audio_vae = true;
@@ -788,15 +794,15 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->audio_encoder_path)) > 0) {
LOG_INFO("loading audio encoder (wav2vec2) from '%s'", sd_ctx_params->audio_encoder_path);
- if (!model_loader.init_from_file(sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
+ if (!load_model("audio_encoder", sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
LOG_WARN("loading audio encoder weights from '%s' failed", sd_ctx_params->audio_encoder_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->motion_module_path)) > 0) {
LOG_INFO("loading motion module (AnimateDiff) from '%s'", sd_ctx_params->motion_module_path);
- if (!model_loader.init_from_file(sd_ctx_params->motion_module_path,
- "model.diffusion_model.motion_module.")) {
+ if (!load_model("motion_module", sd_ctx_params->motion_module_path,
+ "model.diffusion_model.motion_module.")) {
LOG_WARN("loading motion module from '%s' failed", sd_ctx_params->motion_module_path);
} else {
configuration.animatediff_loaded = true;
@@ -810,8 +816,11 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
}
}
+ if (configuration.control_net_file)
+ configuration.record_model("control_net", sd_ctx_params->control_net_path);
+
if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) {
- if (!model_loader.init_from_file(sd_ctx_params->ip_adapter_path)) {
+ if (!load_model("ip_adapter", sd_ctx_params->ip_adapter_path)) {
LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path);
return false;
}
@@ -819,6 +828,8 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->photo_maker_path)) > 0) {
configuration.photomaker_source_available = model_loader.add_file(sd_ctx_params->photo_maker_path, "pmid.");
+ if (configuration.photomaker_source_available)
+ configuration.record_model("photo_maker", sd_ctx_params->photo_maker_path);
if (!configuration.photomaker_source_available) {
LOG_WARN("loading stacked ID embedding from '%s' failed", sd_ctx_params->photo_maker_path);
}
diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h
index 1e41662..c7295d8 100644
--- a/src/pipeline/diffusion_engine.h
+++ b/src/pipeline/diffusion_engine.h
@@ -123,6 +123,7 @@ public:
sd_ctx_params_t params{};
std::list<std::string> strings;
std::vector<sd_embedding_t> embeddings;
+ std::map<std::string, std::string> models;
ModelLoader::FileId control_net_file = 0;
bool use_tae = false;
bool use_audio_vae = false;
@@ -164,13 +165,24 @@ public:
use_audio_vae = other.use_audio_vae;
photomaker_source_available = other.photomaker_source_available;
animatediff_loaded = other.animatediff_loaded;
+ models = other.models;
}
ModelConfig& operator=(const ModelConfig&) = delete;
+ void record_model(const std::string& name, const std::string& path) {
+ if (path.empty()) {
+ models.erase(name);
+ return;
+ }
+ const auto separator = path.find_last_of("/\\");
+ models[name] = separator == std::string::npos ? path : path.substr(separator + 1);
+ }
+
void set_control_net(ModelLoader::FileId id, const std::string& path) {
control_net_file = id;
strings.push_back(path);
params.control_net_path = strings.back().c_str();
+ record_model("control_net", path);
}
};
diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h
index 34aa000..0bc3913 100644
--- a/src/pipeline/generation.h
+++ b/src/pipeline/generation.h
@@ -2,6 +2,7 @@
#define __SD_PIPELINE_GENERATION_H__
#include "conditioning/conditioner.hpp"
+#include "json.hpp"
#include "stable-diffusion.h"
class StableDiffusionGGML;
@@ -54,14 +55,16 @@ namespace sd::pipeline {
bool generate_image(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
sd_image_t** images_out,
- int* num_images_out);
+ int* num_images_out,
+ std::vector<nlohmann::json>* infos_out = nullptr);
bool generate_video(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
sd_image_t** frames_out,
int* num_frames_out,
sd_audio_t** audio_out,
- int* fps_out);
+ int* fps_out,
+ nlohmann::json* info_out = nullptr);
sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
const char* model_path,
diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp
index 046e9cf..c549280 100644
--- a/src/pipeline/image.cpp
+++ b/src/pipeline/image.cpp
@@ -1,4 +1,5 @@
#include "generation.h"
+#include "generation_info.h"
#include <algorithm>
#include <cmath>
@@ -668,7 +669,7 @@ namespace sd::pipeline {
int64_t t4 = ggml_time_ms();
LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000);
- if (decoded_images.empty()) {
+ if (decoded_images.empty() || cancelled) {
LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images");
return nullptr;
}
@@ -829,7 +830,10 @@ namespace sd::pipeline {
bool generate_image(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
sd_image_t** images_out,
- int* num_images_out) {
+ int* num_images_out,
+ std::vector<nlohmann::json>* infos_out) {
+ if (infos_out)
+ infos_out->clear();
if (images_out != nullptr) {
*images_out = nullptr;
}
@@ -851,6 +855,7 @@ namespace sd::pipeline {
int64_t t0 = ggml_time_ms();
sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
GenerationRequest request(sd, sd_img_gen_params);
+ const auto info = sd::generation_info(request, *sd_img_gen_params);
sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
sd_img_gen_params_t processed_params = *sd_img_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -891,7 +896,6 @@ namespace sd::pipeline {
return false;
}
ImageGenerationEmbeds embeds = std::move(*embeds_opt);
-
std::vector<sd::Tensor<float>> final_latents;
int64_t denoise_start = ggml_time_ms();
for (int b = 0; b < request.batch_count; b++) {
@@ -908,7 +912,7 @@ namespace sd::pipeline {
}
int64_t sampling_start = ggml_time_ms();
- int64_t cur_seed = request.seed + b;
+ int64_t cur_seed = request.seed_for_batch(b);
LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed);
sd->rng->manual_seed(cur_seed);
@@ -994,7 +998,7 @@ namespace sd::pipeline {
std::vector<float> hires_sigma_sched =
make_hires_sigma_schedule(sd,
request.hires,
- sd_img_gen_params->sample_params,
+ request.sample_params,
plan.sample_method,
plan.sample_steps,
sd->get_image_seq_len(request.hires.target_height, request.hires.target_width),
@@ -1035,7 +1039,7 @@ namespace sd::pipeline {
LOG_ERROR("cancelling generation during hires fix");
return false;
}
- int64_t cur_seed = request.seed + b;
+ int64_t cur_seed = request.seed_for_batch(b);
sd->rng->manual_seed(cur_seed);
sd->sampler_rng->manual_seed(cur_seed);
@@ -1113,6 +1117,19 @@ namespace sd::pipeline {
return false;
}
+ if (infos_out) {
+ try {
+ infos_out->assign(num_images, info);
+ // Layers and frames decoded from the same latent share its seed.
+ const int images_per_latent = num_images / static_cast<int>(final_latents.size());
+ for (int i = 0; i < num_images; ++i) {
+ (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent));
+ }
+ } catch (...) {
+ free_sd_images(result, num_images);
+ throw;
+ }
+ }
sd->lora_stat();
int64_t t1 = ggml_time_ms();
diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp
index 0693532..2647efb 100644
--- a/src/pipeline/request.cpp
+++ b/src/pipeline/request.cpp
@@ -1,4 +1,5 @@
#include "request.h"
+#include "conditioning/conditioner.hpp"
#include <algorithm>
#include <cmath>
@@ -6,6 +7,8 @@
#include <ctime>
#include "diffusion_engine.h"
+#include "generation.h"
+#include "model/vae/vae.hpp"
#include "model_definition.h"
#include "runtime/denoiser.hpp"
@@ -66,7 +69,7 @@ namespace sd::pipeline {
return DISCRETE_SCHEDULER;
}
- static int64_t resolve_seed(int64_t seed) {
+ int64_t resolve_seed(int64_t seed) {
if (seed >= 0) {
return seed;
}
@@ -127,6 +130,7 @@ namespace sd::pipeline {
seed = sd_img_gen_params->seed;
batch_count = sd_img_gen_params->batch_count;
qwen_image_layers = std::max(0, sd_img_gen_params->qwen_image_layers);
+ is_layered_image = sd->version == VERSION_QWEN_IMAGE_LAYERED;
clip_skip = sd_img_gen_params->clip_skip;
shifted_timestep = sd_img_gen_params->sample_params.shifted_timestep;
strength = sd_img_gen_params->strength;
@@ -137,8 +141,18 @@ namespace sd::pipeline {
pm_params = sd_img_gen_params->pm_params;
pulid_params = sd_img_gen_params->pulid_params;
hires = sd_img_gen_params->hires;
+ vae_tiling_params = sd_img_gen_params->vae_tiling_params;
cache_params = &sd_img_gen_params->cache;
+ sample_params = sd_img_gen_params->sample_params;
resolve(sd);
+ if (!sd_img_gen_params->control_image.data)
+ models.erase("control_net");
+ if (!sd_img_gen_params->ip_adapter_image.data)
+ models.erase("ip_adapter");
+ if (!pm_params.id_images || pm_params.id_images_count <= 0)
+ models.erase("photo_maker");
+ if (!pulid_params.id_embedding_path || !*pulid_params.id_embedding_path)
+ models.erase("pulid_weights");
}
GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
@@ -166,7 +180,20 @@ namespace sd::pipeline {
guidance = sd_vid_gen_params->sample_params.guidance;
high_noise_guidance = sd_vid_gen_params->high_noise_sample_params.guidance;
hires = sd_vid_gen_params->hires;
+ vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
+ sample_params = sd_vid_gen_params->sample_params;
+ high_noise_sample_params = sd_vid_gen_params->high_noise_sample_params;
+ use_high_noise = sd->high_noise_diffusion_model != nullptr;
+ use_vace = sd->diffusion_model && (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
+ sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
+ shifted_timestep = sample_params.shifted_timestep;
+ if (sd->config_ && sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version))
+ frames = std::min(requested_frames, 32);
resolve(sd);
+ models.erase("control_net");
+ models.erase("ip_adapter");
+ models.erase("photo_maker");
+ models.erase("pulid_weights");
if (frames != requested_frames) {
LOG_WARN("align video frames from %d to %d for %s",
requested_frames,
@@ -201,6 +228,10 @@ namespace sd::pipeline {
spatial_multiple);
}
+ int64_t GenerationRequest::seed_for_batch(int index) const {
+ return static_cast<int64_t>((static_cast<uint64_t>(seed) + index) & INT64_MAX);
+ }
+
void GenerationRequest::resolve_hires() {
if (!hires.enabled) {
return;
@@ -307,6 +338,23 @@ namespace sd::pipeline {
}
void GenerationRequest::resolve(StableDiffusionGGML* sd) {
+ if (sd->config_) {
+ models = sd->config_->models;
+ if (!use_high_noise)
+ models.erase("high_noise_diffusion_model");
+ if (!sd->clip_vision)
+ models.erase("clip_vision");
+ if (!sd->audio_vae_model || frames <= 0)
+ models.erase("audio_vae");
+ if (!sd->audio_encoder || frames <= 0)
+ models.erase("audio_encoder");
+ if (!sd->config_->animatediff_loaded || !sd_version_supports_animatediff(sd->version) || frames <= 0)
+ models.erase("motion_module");
+ if (sd->uses_tae() && !sd->tae_preview_only())
+ models.erase("vae");
+ else
+ models.erase("taesd");
+ }
if (sd_version_is_minit2i(sd->version) && (width != 512 || height != 512)) {
width = 512;
height = 512;
@@ -314,7 +362,11 @@ namespace sd::pipeline {
}
align_generation_request_size();
resolve_hires();
+ if (sd->first_stage_model)
+ vae_tiling_params = sd->first_stage_model->resolve_tiling_params(vae_tiling_params);
seed = resolve_seed(seed);
+ if (sd->cond_stage_model)
+ clip_skip = sd->cond_stage_model->resolve_clip_skip(clip_skip);
resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
if (sd->high_noise_diffusion_model) {
@@ -330,33 +382,44 @@ namespace sd::pipeline {
LOG_WARN("timestep shifting is only supported for SDXL models!");
shifted_timestep = 0;
}
+ sample_params.guidance = guidance;
+ sample_params.shifted_timestep = shifted_timestep;
+ sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method);
+ sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method);
+ sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method);
+ if (frames > 0) {
+ high_noise_sample_params.guidance = high_noise_guidance;
+ high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
+ high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method);
+ high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
+ }
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
const GenerationRequest& request) {
- sample_method = sd_img_gen_params->sample_params.sample_method;
- extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args;
- eta = sd_img_gen_params->sample_params.eta;
- sample_steps = sd_img_gen_params->sample_params.sample_steps;
- resolve(sd, &request, &sd_img_gen_params->sample_params);
+ sample_method = request.sample_params.sample_method;
+ extra_sample_args = request.sample_params.extra_sample_args;
+ eta = request.sample_params.eta;
+ sample_steps = request.sample_params.sample_steps;
+ resolve(sd, &request, &request.sample_params);
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request) {
- sample_method = sd_vid_gen_params->sample_params.sample_method;
- extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args;
- eta = sd_vid_gen_params->sample_params.eta;
- sample_steps = sd_vid_gen_params->sample_params.sample_steps;
+ sample_method = request.sample_params.sample_method;
+ extra_sample_args = request.sample_params.extra_sample_args;
+ eta = request.sample_params.eta;
+ sample_steps = request.sample_params.sample_steps;
if (sd->high_noise_diffusion_model) {
- high_noise_sample_steps = sd_vid_gen_params->high_noise_sample_params.sample_steps;
- high_noise_sample_method = sd_vid_gen_params->high_noise_sample_params.sample_method;
- high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args;
- high_noise_eta = sd_vid_gen_params->high_noise_sample_params.eta;
+ high_noise_sample_steps = request.high_noise_sample_params.sample_steps;
+ high_noise_sample_method = request.high_noise_sample_params.sample_method;
+ high_noise_extra_sample_args = request.high_noise_sample_params.extra_sample_args;
+ high_noise_eta = request.high_noise_sample_params.eta;
}
moe_boundary = sd_vid_gen_params->moe_boundary;
- resolve(sd, &request, &sd_vid_gen_params->sample_params);
+ resolve(sd, &request, &request.sample_params);
}
void SamplePlan::resolve(StableDiffusionGGML* sd,
@@ -382,10 +445,10 @@ namespace sd::pipeline {
}
}
} else {
- scheduler_t scheduler = resolve_scheduler(sd,
- sample_params->scheduler,
- sample_method);
- int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
+ scheduler = resolve_scheduler(sd,
+ sample_params->scheduler,
+ sample_method);
+ int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
int latent_frames = ((request->frames - 1) / 8) + 1;
sample_seq_len *= latent_frames;
diff --git a/src/pipeline/request.h b/src/pipeline/request.h
index fac4aba..1cf1cee 100644
--- a/src/pipeline/request.h
+++ b/src/pipeline/request.h
@@ -1,6 +1,7 @@
#ifndef __SD_PIPELINE_REQUEST_H__
#define __SD_PIPELINE_REQUEST_H__
+#include <map>
#include <string>
#include <vector>
@@ -16,6 +17,8 @@ namespace sd::pipeline {
enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
+ int64_t resolve_seed(int64_t seed);
+
float resolve_eta(StableDiffusionGGML* sd,
float eta,
enum sample_method_t sample_method);
@@ -23,38 +26,47 @@ namespace sd::pipeline {
struct GenerationRequest {
std::string prompt;
std::string negative_prompt;
- int width = -1;
- int height = -1;
- int clip_skip = -1;
- int vae_scale_factor = -1;
- int diffusion_model_down_factor = -1;
- int64_t seed = -1;
- bool use_uncond = false;
- bool use_img_uncond = false;
- bool use_high_noise_uncond = false;
- bool use_high_noise_img_uncond = false;
- bool has_ref_images = false;
- const sd_cache_params_t* cache_params = nullptr;
- int batch_count = 1;
- int qwen_image_layers = 3;
- int shifted_timestep = 0;
- float strength = 1.f;
- float control_strength = 0.f;
- float eta = 0.f;
- sd_guidance_params_t guidance = {};
- sd_guidance_params_t high_noise_guidance = {};
- sd_pm_params_t pm_params = {};
- sd_pulid_params_t pulid_params = {};
- sd_hires_params_t hires = {};
- int frames = -1;
- int requested_frames = -1;
- int fps = 16;
- float vace_strength = 1.f;
+ std::map<std::string, std::string> models;
+ int width = -1;
+ int height = -1;
+ int clip_skip = -1;
+ int vae_scale_factor = -1;
+ int diffusion_model_down_factor = -1;
+ int64_t seed = -1;
+ bool use_uncond = false;
+ bool use_img_uncond = false;
+ bool use_high_noise_uncond = false;
+ bool use_high_noise_img_uncond = false;
+ bool has_ref_images = false;
+ bool is_layered_image = false;
+ bool use_high_noise = false;
+ bool use_vace = false;
+ const sd_cache_params_t* cache_params = nullptr;
+ int batch_count = 1;
+ int qwen_image_layers = 3;
+ int shifted_timestep = 0;
+ float strength = 1.f;
+ float control_strength = 0.f;
+ float eta = 0.f;
+ sd_sample_params_t sample_params = {};
+ sd_sample_params_t high_noise_sample_params = {};
+ sd_guidance_params_t guidance = {};
+ sd_guidance_params_t high_noise_guidance = {};
+ sd_pm_params_t pm_params = {};
+ sd_pulid_params_t pulid_params = {};
+ sd_hires_params_t hires = {};
+ sd_tiling_params_t vae_tiling_params = {};
+ int frames = -1;
+ int requested_frames = -1;
+ int fps = 16;
+ float vace_strength = 1.f;
GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
+ int64_t seed_for_batch(int index) const;
+
void align_generation_request_size();
void align_image_size(int* target_width, int* target_height, const char* label);
@@ -81,6 +93,7 @@ namespace sd::pipeline {
int sample_steps = 0;
int high_noise_sample_steps = 0;
int total_steps = 0;
+ scheduler_t scheduler = SCHEDULER_COUNT;
float moe_boundary = 0.f;
std::vector<float> sigmas;
diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp
index 9aef07b..a07b931 100644
--- a/src/pipeline/video.cpp
+++ b/src/pipeline/video.cpp
@@ -1,4 +1,5 @@
#include "generation.h"
+#include "generation_info.h"
#include <algorithm>
#include <cmath>
@@ -1575,7 +1576,13 @@ namespace sd::pipeline {
img_gen_params.ref_image_args = sd_vid_gen_params->ref_image_args;
sd->animatediff_num_frames = n_frames;
- bool ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
+ bool ok = false;
+ try {
+ ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
+ } catch (...) {
+ sd->animatediff_num_frames = 0;
+ throw;
+ }
sd->animatediff_num_frames = 0;
return ok;
}
@@ -1585,14 +1592,24 @@ namespace sd::pipeline {
sd_image_t** frames_out,
int* num_frames_out,
sd_audio_t** audio_out,
- int* fps_out) {
+ int* fps_out,
+ nlohmann::json* info_out) {
+ if (info_out)
+ *info_out = nlohmann::json();
+ GenerationRequest request(sd, sd_vid_gen_params);
+ const auto info = sd::generation_info(request, *sd_vid_gen_params);
if (sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) {
LOG_INFO("AnimateDiff dispatch: %d frames, %dx%d",
sd_vid_gen_params->video_frames, sd_vid_gen_params->width, sd_vid_gen_params->height);
if (fps_out != nullptr) {
*fps_out = std::max(1, sd_vid_gen_params->fps);
}
- return generate_animatediff_video(sd, sd_vid_gen_params, frames_out, num_frames_out);
+ auto params = *sd_vid_gen_params;
+ params.seed = request.seed;
+ const bool ok = generate_animatediff_video(sd, ¶ms, frames_out, num_frames_out);
+ if (ok && info_out)
+ *info_out = info;
+ return ok;
}
sd->reset_cancel_flag();
@@ -1602,7 +1619,6 @@ namespace sd::pipeline {
int64_t t0 = ggml_time_ms();
sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
- GenerationRequest request(sd, sd_vid_gen_params);
sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
sd_vid_gen_params_t processed_params = *sd_vid_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -1734,9 +1750,9 @@ namespace sd::pipeline {
embeds.img_uncond,
sd::Tensor<float>(),
0.f,
- sd_vid_gen_params->sample_params.guidance,
+ request.guidance,
plan.eta,
- sd_vid_gen_params->sample_params.shifted_timestep,
+ request.shifted_timestep,
plan.sample_method,
sd->is_flow_denoiser(),
plan.extra_sample_args,
@@ -1846,7 +1862,7 @@ namespace sd::pipeline {
std::vector<float> hires_sigma_sched =
make_hires_sigma_schedule(sd,
request.hires,
- sd_vid_gen_params->sample_params,
+ request.sample_params,
hires_sample_method,
plan.sample_steps,
sd->get_image_seq_len(hires_request.height, hires_request.width) * T,
@@ -1873,9 +1889,9 @@ namespace sd::pipeline {
embeds.img_uncond,
sd::Tensor<float>(),
0.f,
- sd_vid_gen_params->sample_params.guidance,
+ request.guidance,
hires_eta,
- sd_vid_gen_params->sample_params.shifted_timestep,
+ request.shifted_timestep,
hires_sample_method,
sd->is_flow_denoiser(),
plan.extra_sample_args,
@@ -2007,6 +2023,14 @@ namespace sd::pipeline {
generated_audio->sample_count = want_samples;
}
}
+ if (info_out) {
+ try {
+ *info_out = info;
+ } catch (...) {
+ free_sd_audio(generated_audio);
+ throw;
+ }
+ }
if (audio_out != nullptr) {
*audio_out = generated_audio;
} else {
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index 00c5e8d..bca4b79 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -1,4 +1,5 @@
#include "stable-diffusion.h"
+#include "generation_info.h"
#include <algorithm>
#include <cinttypes>
@@ -752,43 +753,62 @@ enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_me
SD_API bool generate_image(sd_ctx_t* sd_ctx,
const sd_img_gen_params_t* params,
- sd_image_t** images_out,
- int* num_images_out) {
- if (images_out != nullptr)
- *images_out = nullptr;
- if (num_images_out != nullptr)
- *num_images_out = 0;
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || params == nullptr) {
+ sd_image_generation_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
return false;
- }
StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
- return execution.ready && sd::pipeline::generate_image(sd_ctx->sd, params, images_out, num_images_out);
+ if (!execution.ready)
+ return false;
+ sd_image_t* images = nullptr;
+ int count = 0;
+ std::vector<nlohmann::json> infos;
+ try {
+ if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) {
+ free_sd_images(images, count);
+ return false;
+ }
+ } catch (const std::exception& e) {
+ LOG_ERROR("image generation failed: %s", e.what());
+ free_sd_images(images, count);
+ return false;
+ }
+ return sd::make_image_generation_result(images, count, infos, result_out);
}
SD_API bool generate_video(sd_ctx_t* sd_ctx,
- const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_image_t** frames_out,
- int* num_frames_out,
- sd_audio_t** audio_out,
- int* fps_out) {
- if (frames_out != nullptr)
- *frames_out = nullptr;
- if (audio_out != nullptr)
- *audio_out = nullptr;
- if (num_frames_out != nullptr)
- *num_frames_out = 0;
- if (fps_out != nullptr)
- *fps_out = 0;
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_vid_gen_params == nullptr) {
+ const sd_vid_gen_params_t* params,
+ sd_video_generation_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
return false;
- }
-
StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
- if (!execution.ready) {
+ if (!execution.ready)
+ return false;
+ auto* result = static_cast<sd_video_generation_result_t*>(calloc(1, sizeof(sd_video_generation_result_t)));
+ if (!result)
+ return false;
+ try {
+ nlohmann::json info;
+ if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count,
+ &result->audio, &result->fps, &info)) {
+ free_sd_video_generation_result(result);
+ return false;
+ }
+ result->generation_info = sd::copy_generation_info(info);
+ if (!result->generation_info) {
+ free_sd_video_generation_result(result);
+ return false;
+ }
+ } catch (const std::exception& e) {
+ LOG_ERROR("video generation failed: %s", e.what());
+ free_sd_video_generation_result(result);
return false;
}
-
- return sd::pipeline::generate_video(sd_ctx->sd, sd_vid_gen_params, frames_out, num_frames_out, audio_out, fps_out);
+ *result_out = result;
+ return true;
}
SD_API void free_sd_images(sd_image_t* result_images, int num_images) {
diff --git a/src/upscaler.cpp b/src/upscaler.cpp
index dfe5e4b..c9bbbcb 100644
--- a/src/upscaler.cpp
+++ b/src/upscaler.cpp
@@ -1,6 +1,7 @@
#include "upscaler.h"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
+#include "generation_info.h"
#include "model_loader.h"
#include "runtime/tiling.h"
#include "stable-diffusion.h"
@@ -259,14 +260,11 @@ upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str,
bool upscale(upscaler_ctx_t* upscaler_ctx,
sd_image_t input_image,
uint32_t upscale_factor,
- sd_image_t** images_out,
- int* num_images_out) {
- if (images_out != nullptr) {
- *images_out = nullptr;
- }
- if (num_images_out != nullptr) {
- *num_images_out = 0;
- }
+ sd_image_generation_result_t** result_out) try {
+ if (result_out)
+ *result_out = nullptr;
+ if (!result_out)
+ return false;
if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) {
return false;
}
@@ -276,21 +274,17 @@ bool upscale(upscaler_ctx_t* upscaler_ctx,
return false;
}
+ auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); };
+ std::unique_ptr<sd_image_t, decltype(release_images)> image_owner(result_images, release_images);
+ const std::vector<nlohmann::json> infos{sd::upscale_generation_info(upscale_factor)};
result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor);
- if (result_images[0].data == nullptr) {
- free(result_images);
+ if (!result_images[0].data)
return false;
- }
- if (num_images_out != nullptr) {
- *num_images_out = 1;
- }
- if (images_out != nullptr) {
- *images_out = result_images;
- } else {
- free_sd_images(result_images, 1);
- }
- return true;
+ return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out);
+} catch (const std::exception& e) {
+ LOG_ERROR("upscale failed: %s", e.what());
+ return false;
}
int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) {
@@ -329,6 +323,8 @@ int get_upscaler_model_scale(const char* model_path) {
}
void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) {
+ if (!upscaler_ctx)
+ return;
if (upscaler_ctx->upscaler != nullptr) {
delete upscaler_ctx->upscaler;
upscaler_ctx->upscaler = nullptr;