Commit 6cd0ef1 for stable-diffusion.cpp
commit 6cd0ef1e1ed435ebcefbb7d891374ed8074f21d1
Author: leejet <leejet714@gmail.com>
Date: Sun Nov 22 23:42:26 2026 +0800
feat: add Diffusion Graph Language support (#2396)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c40713..b4c954e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,6 +15,7 @@ on:
[
".github/workflows/**",
".dockerignore",
+ "docs/dgl-builtins.md",
"docker/Dockerfile*",
"scripts/ci-cuda-cache.sh",
"**/CMakeLists.txt",
@@ -38,6 +39,7 @@ on:
[
".github/workflows/**",
".dockerignore",
+ "docs/dgl-builtins.md",
"docker/Dockerfile*",
"scripts/ci-cuda-cache.sh",
"**/CMakeLists.txt",
@@ -100,6 +102,7 @@ jobs:
cd build
cmake .. -DSD_BUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH='$ORIGIN'
cmake --build . --config Release
+ cmake --build . --config Release --target dgl-check-builtins
- name: Get commit hash
id: commit
@@ -121,6 +124,8 @@ jobs:
run: |
cp ggml/LICENSE ./build/bin/ggml.txt
cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+ cp thirdparty/libwebm/LICENSE.TXT ./build/bin/libwebm.txt
+ cp thirdparty/libwebp/COPYING ./build/bin/libwebp.txt
chmod +x start-server.sh
zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat
@@ -186,6 +191,8 @@ jobs:
run: |
cp ggml/LICENSE ./build/bin/ggml.txt
cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+ cp thirdparty/libwebm/LICENSE.TXT ./build/bin/libwebm.txt
+ cp thirdparty/libwebp/COPYING ./build/bin/libwebp.txt
chmod +x start-server.sh
zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/* ./start-server.sh ./start-server.bat
@@ -366,6 +373,8 @@ jobs:
run: |
cp ggml/LICENSE ./build/bin/ggml.txt
cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+ cp thirdparty/libwebm/LICENSE.TXT ./build/bin/libwebm.txt
+ cp thirdparty/libwebp/COPYING ./build/bin/libwebp.txt
chmod +x start-server.sh
zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat
@@ -474,8 +483,11 @@ jobs:
ls .\build\bin
throw "Can't find stable-diffusion.dll"
}
+ $binPath = Split-Path -Parent $filePath
+ Copy-Item thirdparty/libwebm/LICENSE.TXT (Join-Path $binPath 'libwebm.txt')
+ Copy-Item thirdparty/libwebp/COPYING (Join-Path $binPath 'libwebp.txt')
if ("${{ matrix.build }}" -eq "cuda12") {
- Copy-Item thirdparty/cutlass/LICENSE.txt (Join-Path (Split-Path -Parent $filePath) 'cutlass.txt')
+ Copy-Item thirdparty/cutlass/LICENSE.txt (Join-Path $binPath 'cutlass.txt')
}
7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath .\start-server.sh .\start-server.bat
@@ -633,6 +645,8 @@ jobs:
- name: Pack artifacts
if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
run: |
+ Copy-Item thirdparty/libwebm/LICENSE.TXT .\build\bin\libwebm.txt
+ Copy-Item thirdparty/libwebp/COPYING .\build\bin\libwebp.txt
7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\* .\start-server.sh .\start-server.bat
- name: Upload artifacts
@@ -796,6 +810,8 @@ jobs:
run: |
cp ggml/LICENSE ./build/bin/ggml.txt
cp LICENSE ./build/bin/stable-diffusion.cpp.txt
+ cp thirdparty/libwebm/LICENSE.TXT ./build/bin/libwebm.txt
+ cp thirdparty/libwebp/COPYING ./build/bin/libwebp.txt
chmod +x start-server.sh
zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin ./start-server.sh ./start-server.bat
diff --git a/.gitignore b/.gitignore
index 13a7754..101dd41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,10 @@ build*/
cmake-build-*/
test/
tests/
+/dgl/*
+!/dgl/.gitkeep
logs/
+output/
.vscode/
.idea/
.cache/
diff --git a/AGENTS.md b/AGENTS.md
index ea93c92..9be6f7c 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -50,8 +50,11 @@ Core implementation lives under `src/`.
Current source layout includes:
+* `src/api/` - C API options, generation requests, and translation to DGL.
* `src/core/` - shared tensor, ggml integration, backend, graph, RNG, and utility
code.
+* `src/dgl/` - DGL compilation, execution, and built-in operations. See the
+ [DGL architecture guide](docs/development/dgl.md) for layer boundaries.
* `src/model/` - model families and model components.
* `src/model_io/` - model file loading, GGUF, safetensors, pickle, and related
serialization helpers.
@@ -92,6 +95,7 @@ documentation.
* `docs/` - documentation for supported models, build options, behavior, and
workflows; start from `docs/README.md`.
+* `docs/dgl/` - runnable DGL examples grouped by model.
* `docs/models/` - model and component usage guides, including LoRAs, adapters,
auxiliary models, and model-specific examples.
* `docs/guides/` - shared usage workflows, configuration, and performance guides.
@@ -127,6 +131,12 @@ documentation.
6. Keep edits scoped to the requested behavior.
7. Run the narrowest useful build, test, or inspection command available.
+Regenerate [the DGL built-in catalog](docs/dgl-builtins.md) whenever a
+registered function is added, removed, renamed, or changes its parameters,
+defaults, or outputs. Run the `dgl-update-builtins` CMake target to
+regenerate the catalog and `dgl-check-builtins` to verify it. Update the
+DGL guide or examples when a built-in changes behavior.
+
Follow `CONTRIBUTING.md` for formatting, naming, PR expectations, dependency
update policy, and security rules. For tokenizer additions, follow its embedded-data
allowlist and default to an external `tokenizer.json`.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 59107d4..550cc3d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,14 +9,8 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
endif()
if (MSVC)
- cmake_host_system_information(RESULT SD_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
- if (SD_LOGICAL_CORES GREATER 8)
- math(EXPR SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES} - 2")
- elseif (SD_LOGICAL_CORES GREATER 2)
- math(EXPR SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES} - 1")
- else()
- set(SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES}")
- endif()
+ cmake_host_system_information(RESULT SD_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES)
+ set(SD_DEFAULT_MSVC_MP_COUNT "${SD_PHYSICAL_CORES}")
if (NOT SD_DEFAULT_MSVC_MP_COUNT GREATER 0)
set(SD_DEFAULT_MSVC_MP_COUNT 1)
endif()
@@ -24,6 +18,7 @@ if (MSVC)
if (NOT SD_MSVC_MP_COUNT MATCHES "^[1-9][0-9]*$" OR SD_MSVC_MP_COUNT GREATER 65536)
message(FATAL_ERROR "SD_MSVC_MP_COUNT must be an integer between 1 and 65536")
endif()
+ message("-- SD_MSVC_MP_COUNT: ${SD_MSVC_MP_COUNT}")
add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
# /MP is MSVC-only: icx rejects it outright once offloading is enabled.
@@ -98,7 +93,7 @@ endif()
option(SD_BUILD_EXAMPLES "sd: build examples" ${SD_STANDALONE})
option(SD_WEBP "sd: enable WebP image I/O support" ${SD_WEBP_DEFAULT})
option(SD_USE_SYSTEM_WEBP "sd: link against system libwebp" OFF)
-option(SD_WEBM "sd: enable WebM video output support" ${SD_WEBM_DEFAULT})
+option(SD_WEBM "sd: enable WebM video I/O support" ${SD_WEBM_DEFAULT})
option(SD_USE_SYSTEM_WEBM "sd: link against system libwebm" OFF)
option(SD_CUDA "sd: cuda backend" OFF)
option(SD_HIPBLAS "sd: rocm backend" OFF)
@@ -184,13 +179,21 @@ if(SD_WEBP)
"Expected WebP::libwebpmux or WebP::webpmux."
)
endif()
+ if(TARGET WebP::webpdemux)
+ add_library(webpdemux ALIAS WebP::webpdemux)
+ elseif(TARGET WebP::libwebpdemux)
+ add_library(webpdemux ALIAS WebP::libwebpdemux)
+ else()
+ message(FATAL_ERROR "System WebP package must provide WebP::webpdemux or WebP::libwebpdemux.")
+ endif()
else()
find_package(PkgConfig REQUIRED)
pkg_check_modules(WebP REQUIRED IMPORTED_TARGET GLOBAL libwebp)
pkg_check_modules(WebPMux REQUIRED IMPORTED_TARGET GLOBAL libwebpmux)
- link_libraries(PkgConfig::WebP)
- link_libraries(PkgConfig::WebPMux)
+ pkg_check_modules(WebPDemux REQUIRED IMPORTED_TARGET GLOBAL libwebpdemux)
+ add_library(webp ALIAS PkgConfig::WebP)
add_library(libwebpmux ALIAS PkgConfig::WebPMux)
+ add_library(webpdemux ALIAS PkgConfig::WebPDemux)
endif()
endif()
endif()
@@ -210,7 +213,7 @@ if(SD_WEBM)
pkg_check_modules(WebM REQUIRED IMPORTED_TARGET GLOBAL libwebm)
endif()
if(PkgConfig_FOUND AND WebM_FOUND)
- link_libraries(PkgConfig::WebM)
+ add_library(webm ALIAS PkgConfig::WebM)
else()
find_path(WEBM_INCLUDE_DIR
NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
@@ -236,6 +239,17 @@ endif ()
set(SD_LIB stable-diffusion)
+file(GLOB SD_API_TEMPLATES CONFIGURE_DEPENDS "src/api/templates/*.dgl")
+set(SD_API_TEMPLATE_CONTENT "#ifndef __SD_API_TEMPLATES_H__\n#define __SD_API_TEMPLATES_H__\n#include <map>\n#include <string>\n#include <string_view>\nnamespace sd::api::templates {\ninline const std::map<std::string, std::string_view> sources = {\n")
+foreach(SD_API_TEMPLATE IN LISTS SD_API_TEMPLATES)
+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SD_API_TEMPLATE}")
+ get_filename_component(SD_API_TEMPLATE_NAME "${SD_API_TEMPLATE}" NAME_WE)
+ file(READ "${SD_API_TEMPLATE}" SD_API_TEMPLATE_TEXT)
+ string(APPEND SD_API_TEMPLATE_CONTENT "{\"${SD_API_TEMPLATE_NAME}\", R\"SD_DGL(${SD_API_TEMPLATE_TEXT})SD_DGL\"},\n")
+endforeach()
+string(APPEND SD_API_TEMPLATE_CONTENT "};\n}\n#endif\n")
+file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/generated/api_templates.h" CONTENT "${SD_API_TEMPLATE_CONTENT}")
+
file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
"src/*.h"
"src/*.cpp"
@@ -255,11 +269,19 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
"src/extensions/*.h"
"src/extensions/*.cpp"
"src/extensions/*.hpp"
+ "src/dgl/*.h"
+ "src/dgl/*.cpp"
+ "src/dgl/syntax/*.h"
+ "src/dgl/syntax/*.cpp"
+ "src/dgl/bindings/*.h"
+ "src/dgl/bindings/*.cpp"
+ "src/dgl/models/*.h"
+ "src/dgl/models/*.cpp"
+ "src/dgl/ops/*.h"
+ "src/dgl/ops/*.cpp"
"src/model/*/*.h"
"src/model/*/*.cpp"
"src/model/*/*.hpp"
- "src/pipeline/*.h"
- "src/pipeline/*.cpp"
"src/runtime/*.h"
"src/runtime/*.cpp"
"src/runtime/*.hpp"
@@ -363,6 +385,14 @@ target_link_libraries(${SD_LIB} PUBLIC ggml)
find_package(Threads REQUIRED)
target_link_libraries(${SD_LIB} PRIVATE Threads::Threads)
target_link_libraries(${SD_LIB} PRIVATE onig sd-utf8proc)
+if(SD_WEBP)
+ target_compile_definitions(${SD_LIB} PRIVATE SD_USE_WEBP)
+ target_link_libraries(${SD_LIB} PRIVATE webp webpdemux)
+endif()
+if(SD_WEBM)
+ target_compile_definitions(${SD_LIB} PRIVATE SD_USE_WEBM)
+ target_link_libraries(${SD_LIB} PRIVATE webm)
+endif()
if (SD_CUDA)
find_package(CUDAToolkit REQUIRED)
# Keep the driver stub on downstream link lines when no driver is installed.
@@ -430,7 +460,7 @@ if (SD_HIPBLAS)
endif()
endif()
target_include_directories(${SD_LIB} PUBLIC include)
-target_include_directories(${SD_LIB} PRIVATE src src/core)
+target_include_directories(${SD_LIB} PRIVATE src src/core "${CMAKE_CURRENT_BINARY_DIR}/generated")
target_include_directories(${SD_LIB} PUBLIC thirdparty)
target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
set_target_properties(${SD_LIB} PROPERTIES
@@ -444,6 +474,7 @@ endif()
+
#
# install
#
@@ -459,12 +490,9 @@ set(SD_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of h
set(SD_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files")
set(SD_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files")
-set(SD_PUBLIC_HEADERS
- ${CMAKE_CURRENT_SOURCE_DIR}/include/stable-diffusion.h)
-
set_target_properties(${SD_LIB}
PROPERTIES
- PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
+ PUBLIC_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/include/stable-diffusion.h")
install(TARGETS ${SD_LIB}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1b5cc65..ee2b9c3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -43,6 +43,12 @@ Link new documentation from [the documentation index](docs/README.md) or
[the model index](docs/models/README.md), as appropriate. Keep supported model
entries and their guide links in the project README.
+Use **DGL** for the language name in headings, prose, link text, UI labels,
+comments, logs, and diagnostics. Preserve literal spellings in commands, files,
+paths, and protocol or metadata fields, such as `-M dgl`, `.dgl`, `docs/dgl/`,
+and the metadata key `dgl`. Code identifiers follow their existing naming style,
+such as `sd_dgl_run`, `sd::dgl`, `SD_DGL_VALUE_TENSOR`, and `DglWorkspace`.
+
### Model Documentation
Every new model or component integration must include a user guide in
@@ -60,7 +66,9 @@ omitting other sections when they have no content:
- `Model files`, with component, weights/downloads, CLI option, and requirement
columns. Include tokenizers, projectors, connectors, downloads, conversion, and
required preprocessing.
-- `Usage`, with `Web UI` and `CLI` subsections where applicable. Link to the
+- `Usage`, with `Web UI`, `DGL`, and `Legacy CLI` subsections in that order
+ where applicable. Group DGL scripts in a task table with descriptive links;
+ put shared run instructions below the table. Link to the
server's shared Web UI guide and list compatible recipes/components. Group
complete CLI examples by task or variant within this section; do not add a
separate `Quick start` or `Examples` section. Keep API usage in its own
diff --git a/README.md b/README.md
index 6d62d57..e27eacb 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ API and command-line option may change frequently.***
- [Qwen Image 2.1](./docs/models/qwen_image_2.1.md)
- [PiD](./docs/models/pid.md)
- [LongCat Image](./docs/models/longcat_image.md)
- - [Z-Image](./docs/models/z_image.md)
+ - [Z-Image](./docs/models/z_image.md), including [DGL ControlNet](./docs/models/z_image.md#controlnet)
- [Z-Image L2P](./docs/models/z_image_l2p.md)
- [MiniT2I](./docs/models/minit2i.md)
- [P-H3-Image](./docs/models/p_h3_image.md)
@@ -213,6 +213,7 @@ Browse the [documentation index](./docs/README.md) or jump to a topic:
- [Getting started: build, Docker, CLI, and server](./docs/README.md#get-started)
- [Models and components](./docs/models/README.md)
- [Generation and inputs](./docs/README.md#generation-and-inputs)
+- [DGL generation scripts](./docs/dgl.md)
- [Devices and performance](./docs/README.md#devices-and-performance)
- [Weights and quantization](./docs/README.md#weights-and-quantization)
- [Troubleshooting](./docs/troubleshooting.md)
diff --git a/dgl/.gitkeep b/dgl/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/docs/README.md b/docs/README.md
index 5490146..a421455 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -15,6 +15,8 @@ For the project overview and supported models, see the [project README](../READM
## Generation and inputs
+- [DGL generation scripts](dgl.md)
+- [DGL model examples](dgl/README.md)
- [C API and migration](guides/c_api.md)
- [Sampling methods and schedulers](guides/sampling.md)
- [Image editing](guides/edit.md)
@@ -40,6 +42,7 @@ For the project overview and supported models, see the [project README](../READM
## Development
- [C API internals](development/c_api.md)
+- [DGL architecture](development/dgl.md)
- [Model configuration and integration conventions](development/model_config.md)
- [ggml operator extensions](development/ggml_extensions.md)
- [Backend performance probes](development/backend_probe.md)
diff --git a/docs/build.md b/docs/build.md
index 1533cfb..484eede 100644
--- a/docs/build.md
+++ b/docs/build.md
@@ -59,9 +59,22 @@ including `GGML_MAX_NAME`. For an upstream package, also set
`SD_USE_UPSTREAM_GGML=ON` and point `SD_GGML_SOURCE_DIR` to its matching source
tree for the compatibility layer's private headers.
-## WebP and WebM Support in Examples
-
-The example applications (`examples/cli` and `examples/server`) use `libwebp` to support WebP image I/O, and `examples/cli` can also use `libwebm` for `.webm` video output. Both are enabled by default. WebM output currently reuses `libwebp` to encode each frame as VP8 before muxing with `libwebm`.
+## WebP and WebM Support
+
+WebP and WebM support are enabled by default. The library links `libwebp`,
+`libwebpdemux` and `libwebm` so DGL `video::load` works through the C API as
+well as CLI and server. WebP input supports static images and animation;
+WebM input supports this project's independent VP8 frames and PCM16 audio.
+No libvpx dependency is required. See [DGL media I/O](dgl.md) for frame-rate
+and codec restrictions.
+
+The CLI and server also use `libwebp` for WebP images. CLI WebM output reuses
+`libwebp` to encode each frame as VP8 before muxing with `libwebm`. System WebP
+packages must provide the decoder, demuxer and muxer libraries.
+
+DGL `video::encode` supports silent WebM and AVI with PCM16 audio. Requesting
+WebM audio reports an unsupported-encoder error; use `format="avi"` to include
+an audio track.
If you do not want WebP/WebM support, you can disable them at configure time:
diff --git a/docs/development/backend_probe.md b/docs/development/backend_probe.md
index c420326..f40876d 100644
--- a/docs/development/backend_probe.md
+++ b/docs/development/backend_probe.md
@@ -37,10 +37,13 @@ measurement pair; it cannot interrupt an in-flight backend operation.
Only use `candidate_faster` when `completed` is true. On incomplete probes,
`error` reports why comparison failed and the timing totals may be partial.
The caller handles logging, known architecture rules, fallback policy, and
-context-lifetime caching. Keep caches separate by operation (or key a shared
-cache by both operation and backend); an attention result must not select a
-convolution implementation. If shapes or settings vary within one context,
-include them in the corresponding cache key as well.
+context-lifetime caching. DGL uses one `BackendProbeCache` shared by all models
+in an execution context, including LoRA variants and models loaded by later
+programs. It caches both enabled and disabled decisions by operation and compute
+backend; CPU attention additionally keys the effective thread count. GPU results
+survive changes to the context's thread count. The cache serializes first-time
+probes and their decision logs. Explicit operator `on` and `off` bypass the cache.
+If probe shapes or other relevant settings become variable, add them to the key.
Convolution results are cached separately for 2D and 3D, once per compute backend
in the context, just like attention results. Each probe logs its backend, decision,
diff --git a/docs/development/c_api.md b/docs/development/c_api.md
index 58587a6..b3f4da6 100644
--- a/docs/development/c_api.md
+++ b/docs/development/c_api.md
@@ -13,26 +13,30 @@ See the [C API guide](../guides/c_api.md) for keys, value types and ownership ru
| `src/core/media.h` | Private raw media views used by inference and codecs. |
| `src/api/result.h/.cpp` | Internal result storage, opaque result accessors, metadata attachment and media ownership. |
| `src/api/api.h/.cpp` | Context, generation, callback and query entry points, plus thread-local diagnostics and exception-to-status handling. |
-| `src/generation_info.h/.cpp` | Construction of generation metadata JSON. |
-| `src/pipeline/generation.h` | Internal generation entry declarations. |
-| `src/stable-diffusion.cpp` | Internal typed image/video generation and model-specific validation. |
+| `src/api/generation.*` | Persistent DGL context, metadata probing, request execution, callbacks and cancellation. |
+| `src/api/generation_program.*`, `generation_image.cpp`, `generation_video.cpp`, `generation_h3.cpp`, `generation_prompt.cpp` | Template selection, literal rendering and host input bindings. |
+| `src/api/generation_template.*` | Embedded template parsing, variable formatting and conditional rendering. |
+| `src/api/templates/` | Complete DGL execution flows and a few shared helpers. |
+| `src/api/program_result.cpp` | Adaptation of DGL tensors and dictionaries to C results. |
+| `src/api/generation_metadata.cpp` | Image parameter and WebUI infotext extraction through the metadata C API. |
+| `src/api/image_metadata.cpp` | Container metadata reading and text/JSON reports through `sd_read_metadata()`. |
+| `src/api/media_metadata.cpp`, `metadata.h` | Shared metadata encoding and C access to PNG/JPEG/XMP/AVI chunks. |
| `src/core/option_types.h/.cpp` | Internal mode enums, name lookup and parsing. |
-Options are validated and converted once at the API boundary. The runtime reads
-typed configuration fields. Internal callers such as ADetailer pass typed
-configuration directly to the internal generation entry; they do not rebuild
-options or call public setters.
+Options are validated and converted at the API boundary. The translator renders
+a concrete DGL program and binds media through `main` tensor inputs. All retained
+inference entries use the same DGL compiler, execution context and model cache.
+There is no separate native pipeline.
The option map alone records whether a key was supplied. Zero, false, empty
strings and explicit automatic choices remain entries until `unset` removes
them. Each conversion starts with a fresh `OptConfig`, then overlays supplied
values and validates combinations. Reusing the output configuration cannot
-retain a previous request's overrides. ADetailer applies its operation-specific
-strength default before the overlay.
+retain a previous request's overrides.
Fixed defaults live in the configuration constructors. Internal automatic
policies (such as model-selected sampler/scheduler and automatic eta/flow shift)
-are resolved by the pipeline after refreshing the model sources. Their internal
+are resolved by the translator after refreshing the model metadata. Their internal
sentinel values describe an automatic policy, not the presence of an input key.
Do not infer presence by comparing a typed value against its default.
@@ -46,18 +50,43 @@ resource policy. Parameter ranges are checked by the public setters used by the
option builders; combinations and model applicability are checked when executing.
Examples include only public library headers and do not query library defaults.
Capability defaults describe configured overrides; null means that the library
-selects the value, and result metadata reports effective values. Post-generation
-ADetailer clears inherited dimensions and strength before applying pass overrides,
-so its operation defaults also come from the library.
+selects the value, and the executed DGL records the effective values.
Result storage owns tensors and metadata. Tensor handles contain a dtype-specific
`sd::Tensor<T>` and an atomic reference count; options retain handles rather than
-copying tensor payloads. Raw pipeline output buffers are converted to tensors at
-the boundary and released there. Internal producers with an `sd::Tensor<T>` can
+copying tensor payloads. DGL outputs are adapted at the result boundary.
+Internal producers with an `sd::Tensor<T>` can
move it directly into a handle. Video results keep per-frame tensors and assemble
a contiguous video tensor only when requested. Cached assembly is synchronized.
Preview tensors are independent snapshots and may be retained by callbacks.
-Metadata construction is separate from its attachment to a result.
+Results retain the exact compiled DGL source. WebUI conversion and media metadata
+embedding are separate from inference and never rebuild the execution program.
+`src/api/generation_metadata.cpp` parses the source once to produce image
+parameters and WebUI infotext together. Examples obtain this JSON through
+`sd_get_generation_metadata()` without compiling or calling the DGL parser.
+Container reading and metadata chunk generation also use the public C API.
+Examples have no private `src` include path and retain generated DGL source
+without legacy JSON conversion or LoRA path rewriting.
+The translator uses an explicit `llm_path` as the text encoder's main `path`, falling back to
+the checkpoint and other configured encoder paths when it is absent.
+
+Context creation snapshots options without loading weights. Family probing is
+lazy and tracks actual file dependencies. The complete request stays busy during
+probing, compilation, batch gaps and execution. File permissions are checked
+before probing and on every DGL resource access, including cache hits.
+`sd_ctx_prepare()` runs a component-only template using the same loading
+parameters as generation. Server startup and model selection call it after
+installing file permissions, before publishing `ready`. Capability queries read
+an atomic snapshot of the last detected type and do not acquire a generation
+run or reset its cancellation state. Only an initial probe takes the context
+lock.
+
+C++ validates inputs, resolves model defaults, prepares geometry and binds host
+tensors. Templates describe complete execution flows and optional stages;
+operations and models retain computation algorithms. Split templates when the
+flow changes substantially, and use conditions for smaller variations. Do not
+search and patch rendered source to change execution. See the
+[template guide](../../src/api/templates/README.md).
These headers are private implementation details and are not installed as part
of the public C API.
diff --git a/docs/development/dgl.md b/docs/development/dgl.md
new file mode 100644
index 0000000..14a5f31
--- /dev/null
+++ b/docs/development/dgl.md
@@ -0,0 +1,69 @@
+# DGL architecture
+
+A DGL program describes a workflow in terms of model and tensor operations.
+The CLI can run a handwritten script. The generation C API renders a script
+from request parameters and binds input tensors to its entry point. Both paths
+use the same compiler and executor. The C API keeps the rendered source in
+`generation_info`. See the [language guide](../dgl.md) and
+[C API internals](c_api.md).
+
+## From source to execution
+
+1. **Compile the script.** The compiler parses the source, resolves calls and
+ arguments against the built-in registry or user functions, and checks their
+ types. It records which values each statement needs and when operations
+ use model weights. For a configuration-dependent `if`, it checks both
+ branches and records resource uses from the selected branch. This is enough
+ for `--check` to validate a program without loading model weights.
+2. **Plan the run.** With auto-fit enabled, the runtime combines the graph's
+ weight-use phases with model parameter sizes, weights already resident in
+ the context, and current RAM and device budgets. It chooses where to keep
+ weights and respects explicit backend and split settings. Models first
+ encountered inside an iteration join the plan when their loader executes.
+3. **Execute the graph.** The executor evaluates the bound calls in source
+ order and releases intermediate values after their last use. As execution
+ advances, the model manager uses the plan's next-use information to reclaim
+ idle weight buffers under memory pressure. The value returned by `main`
+ becomes the program result.
+
+## Built-ins, operations and models
+
+The registry describes a built-in's script name, parameters, defaults and
+return type. The compiler uses that contract to check calls. At execution,
+the binding converts script values to typed C++ arguments, calls an operation
+or model loader, and wraps its result for the script. The same registry supplies
+the [built-in function catalog](../dgl-builtins.md).
+
+Operations implement work such as sampling, tensor slicing and media decoding.
+They take concrete tensors, model objects, options and event interfaces. Models
+own their runners, weight registrations and model-specific inference. Operations
+can call models. Both use shared tensor, backend and runtime primitives. Script
+values, syntax, registry entries and executor state stay on the language side
+of the binding. Model code is independent of operations and bindings.
+
+For example, a sampling run can use a classifier-free guidance (CFG) predictor.
+
+```dgl
+pos = predictor(model, positive)
+guided = cfg(pos, predictor(model, negative), scale = 7)
+latent = sample(start, sigmas, predict = guided, rng = rng)
+```
+
+The first two statements describe which model and conditions to evaluate and how
+to combine the predictions. They keep the model reachable. The sampling
+operation drives the solver and evaluates that predictor at each required
+step. The resource plan therefore accounts for diffusion weights during
+`sample`. Each sampling call owns its own prediction history and progress state.
+
+## What the context retains
+
+A context owns backend settings, memory budgets and the model cache. A
+persistent context can reuse a loaded component across scripts when its
+normalized loader settings and source dependencies still match. A run-once
+context can release it after its last use. Releasing a script binding and
+evicting cached weights are therefore separate lifetime decisions.
+
+One context runs one program at a time. Each run gets a fresh placement plan
+and its own progress, preview and cancellation state. Two runs of the same
+source can choose different weight placement as cached resources or available
+memory change. See [context execution settings](../dgl.md#context-execution-settings).
diff --git a/docs/development/model_config.md b/docs/development/model_config.md
index 3982116..7dba832 100644
--- a/docs/development/model_config.md
+++ b/docs/development/model_config.md
@@ -132,8 +132,9 @@ To add a model:
1. Add its identity to `SDVersion` without renumbering existing values.
2. Implement the network and `*Config::detect_from_weights` following the
conventions above, or reuse an existing runner.
-3. Add a definition in its integration file, including its component
- factory, traits, geometry, and sampling policy.
+3. Add a definition in its integration file with its display name, stable
+ lowercase DGL family ID, component factory, traits, geometry, and sampling
+ policy. Use underscores between words in the ID.
4. Add the family's explicit registration in `src/model_registry.cpp`. For a new
variant of a registered family, add its definition to that family's list.
5. Add version detection signatures and any required weight-name mappings in
@@ -175,6 +176,15 @@ and the output runners. Backend checks, optional IP-Adapter setup, scale
overrides, and tokenizer usage validation remain in the common builder. A
factory returns failure if any required additional backend cannot initialize.
+`Context::params` contains `ModelBuildParams`, limited to model construction.
+The native pipeline extracts these values from its `ContextConfig` at the
+builder boundary. DGL uses its own typed `ModelLoadParams` for common loader
+options and separate DiT, text encoder and VAE parameters for specialized
+options. It does not store or convert through `ContextConfig`. Threading,
+file mapping, caching and memory budgets belong to the DGL execution context.
+The placement cache takes `PlacementOptions` rather than an entry point's
+context configuration.
+
LLM architecture selection, tokenizer requirements, prompt preparation, and
encoding remain centralized in `src/conditioning/conditioner.hpp`. Integration
factories instantiate the conditioner with the model version; models requiring
@@ -204,28 +214,35 @@ the values that differ from the struct defaults. `prediction_from_weights` and
`flow_shift_from_weights` preserve checkpoint-dependent defaults during automatic
prediction selection; `probe_v_prediction` enables SD2's runtime v-prediction
probe. Explicit prediction settings take precedence over automatic selection.
+In DGL the override belongs to `predictor`, rather than a model loader. A
+resolved `Denoising` configuration supplies prediction scalings, time mapping
+and noise conventions without changing the model's resource identity. Model
+flow-shift defaults are resolved independently of an explicit prediction override.
MOSS's default unconditional skip layer stays
in the diffusion engine, with active unconditional SLG taking precedence.
The resolved `prediction_t` selects the denoiser directly. Flux3, MiniMax H3 AV,
and Ming-Image use `FLUX3_FLOW_PRED`, `H3_AV_FLOW_PRED`, and
`MING_IMAGE_FLOW_PRED`; their CLI overrides are `flux3_flow`, `h3_av_flow`, and
-`ming_image_flow`. Explicit `sd3_flow` always selects `DiscreteFlowDenoiser`,
+`ming_image_flow`. Explicit `flow` always selects `DiscreteFlowDenoiser`,
and `flux_flow` always selects `FluxFlowDenoiser`, regardless of model version.
Automatic selection retains each model's default implementation. New prediction
values are appended before `PREDICTION_COUNT`, preserving existing valid values;
clients using the automatic-selection sentinel must rebuild with the new header.
`ModelGeometry` declares `diffusion_down_factor`, `latent_channels`, and
-`image_channels`. `ref_image_preset` defaults to `"default"`; set it only for models
-that need another reference-image preset.
-
-The common pipeline executes these policies; specialized video, multimodal,
-cache, and extension algorithms remain in their existing runtime components.
-Model-specific diffusion extra parameters are assembled in the diffusion engine.
-Video frame conversion and alignment also remain in the diffusion engine.
-`get_vae_scale_factor()` retains PiD's pixel-space special case and otherwise
-uses the VAE's scale factor.
+`image_channels`. Reference-image workflow defaults belong to API program
+generation and its DGL templates. Model definitions do not select shared
+reference presets. C++ prepares reference sizes, buckets and crop coordinates;
+templates express the pixel transformations and encoding calls. The rendered
+program explicitly supplies prompt media and predictor reference latents.
+
+DGL executes the resulting program. Diffusion models consume reference latents,
+`ref_index_mode` and `force_ref_timestep_zero`; a separate enable flag is not
+needed. Text encoders retain their native patch alignment, normalization and
+model-specific image processing. These model operations do not receive API
+resizing or routing policies. Specialized video, multimodal, cache and extension
+algorithms remain in their typed model and runtime components.
Validate migrations against the previous implementation using synthetic metadata,
name conversions, prompt preparation, default/override selection, and focused
diff --git a/docs/development/pulid.md b/docs/development/pulid.md
index b9ed66b..1bb8764 100644
--- a/docs/development/pulid.md
+++ b/docs/development/pulid.md
@@ -38,9 +38,9 @@ a new combination of model + backend + hardware:
| Run | Expected hash relation |
|----------------------------------------------|------------------------------------|
-| A: no `--pulid-*` flags | baseline |
-| B: PuLID flags, `--pulid-id-weight 0.0` | **byte-identical to A** |
-| C: PuLID flags, `--pulid-id-weight 1.0` | **different from A,B**, preserves source identity |
+| A: DGL condition without `pulid::apply` | baseline |
+| B: same script with `pulid::apply(..., strength = 0)` | **byte-identical to A** |
+| C: same script with `pulid::apply(..., strength = 1)` | **different from A,B**, preserves source identity |
If A and C differ but A and B differ too, the injection is allocating
or computing something even at zero weight -- likely a bug.
diff --git a/docs/dgl-builtins.md b/docs/dgl-builtins.md
new file mode 100644
index 0000000..4905a7f
--- /dev/null
+++ b/docs/dgl-builtins.md
@@ -0,0 +1,215 @@
+<!-- Generated by sd-cli -M dgl. Update with the dgl-update-builtins CMake target. -->
+# Built-in DGL functions
+
+Optional parameters are marked with `?`. Default values follow `=`. See [DGL syntax](dgl.md#dgl-syntax) for `fn`, `map` and `fold`.
+
+| Function | Parameters | Returns |
+|---|---|---|
+| `add_noise` | `predict: predictor`, `base: tensor`, `noise: tensor`, `sigma: number` | `tensor` |
+| `animatediff::apply` | `model: denoiser`, `motion: animatediff` | `denoiser` |
+| `animatediff::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `animatediff` |
+| `apg` | `positive: predictor`, `negative: predictor`, `scale: number`, `image_only?: predictor`, `image_scale?: number = 1`, `schedule?: list<number>`, `eta?: number = 1`, `momentum?: number = 0`, `norm_threshold?: number = 0`, `norm_threshold_smoothing?: number = 0` | `predictor` |
+| `apply_lora` | `model: model`, `lora: lora`, `weight?: number = 1.0`, `mode?: string` | same type as `model` |
+| `audio::load` | `path: string` | `dict`<br>`sample_rate: int`<br>`samples: tensor` |
+| `audio::save` | `path: string`, `audio: dict`, `format?: string = "f32"` | `void` |
+| `audio_latent_shape` | `vae: audio_vae`, `frames: int`, `fps: int` | `tensor` |
+| `audio_vae_decode` | `vae: audio_vae`, `latent: tensor` | `dict`<br>`sample_rate: int`<br>`samples: tensor` |
+| `audio_vae_encode` | `vae: audio_vae`, `samples: tensor`, `sample_rate: int` | `tensor` |
+| `cache` | `mode?: string = "disabled"`, `reuse_threshold?: number`, `start_percent?: number`, `end_percent?: number`, `error_decay_rate?: number`, `use_relative_threshold?: bool`, `reset_error_on_compute?: bool`, `residual_diff_threshold?: number`, `max_warmup_steps?: int`, `max_cached_steps?: int`, `max_continuous_cached_steps?: int`, `taylorseer_n_derivatives?: int`, `taylorseer_skip_interval?: int`, `scm_mask?: string`, `spectrum_w?: number`, `spectrum_m?: int`, `spectrum_lam?: number`, `spectrum_window_size?: int`, `spectrum_flex_window?: number`, `spectrum_warmup_steps?: int`, `spectrum_stop_percent?: number` | `dict`<br>`end_percent: number`<br>`error_decay_rate: number`<br>`max_cached_steps: int`<br>`max_continuous_cached_steps: int`<br>`max_warmup_steps: int`<br>`mode: string`<br>`reset_error_on_compute: bool`<br>`residual_diff_threshold: number`<br>`reuse_threshold: number`<br>`scm_mask: string`<br>`spectrum_flex_window: number`<br>`spectrum_lam: number`<br>`spectrum_m: int`<br>`spectrum_stop_percent: number`<br>`spectrum_w: number`<br>`spectrum_warmup_steps: int`<br>`spectrum_window_size: int`<br>`start_percent: number`<br>`taylorseer_n_derivatives: int`<br>`taylorseer_skip_interval: int`<br>`use_relative_threshold: bool` |
+| `cfg` | `positive: predictor`, `negative: predictor`, `scale: number`, `image_only?: predictor`, `image_scale?: number = 1`, `schedule?: list<number>` | `predictor` |
+| `controlnet::apply` | `condition: dict`, `model: control_net`, `hint: tensor`, `strength?: number = 1`, `start_sigma?: number`, `end_sigma?: number`, `type?: string = "none"` | same type as `condition`<br>`controls: control_bindings` |
+| `controlnet::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `control_net` |
+| `denoiser_scalings` | `predict: predictor`, `sigma: number` | `dict`<br>`input_scale: number`<br>`output_scale: number`<br>`skip_scale: number` |
+| `detections::box_masks` | `image: tensor`, `detections: dict` | `list<tensor>` |
+| `detections::detect` | `detector: detector`, `image: tensor`, `input_size?: int = 640`, `confidence?: number = 0.3`, `nms?: number = 0.45`, `max_detections?: int = 100` | `dict`<br>`class_ids: tensor`<br>`scores: tensor`<br>`xyxy: tensor` |
+| `detections::filter` | `image: tensor`, `detections: dict`, `top_k?: int = 0`, `top_k_by?: string = "area"`, `min_area_ratio?: number = 0`, `max_area_ratio?: number = 1`, `sort_by?: string = "none"` | `dict`<br>`class_ids: tensor`<br>`scores: tensor`<br>`xyxy: tensor` |
+| `detections::to_regions` | `image: tensor`, `detections: dict` | `dict`<br>`boxes: tensor`<br>`height: int`<br>`width: int` |
+| `dit_to_vae_latents` | `vae: vae`, `latent: tensor` | `tensor` |
+| `encode_audio` | `encoder: audio_encoder`, `samples: tensor`, `sample_rate: int` | `tensor` |
+| `encode_prompt` | `encoder: text_encoder`, `text: string`, `width?: int = 512`, `height?: int = 512`, `clip_skip?: int = -1`, `system?: string`, `media?: list<any>`, `image_bindings?: dict`, `raw?: bool = false` | `dict`<br>`concat: tensor`<br>`crossattn: tensor`<br>`extra_crossattn: list<tensor>`<br>`image_embed_indices: list<tensor>`<br>`image_embeds: list<tensor>`<br>`input_ids: tensor`<br>`position_ids: tensor`<br>`ref_audios: list<tensor>`<br>`ref_images: list<tensor>`<br>`reference_blocks: list<dict>`<br>`reference_blocks.audio: int`<br>`reference_blocks.kind: string`<br>`reference_blocks.video: int`<br>`t5_ids: tensor`<br>`t5_weights: tensor`<br>`token_types: tensor`<br>`vector: tensor`<br>`vinput_mask: tensor` |
+| `encode_vision` | `encoder: vision_encoder`, `image: tensor`, `pooled?: bool = false`, `clip_skip?: int = -2`, `zero?: bool = false`, `zero_input?: bool = false` | `tensor` |
+| `image::canny` | `image: tensor`, `high_threshold?: number = 0.08`, `low_threshold?: number = 0.04`, `weak?: number = 0.5`, `strong?: number = 1`, `invert?: bool = false` | `tensor` |
+| `image::encode` | `image: tensor`, `format?: string = "png"` | `dict`<br>`data: bytes`<br>`mime_type: string` |
+| `image::load` | `path: string` | `tensor` |
+| `image::load_dir` | `path: string`, `max_images?: int = 0` | `list<tensor>` |
+| `image::save` | `path: string`, `image: tensor` | `void` |
+| `image::to_f32` | `image: tensor`, `channels?: int = 0` | `tensor` |
+| `image::to_u8` | `pixels: tensor` | `tensor` |
+| `ip_adapter::apply` | `condition: dict`, `adapter: ip_adapter`, `tokens: tensor`, `strength?: number = 1`, `start_sigma?: number`, `end_sigma?: number` | same type as `condition`<br>`ip_adapters: ip_adapter_bindings` |
+| `ip_adapter::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `ip_adapter` |
+| `ip_adapter::project` | `adapter: ip_adapter`, `embedding: tensor` | `tensor` |
+| `latent_shape` | `vae: vae`, `width: int`, `height: int`, `batch?: int = 1`, `frames?: int` | `tensor` |
+| `list::replace` | `items: list<any>`, `index: int`, `value: any` | `list<any>` |
+| `load_audio_encoder` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `audio_encoder` |
+| `load_audio_vae` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false`, `family_from?: string = ""` | `audio_vae` |
+| `load_detector` (resource) | `path: string`, `backend?: string = ""`, `params_backend?: string = ""` | `detector` |
+| `load_dit` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `denoiser` |
+| `load_latent_upscaler` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `latent_upscaler` |
+| `load_lora` (resource) | `path: string` | `lora` |
+| `load_tae` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false`, `family_from?: string = ""`, `decode_only?: bool = false` | `vae` |
+| `load_text_encoder` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false`, `family_from?: string = ""`, `tokenizer?: string = ""`, `vision_path?: string = ""`, `projection_path?: string = ""`, `connectors_path?: string = ""`, `t5_path?: string = ""`, `clip_l_path?: string = ""`, `clip_g_path?: string = ""` | `text_encoder` |
+| `load_unet` (resource) (alias of `load_dit`) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `denoiser` |
+| `load_upscaler` (resource) | `path: string`, `tile_size?: int = 128`, `backend?: string = ""`, `params_backend?: string = ""`, `direct?: bool = false` | `upscaler` |
+| `load_vae` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false`, `family_from?: string = ""`, `vae_format?: string = "auto"` | `vae` |
+| `load_vision_encoder` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `vision_encoder` |
+| `ltx::video_positions` | `like: tensor`, `keyframe_latent_frames: int`, `keyframe_frame_index: int`, `keyframe_pixel_frames: int`, `fps?: int = 24`, `spatial_scale?: int = 32`, `temporal_scale?: int = 8`, `causal?: bool = true` | `tensor` |
+| `make_rng` | `algorithm?: string = "cuda"`, `seed?: int = 0` | `rng` |
+| `make_sigmas` | `predict: predictor`, `latent: tensor`, `scheduler?: string = "discrete"`, `steps: int`, `flow_shift?: number = 0`, `sequence_length?: int`, `extra_args?: string = ""` | `tensor` |
+| `math::add` | `a: number`, `b: number` | `number` |
+| `math::ceil` | `value: number` | `int` |
+| `math::ceil_div` | `numerator: int`, `denominator: int` | `int` |
+| `math::div` | `a: number`, `b: number` | `number` |
+| `math::floor` | `value: number` | `int` |
+| `math::min` | `a: int`, `b: int` | `int` |
+| `math::mul` | `a: number`, `b: number` | `number` |
+| `math::product` | `items: i64 tensor or list<int>` | `int` |
+| `math::round` | `value: number` | `int` |
+| `math::sub` | `a: number`, `b: number` | `number` |
+| `media::save` | `path: string`, `media: dict` | `void` |
+| `photomaker::apply` | `condition: dict`, `photomaker: photomaker`, `tokens: tensor`, `mask: tensor` | same type as `condition`<br>`crossattn: tensor` |
+| `photomaker::encode_images` | `photomaker: photomaker`, `images: list<tensor>`, `id_embedding?: tensor` | `tensor` |
+| `photomaker::encode_prompt` | `encoder: text_encoder`, `text: string`, `tokens: tensor`, `width?: int = 1024`, `height?: int = 1024`, `clip_skip?: int = -1` | `dict`<br>`condition: dict`<br>`condition.concat: tensor`<br>`condition.crossattn: tensor`<br>`condition.extra_crossattn: list<tensor>`<br>`condition.image_embed_indices: list<tensor>`<br>`condition.image_embeds: list<tensor>`<br>`condition.input_ids: tensor`<br>`condition.position_ids: tensor`<br>`condition.ref_audios: list<tensor>`<br>`condition.ref_images: list<tensor>`<br>`condition.reference_blocks: list<dict>`<br>`condition.reference_blocks.audio: int`<br>`condition.reference_blocks.kind: string`<br>`condition.reference_blocks.video: int`<br>`condition.t5_ids: tensor`<br>`condition.t5_weights: tensor`<br>`condition.token_types: tensor`<br>`condition.vector: tensor`<br>`condition.vinput_mask: tensor`<br>`mask: tensor`<br>`plain: dict`<br>`plain.concat: tensor`<br>`plain.crossattn: tensor`<br>`plain.extra_crossattn: list<tensor>`<br>`plain.image_embed_indices: list<tensor>`<br>`plain.image_embeds: list<tensor>`<br>`plain.input_ids: tensor`<br>`plain.position_ids: tensor`<br>`plain.ref_audios: list<tensor>`<br>`plain.ref_images: list<tensor>`<br>`plain.reference_blocks: list<dict>`<br>`plain.reference_blocks.audio: int`<br>`plain.reference_blocks.kind: string`<br>`plain.reference_blocks.video: int`<br>`plain.t5_ids: tensor`<br>`plain.t5_weights: tensor`<br>`plain.token_types: tensor`<br>`plain.vector: tensor`<br>`plain.vinput_mask: tensor` |
+| `photomaker::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `photomaker` |
+| `predictor` | `model: denoiser`, `condition: dict`, `prediction?: string = "auto"`, `concat?: tensor`, `ref_latents?: list<tensor>`, `ref_audios?: list<tensor>`, `reference_blocks?: list<dict>`, `keyframe_indices?: list<int>`, `ref_index_mode?: string = "fixed"`, `force_ref_timestep_zero?: bool = false`, `guidance?: number = 3.5`, `skip_layers?: list<int>`, `vace_context?: tensor`, `vace_strength?: number = 1`, `audio_embedding?: tensor`, `video_positions?: tensor`, `audio_length?: int = 0`, `fps?: number = 24`, `video_sigma_shift?: number = 12`, `audio_sigma_shift?: number = 3`, `circular_x?: bool = false`, `circular_y?: bool = false`, `transform?: fn`, `transform_args?: dict` | `predictor` |
+| `preview` | `decoder?: vae`, `circular_x?: bool = false`, `circular_y?: bool = false` | `sampling_observer` |
+| `pulid::apply` | `condition: dict`, `pulid: pulid`, `embedding: tensor`, `strength?: number = 1`, `start_sigma?: number`, `end_sigma?: number` | same type as `condition`<br>`pulid: pulid_bindings` |
+| `pulid::load` (resource) | `path: string`, `family?: string = "auto"`, `wtype?: string = "auto"`, `conv_scale?: number = 0`, `linear_scale?: number = 0`, `attn_scale?: number = 0`, `backend?: string = ""`, `params_backend?: string = ""`, `split_mode?: string = ""`, `model_args?: string = ""`, `tensor_type_rules?: string = ""`, `flash_attn_mode?: string = "auto"`, `conv_direct_mode?: string = "auto"`, `sage_attn?: bool = false` | `pulid` |
+| `pulid::load_embedding` | `path: string` | `tensor` |
+| `regions::crop` | `image: tensor`, `regions: dict` | `list<tensor>` |
+| `regions::from_boxes` | `width: int`, `height: int`, `boxes: any` | `dict`<br>`boxes: tensor`<br>`height: int`<br>`width: int` |
+| `regions::paste` | `background: tensor`, `foregrounds: list<tensor>`, `regions: dict` | `tensor` |
+| `regions::plan_mask_crops` | `masks: list<tensor>`, `aspect?: i64 tensor or list<int> = [1, 1]`, `padding?: int = 32`, `only_masked?: bool = true` | `list<dict>`<br>`box: tensor`<br>`index: int`<br>`mask: tensor` |
+| `relayvsr::decode` | `flash: vae`, `latent: tensor`, `frames: tensor`, `keyframe_indices: tensor`, `detail?: number = 1` | `tensor` |
+| `relayvsr::latent_shape` | `dit: denoiser`, `keyframes: tensor` | `tensor` |
+| `relayvsr::positive_embedding` | `dit: denoiser` | `tensor` |
+| `sample` | `start: tensor`, `sigmas: tensor`, `predict: predictor`, `rng: rng`, `method?: string = "euler"`, `eta?: number = 1`, `base?: tensor`, `denoise_mask?: tensor`, `mask_input?: bool = false`, `extra_args?: string = ""`, `finish?: bool`, `cache?: dict`, `observer?: sampling_observer` | `tensor` |
+| `seedvr2::positive_embedding` | `dit: denoiser` | `tensor` |
+| `sefi::make_sigmas` | `steps: int`, `alpha?: number = 1`, `delta?: number = 0.1` | `dict`<br>`semantic_sigmas: tensor`<br>`sigmas: tensor` |
+| `sefi::sample` | `start: tensor`, `sigmas: tensor`, `semantic_sigmas: tensor`, `predict: predictor`, `rng: rng`, `method?: string = "euler"`, `eta?: number = 1`, `base?: tensor`, `denoise_mask?: tensor`, `mask_input?: bool = false`, `extra_args?: string = ""`, `finish?: bool`, `cache?: dict`, `observer?: sampling_observer` | `tensor` |
+| `slg` | `base: predictor`, `positive: predictor`, `skipped: predictor`, `scale: number`, `start_sigma?: number`, `end_sigma?: number`, `start_step_fraction?: number`, `stop_step_fraction?: number` | `predictor` |
+| `switch_predictor` | `before: predictor`, `after: predictor`, `after_steps: int` | `predictor` |
+| `t_to_sigma` | `predict: predictor`, `t: number`, `flow_shift?: number = 0`, `latent?: tensor`, `sequence_length?: int` | `number` |
+| `tensor::add` | `a: any`, `b: any` | `tensor` |
+| `tensor::affine` | `input: tensor`, `scale?: number = 1`, `offset?: number = 0` | `tensor` |
+| `tensor::arange` | `start: int`, `end: int`, `step?: int = 1` | `tensor` |
+| `tensor::broadcast_to` | `input: tensor`, `shape: i64 tensor or list<int>` | `tensor` |
+| `tensor::cast` | `input: tensor`, `dtype: string` | `tensor` |
+| `tensor::clamp` | `input: tensor`, `min: number`, `max: number` | `tensor` |
+| `tensor::concat` | `items: list<tensor>`, `axis: int` | `tensor` |
+| `tensor::count_nonzero` | `input: tensor` | `int` |
+| `tensor::crop` | `input: tensor`, `axes: i64 tensor or list<int>`, `start: i64 tensor or list<int>`, `size: i64 tensor or list<int>` | `tensor` |
+| `tensor::div` | `a: any`, `b: any` | `tensor` |
+| `tensor::full` | `shape: i64 tensor or list<int>`, `value: number`, `dtype?: string = "f32"` | `tensor` |
+| `tensor::gaussian_blur` | `input: tensor`, `sigma: number`, `radius: int`, `axes?: i64 tensor or list<int>`, `precision?: string = "auto"` | `tensor` |
+| `tensor::grayscale` | `input: tensor`, `axis?: int = 0` | `tensor` |
+| `tensor::index` | `input: tensor`, `indices: list<dict>` | `any` |
+| `tensor::index_fill` | `input: tensor`, `axis: int`, `indices: i64 tensor or list<int>`, `value: number` | `tensor` |
+| `tensor::index_select` | `input: tensor`, `axis: int`, `indices: i64 tensor or list<int>` | `tensor` |
+| `tensor::interpolate` | `input: tensor`, `size: i64 tensor or list<int>`, `axes?: i64 tensor or list<int>`, `mode?: string = "bilinear"`, `antialias?: bool = false` | `tensor` |
+| `tensor::lerp` | `a: any`, `b: any`, `weight: any` | `tensor` |
+| `tensor::literal` | `dtype: string`, `shape: i64 tensor or list<int>`, `data: string` | `tensor` |
+| `tensor::load` | `path: string` | `tensor` |
+| `tensor::max_pool2d` | `input: tensor`, `kernel: i64 tensor or list<int>`, `axes?: i64 tensor or list<int>`, `stride?: i64 tensor or list<int>`, `padding_before?: i64 tensor or list<int>`, `padding_after?: i64 tensor or list<int>`, `pad_value?: number` | `tensor` |
+| `tensor::maximum` | `a: tensor`, `b: tensor` | `tensor` |
+| `tensor::min_pool2d` | `input: tensor`, `kernel: i64 tensor or list<int>`, `axes?: i64 tensor or list<int>`, `stride?: i64 tensor or list<int>`, `padding_before?: i64 tensor or list<int>`, `padding_after?: i64 tensor or list<int>`, `pad_value?: number` | `tensor` |
+| `tensor::mul` | `a: any`, `b: any` | `tensor` |
+| `tensor::numel` | `input: tensor` | `int` |
+| `tensor::pad` | `input: tensor`, `pad: i64 tensor or list<int>`, `mode?: string = "constant"`, `value?: number = 0` | `tensor` |
+| `tensor::permute` | `input: tensor`, `axes: i64 tensor or list<int>` | `tensor` |
+| `tensor::randn_like` | `like: tensor`, `rng: rng` | `tensor` |
+| `tensor::replace_slice` | `input: tensor`, `value: tensor`, `axis: int`, `start: int` | `tensor` |
+| `tensor::reshape` | `input: tensor`, `shape: i64 tensor or list<int>` | `tensor` |
+| `tensor::save` | `path: string`, `tensor: tensor` | `void` |
+| `tensor::shape` | `input: tensor` | `tensor` |
+| `tensor::threshold` | `input: tensor`, `threshold: number`, `low?: number = 0`, `high?: number = 1` | `tensor` |
+| `tensor::to_list` | `input: tensor`, `element?: string = "auto"` | `list<any>` |
+| `tensor::translate` | `input: tensor`, `axes: i64 tensor or list<int>`, `offset: i64 tensor or list<int>`, `value?: number = 0` | `tensor` |
+| `tensor::unsqueeze` | `input: tensor`, `axis: int` | `tensor` |
+| `tensor::vector` | `values: list<number>`, `dtype?: string = "f32"` | `tensor` |
+| `tensor::zeros` | `shape: i64 tensor or list<int>`, `dtype?: string = "f32"` | `tensor` |
+| `tensor::zeros_like` | `like: tensor` | `tensor` |
+| `tiling` | `enabled?: bool = true`, `temporal_tiling?: bool = false`, `tile_size_w?: int = 0`, `tile_size_h?: int = 0`, `target_overlap?: number = 0.5`, `rel_size_w?: number = 0`, `rel_size_h?: number = 0`, `extra_args?: string = ""` | `dict`<br>`enabled: bool`<br>`extra_args: string`<br>`rel_size_h: number`<br>`rel_size_w: number`<br>`target_overlap: number`<br>`temporal_tiling: bool`<br>`tile_size_h: int`<br>`tile_size_w: int` |
+| `unet_to_vae_latents` (alias of `dit_to_vae_latents`) | `vae: vae`, `latent: tensor` | `tensor` |
+| `upscale` | `upscaler: upscaler`, `image: tensor`, `factor?: int` | `tensor` |
+| `upscale_latent` | `upscaler: latent_upscaler`, `latent: tensor`, `width?: int`, `height?: int` | `tensor` |
+| `vae_decode` | `vae: vae`, `latent: tensor`, `tiling?: dict`, `circular_x?: bool = false`, `circular_y?: bool = false`, `dtype?: string = "u8"` | `tensor` |
+| `vae_encode` | `vae: vae`, `pixels: tensor`, `rng: rng`, `tiling?: dict`, `circular_x?: bool = false`, `circular_y?: bool = false`, `posterior?: string = "sample"` | `tensor` |
+| `vae_to_dit_latents` | `vae: vae`, `latent: tensor` | `tensor` |
+| `vae_to_unet_latents` (alias of `vae_to_dit_latents`) | `vae: vae`, `latent: tensor` | `tensor` |
+| `video::encode` | `video: dict`, `audio?: dict`, `format?: string = "webm"` | `dict`<br>`data: bytes`<br>`mime_type: string` |
+| `video::join_windows` | `windows: list<tensor>`, `overlap?: int = 0` | `tensor` |
+| `video::load` | `path: string`, `max_frames?: int = 0`, `fps_num?: int = 0`, `fps_den?: int = 1`, `audio?: bool = true` | `dict`<br>`audio: dict`<br>`audio.sample_rate: int`<br>`audio.samples: tensor`<br>`video: dict`<br>`video.fps_den: int`<br>`video.fps_num: int`<br>`video.frames: tensor` |
+| `video::save` | `path: string`, `video: dict`, `audio?: dict`, `trim_audio?: bool = false`, `codec?: string = "auto"`, `quality?: int = 90`, `audio_format?: string = "f32"` | `void` |
+| `video::split_windows` | `frames: tensor`, `size: int`, `overlap?: int = 0` | `list<tensor>` |
+| `wan::s2v_audio_window` | `embedding: tensor`, `latent_frames: int` | `tensor` |
+| `zero_out` | `condition: dict`, `fields?: list<string>` | same type as `condition` |
+
+## Model family IDs
+
+Pass the ID as the component loader's `family` argument. The default is `"auto"`.
+
+| ID | Model family |
+|---|---|
+| `sd1` | SD 1.x |
+| `sd1_inpaint` | SD 1.x Inpaint |
+| `sd1_pix2pix` | Instruct-Pix2Pix |
+| `sd1_tiny_unet` | SD 1.x Tiny UNet |
+| `sd2` | SD 2.x |
+| `sd2_inpaint` | SD 2.x Inpaint |
+| `sd2_tiny_unet` | SD 2.x Tiny UNet |
+| `sdxs_512_ds` | SDXS (512-DS) |
+| `sdxs_09` | SDXS (09) |
+| `sdxl` | SDXL |
+| `sdxl_inpaint` | SDXL Inpaint |
+| `sdxl_pix2pix` | SDXL Instruct-Pix2Pix |
+| `sdxl_vega` | SDXL (Vega) |
+| `sdxl_ssd1b` | SDXL (SSD1B) |
+| `svd` | SVD |
+| `sd3` | SD3.x |
+| `flux` | Flux |
+| `flux_fill` | Flux Fill |
+| `flux_control` | Flux Control |
+| `flex2` | Flex.2 |
+| `chroma_radiance` | Chroma Radiance |
+| `wan2` | Wan 2.x |
+| `wan2_2_i2v` | Wan 2.2 I2V |
+| `wan2_2_ti2v` | Wan 2.2 TI2V |
+| `wan2_2_s2v` | Wan 2.2 S2V |
+| `lingbot_video` | LingBot Video |
+| `qwen_image` | Qwen Image |
+| `qwen_image_layered` | Qwen Image Layered |
+| `qwen_image_2_1` | Qwen Image 2.1 |
+| `hunyuan_video` | Hunyuan Video |
+| `anima` | Anima |
+| `flux2` | Flux.2 |
+| `flux2_klein` | Flux.2 klein |
+| `ltxav` | LTXAV |
+| `minimax_h3` | MiniMax-H3 |
+| `hidream_o1` | HiDream O1 |
+| `z_image` | Z-Image |
+| `boogu_image` | Boogu Image |
+| `ovis_image` | Ovis Image |
+| `ernie_image` | Ernie Image |
+| `lens` | Lens |
+| `minit2i` | MiniT2I |
+| `longcat_image` | Longcat-Image |
+| `pid` | PiD |
+| `ideogram4` | Ideogram 4 |
+| `sefi_image` | SeFi-Image |
+| `krea2` | Krea2 |
+| `mage_flow` | Mage Flow |
+| `sensenova_u1_5` | SenseNova U1.5 |
+| `llada_image` | LLaDA-Image |
+| `esrgan` | ESRGAN |
+| `pixart` | PixArt |
+| `ming_image` | Ming-Image |
+| `z_image_l2p` | Z-Image L2P |
+| `iris_3b` | Iris-3B |
+| `flux3_klein` | Flux.3 Image Klein |
+| `moss_image` | MOSS-Image |
+| `fibo2` | Fibo-2.0 |
+| `chroma` | Chroma |
+| `p_h3_image` | P-H3-Image |
+| `minimax_h3_image` | MiniMax-H3-Image |
+| `relayvsr` | RelayVSR |
+| `seedvr2` | SeedVR2 |
diff --git a/docs/dgl.md b/docs/dgl.md
new file mode 100644
index 0000000..1adb624
--- /dev/null
+++ b/docs/dgl.md
@@ -0,0 +1,1943 @@
+# DGL scripting guide
+
+Use DGL (Diffusion Graph Language) to compose model and tensor operations in a
+script. Values can feed several calls, and functions can process lists. A script
+can load its own models and media or receive named input tensors from a host.
+Applications can generate a complete script from request settings.
+
+- **Getting started** · [Build and run](#build-and-run) · [Model examples](dgl/README.md)
+- **Writing scripts** · [Language syntax](#dgl-syntax) · [Tensor operations and indexing](#tensor-functions) · [Built-in catalog](dgl-builtins.md)
+- **Generation** · [Models and sampling](#independent-generation-components) · [Media I/O](#image-and-video-files) · [Regions and masks](#regions-and-masks) · [Worked examples](#advanced-component-programs)
+- **Embedding DGL** · [C API](#c-api) · [Architecture](development/dgl.md)
+
+## Build and run
+
+```sh
+cmake -S . -B build-dgl -DSD_BUILD_EXAMPLES=ON -DSD_WEBP=OFF -DSD_WEBM=OFF
+cmake --build build-dgl --config Release --target sd-cli
+```
+
+Use the normal backend build options, such as `-DSD_CUDA=ON`, when needed.
+
+Save the following program as `tensor-example.dgl`. It runs without model weights.
+
+```dgl
+# version: 0.0.0
+fn main() {
+ return tensor::zeros([4, 4])
+}
+```
+
+```sh
+./build-dgl/bin/sd-cli -M dgl --script tensor-example.dgl --tensor result tensor.npy
+```
+
+For generation, edit the model path, prompt and seed in the [SD 1.5 example](dgl/sd/sd1-5-txt2img.dgl).
+
+```sh
+./build-dgl/bin/sd-cli -M dgl --script docs/dgl/sd/sd1-5-txt2img.dgl --check
+./build-dgl/bin/sd-cli -M dgl --script docs/dgl/sd/sd1-5-txt2img.dgl --tensor latent latent.npy
+```
+
+To repeat a run, execute the same `.dgl` file again with the same input files.
+
+`--check` checks syntax, function signatures, references and static types without
+loading weights. It does not check model-specific tensor compatibility.
+
+`--tensor NAME FILE.npy` exports a named output, and can be repeated. The result
+summary and tensor names appear in the verbose log (`-v`). `image::save(..., pixels)`
+writes PNG files during execution. Rerunning the script writes to the same paths.
+Paths are relative to the process working directory.
+
+DGL mode uses the CLI's log levels and prints logs to stderr. The default level
+is `info` (`[I]`). Use `-v` or `--verbose` for verbose (`[V]`) logs, and
+`--log-level debug` to include debug (`[D]`) logs.
+Sampling reports its method and step count at the start, and elapsed time on
+completion, at the info level.
+
+## DGL syntax
+
+Execution starts at the single `fn main()` declaration. Standalone scripts use
+a zero-argument entry point.
+
+Use an optional `var { ... }` block before the functions to group values
+such as model paths, prompts, and seeds. Each entry is a scalar, list, or
+dict literal. The names are available directly in `main()` as if their
+bindings appeared at the start of its body.
+
+```dgl
+var {
+ seeds = [41, 42]
+}
+
+fn next_seed(seed) {
+ return seed + 1
+}
+
+fn main() {
+ return map(seeds, process = next_seed)
+}
+```
+
+### Functions and bindings
+
+- Declare user-defined functions at file scope using the `fn` keyword and
+ pass values to them as arguments. Put local bindings inside a function.
+ When constructing a script, encode string values as JSON string literals.
+- Bind each name once, and reference values declared earlier in the same scope.
+ Names in `var` share the entry point's binding scope.
+ Positional arguments precede named arguments. Unknown or duplicate arguments
+ are errors. Argument expressions are evaluated left to right as written at the
+ call site. A `return` statement, when present, must be last. A function may
+ finish without a return value, or use bare `return;` to finish explicitly.
+- Call functions such as `image::save(...)` directly as statements. Saving
+ functions return `void`; using them in assignments, arguments, containers
+ or returned expressions is a compile-time error.
+- Strings use JSON escaping. Comments start with `#` or `//`, on their own line
+ or after code. Only `# version: ...` is reserved for the file's version header.
+ Semicolons are optional. Lists use `[a, b]`. A dictionary (`dict`) groups
+ named values using `{name = value, ...}`. Access a field with `dict_value.name`.
+ A prompt condition contains the fields supplied by its encoder. Reading an
+ absent field reports its name and source location. See how to read the
+ [built-in return types](#built-in-functions).
+- Functions run sequentially. Recursion, cycles and runtime-dependent control
+ flow are not supported. Intermediate values are released after their last use
+ unless a function retains them or the program returns them.
+
+### Lists, map and fold
+
+- `map(items, process = function, other_parameter = value)` passes each item as
+ the first function parameter. Additional arguments are shared across calls.
+- `fold(items, initial, process = function, other_parameter = value)` passes
+ the previous result and the current item as the first two parameters. The
+ function must return the same type and fields as `initial`. An empty typed
+ list returns `initial`.
+- `items[index]` selects a list element; `items[start:end]` selects a list slice.
+ `tensor::to_list(tensor::arange(seed, seed + count))` supplies consecutive
+ integer seeds to `map`. The range end is exclusive.
+- Lists contain one element type; integer and floating-point literals may be
+ mixed in numeric lists. With a list of dictionaries, `map`, `fold` and
+ dynamic indexing expose shared fields and consistent tensor dtype/rank facts.
+ A known index into a known list keeps the selected item's own information.
+ `list::replace` requires a replacement compatible with the list's element
+ type; incompatible types are rejected during compilation.
+
+### Conditional expressions
+
+Use `if condition { expression } else { expression }` to choose a value:
+
+```dgl
+result = if enable_refiner { refine(latent) } else { latent }
+```
+
+The condition must be a boolean known before resource planning, such as a
+`var` value or a literal passed to a function. Conditions derived from tensors,
+model outputs or runtime function calls are rejected. Both branches are checked
+for valid calls and matching result types and dictionary fields, even when one
+is disabled. Each branch can contain local bindings and standalone calls,
+followed by a final expression that supplies its result. An `if` used as a value
+requires `else`. Conditional expressions can be nested, used as arguments,
+indexed or returned.
+A conditional used as a standalone statement discards its result, including
+`void` results.
+
+```dgl
+result = if enable_refiner {
+ refiner = load_unet(refiner_weights)
+ enlarged = tensor::interpolate(latent, [128, 128])
+ refine(enlarged, refiner)
+} else {
+ latent
+}
+```
+
+Each branch has its own scope and can read outer bindings. Local names are
+unavailable outside the branch, and the two branches may use the same local
+names. Bind each name once; shadowing an outer binding is rejected. Use the
+final expression instead of `return`, which remains a function statement.
+An empty block or one ending in a binding has type `void`. Semicolons remain
+optional separators and do not change the final expression's result.
+Insert `;` before a following list expression when it could otherwise be read
+as indexing of the preceding expression.
+
+A standalone `if` may omit `else` and has type `void`:
+
+```dgl
+if save_latent {
+ tensor::save("latent.npy", latent)
+}
+```
+
+The selected block executes for its side effects. Assignments, arguments,
+containers and returned values still require the two-branch form.
+
+Only the selected branch contributes model resources and weight-use phases to
+the run plan, and only that branch executes. Each statement in the selected
+block retains its own planning boundary, including blocks nested in function
+calls or argument expressions. Load optional models inside the branch or inside
+the function it calls. Loading a model before the conditional
+still loads it unconditionally. For example:
+
+```dgl
+# version: 0.0.0
+
+var {
+ enable_refiner = false
+ refiner_weights = "optional-refiner.safetensors"
+}
+
+fn main() {
+ model = if enable_refiner { load_unet(refiner_weights) } else { load_unet("base.safetensors") }
+ return model
+}
+```
+
+Here the disabled refiner file need not exist. Keep optional model loads
+inside their selected branch. Use `map` and `fold` for collections of work. Loops whose next step depends on a computed result are
+not supported.
+
+### DGL versions
+
+The optional version header declares the minimum DGL runtime version required
+within one major version.
+
+```dgl
+# version: 0.0.0
+```
+
+Write three unsigned 32-bit decimal integers (`MAJOR.MINOR.PATCH`). The header
+may appear once, before all code. Blank lines and ordinary comments may precede
+it. Omitting it uses the current runtime. Forms such as `1`, `1.2` and `v1.2.0`
+are format errors.
+
+**Development.** The runtime, examples and C API entry templates currently use
+`0.0.0`. Development snapshots have no compatibility guarantee. Identify a
+snapshot by its repository commit. The first stable release, `1.0.0`, establishes
+compatibility for syntax, built-in signatures and documented behavior.
+Development and stable versions reject each other. Verify a development script
+before updating its declaration.
+
+**Stable releases.** The runtime must have the same major version and meet or
+exceed the declared minimum. Components are compared numerically. A `1.2.0`
+script can run on `1.2.3`, `1.3.0` or `1.10.0`. On `1.1.9` or `2.0.0` the
+compiler reports the required and supported versions. Compatible scripts use
+the current implementation and its available built-ins.
+
+| Increment | When to use it |
+| --- | --- |
+| Major | Existing legal scripts need edits, or their documented meaning changes. |
+| Minor | A new capability preserves the behavior of existing legal scripts. |
+| Patch | A fix restores documented behavior, or an optimization preserves it. |
+
+Use the highest applicable increment for a release. DGL is versioned independently
+of sd.cpp, the C API and its templates. Raise a script's declaration when it
+depends on a new capability or fix. Each complete C API entry template declares
+its own minimum, which also covers the helpers it includes. The rendered header
+is part of the source retained in `generation_info` and embedded in generation
+metadata when enabled.
+
+### Built-in functions
+
+The [built-in function catalog](dgl-builtins.md) lists registered functions,
+parameters, defaults and return types. For `encode_prompt` and the nested
+conditions from `photomaker::encode_prompt`, the **Returns** column lists
+possible fields across models. For example, a condition contains `ref_audios`
+only when the encoder supplies audio references. Run
+`sd-cli -M dgl --list-builtins` to inspect the catalog for your executable.
+
+To regenerate the checked-in catalog after changing a registration, run
+`cmake --build build-dgl --config Release --target dgl-update-builtins`.
+The `dgl-check-builtins` target verifies that the file matches the registry.
+
+## Tensor functions
+
+Tensor types are U8, I32, I64, F32 and F64. Dimensions are stored contiguous-first:
+
+| Value | Type | Shape |
+|---|---|---|
+| RGB image | U8 | `[3, width, height]` |
+| RGB video | U8 | `[3, width, height, frame_count]` |
+| Audio samples | F32 | `[channels, samples]` |
+| Diffusion latent | F32 | Model-specific, including four- and five-dimensional layouts |
+| Region boxes | I64 | `[4, region_count]` |
+| Detection coordinates | F32 | `[4, detection_count]` |
+| Detection scores / class IDs | F32 / I64 | `[detection_count]` |
+| Conditioning | F32/I32 | Model-specific; inspect each tensor |
+
+Rank-zero tensors are scalars. Zero-sized dimensions represent empty tensors.
+
+### Operations and broadcasting
+
+Qualified function names use `::`. Call a tensor function with
+`tensor::reshape(x, [4, 8])` and access a dictionary field with `condition.crossattn`.
+
+| Function | Arguments and result |
+|---|---|
+| `tensor::zeros`, `tensor::full` | Take a shape and optional `dtype` (default `f32`). `full` also takes a scalar fill value. |
+| `tensor::zeros_like`, `tensor::randn_like` | Use the input shape and dtype. `randn_like` supports floating-point tensors and requires an RNG object. |
+| `tensor::shape`, `tensor::numel`, `tensor::reshape` | Read the shape as a rank-one I64 tensor or the element count as a scalar. Reshape preserves element count and dtype. |
+| `tensor::unsqueeze` | Insert a length-one axis, preserving dtype and element order. `axis` accepts `[-rank-1, rank]`; output rank is at most 8. |
+| `tensor::pad` | Constant padding with `[before0, after0, before1, after1, ...]` in DGL axis order. `mode="constant"` and `value=0` are defaults. |
+| `tensor::vector` | Construct a rank-one tensor from numeric values. The default dtype is `f32`. Read elements with bracket indexing. |
+| `tensor::to_list` | Convert a rank-one tensor of at most 10,000 elements into `list<int>` or `list<number>`. `element="auto"` infers the element type from a known dtype; `"int"` and `"number"` declare the expected category. |
+| `tensor::arange`, `tensor::index_select` | Generate a rank-one I64 range with exclusive end, or gather slices in index order along one axis. |
+| `tensor::crop`, `tensor::replace_slice`, `tensor::concat` | Crop takes lists of axes, starts and sizes. Replacement and concatenation require matching dtype and dimensions outside the selected axis. Use bracket syntax for basic slicing. |
+| `tensor::cast` | Explicitly convert to a `dtype`. Integer conversion truncates toward zero and rejects out-of-range values. |
+| `tensor::broadcast_to` | Broadcast a tensor to an explicit shape and preserve its dtype. |
+| `tensor::add`, `tensor::mul` | Accept floating-point tensors of the same dtype and numeric scalars. |
+| `tensor::lerp` | Compute `a * (1 - weight) + b * weight`. Two U8 endpoints accept a scalar or floating-point weight tensor, with output rounded and clamped to U8. |
+| `tensor::affine`, `tensor::clamp` | Apply `scale` and `offset`, or constrain values to `min` and `max`. Both take a floating-point tensor. |
+| `tensor::interpolate` | Resize two axes of a U8 or floating-point tensor using `size=[width, height]`, `axes=[0, 1]` by default, and a `mode` such as `bilinear` or `bicubic`. |
+| `tensor::translate`, `tensor::index_fill` | Shift along selected axes with constant fill, or fill entire slices at selected indices along one axis. |
+| `tensor::grayscale`, `tensor::threshold` | Convert the channel axis to grayscale or select `low` below the threshold and `high` at or above it. |
+| `tensor::maximum` | Take the elementwise maximum of two tensors with the same dtype and broadcastable shapes. |
+| `tensor::max_pool2d`, `tensor::min_pool2d` | Pool over two axes with a configurable kernel and stride. `padding_after` defaults to `padding_before`. Set `pad_value` to include a constant outside the tensor. |
+| `tensor::gaussian_blur` | Blur two axes with edge replication. `radius` sets the kernel half-width and `sigma` sets the Gaussian spread. |
+| `tensor::literal` | Construct a tensor from its `dtype`, `shape` and little-endian Base64 `data`. |
+
+Broadcasting aligns dimensions from the lowest axis and treats missing higher
+dimensions as 1. A numeric scalar broadcasts to all elements.
+Shape, axis and size arguments accept rank-one I64 tensors and lists of
+integers. List elements can be scalar expressions, including indexed dimensions;
+there is no need to create or modify a tensor just to describe a shape:
+
+```dgl
+shape = tensor::shape(latent)
+mask_shape = [shape[0], shape[1], shape[2], 4, shape[4]]
+mask = tensor::index_fill(tensor::zeros(mask_shape), axis = 2, indices = [0], value = 1)
+```
+
+`latent_shape(vae, width, height, ...)` and
+`audio_latent_shape(audio_vae, frames, fps)` also return I64 shape tensors.
+The audio shape uses the loaded audio VAE's model family and the video duration
+(`frames / fps`), keeping each family's latent layout and rounding rules.
+Both are metadata queries; `audio_latent_shape` does not load device weights
+or run the audio VAE.
+
+```dgl
+sound_shape = audio_latent_shape(audio_vae, frames = frames, fps = 24)
+sound_base = tensor::zeros(sound_shape)
+```
+
+`math::product` multiplies dimensions, and `math::ceil_div`
+divides nonnegative integers rounding upward. Schedule functions use scalar arithmetic for indices and `math::floor(steps * strength)`
+to convert a fraction of steps to a count.
+
+### Read-only list and tensor indexing
+
+Use brackets after any list or tensor expression, including a function call:
+
+```dgl
+first_sigma = sigmas[0]
+refine_sigmas = full_sigmas[-13:]
+width = tensor::shape(pixels)[0]
+patch = x[1:3, :, 0]
+```
+
+Lists accept exactly one integer index or slice. Integer indices may be negative;
+out-of-range indices fail. Slices are half-open, accept omitted or negative
+endpoints and clip to the list bounds. A slice preserves the input's element type,
+even when empty or when a `list<number>` slice happens to contain only integers.
+Integer values do not need to be converted to floating point.
+
+```dgl
+first = items[0]
+last = items[-1]
+batch = items[2:5]
+rest = items[1:]
+final_steps = items[-3:]
+pixel = patches[0][1, 2, 3]
+```
+
+`list::at` and the global `tail` function have been removed. To select a suffix,
+use `items[-count:]`. This clips to the whole list when `count` exceeds its length;
+`count=0` gives `items[-0:]`, equivalent to `items[:]`, rather than an empty list.
+Use `items[:0]` for an empty slice. Empty `map` results retain the callback's
+inferred return type for that call, such as `list<int>` or `list<tensor>`.
+`list::replace` remains the functional update operation; `map` and `fold`
+continue to iterate lists only.
+
+Axes follow DGL shape order: axis 0 is the lowest, contiguous dimension.
+Integer indices remove their axis; slices keep it. Omitted trailing axes are
+retained. For `x` shaped `[8,6,4,2]`:
+
+| Expression | Result |
+|---|---|
+| `x[1:3, :, 0]` | Tensor `[2,6,2]` |
+| `x[1:3, :, 0:1]` | Tensor `[2,6,1,2]` |
+| `x[-1]` | Tensor `[6,4,2]` |
+| `x[1,2,0,0]` | Numeric scalar |
+| `x[3:1]` | Empty tensor `[0,6,4,2]` |
+
+Integer indices and slice endpoints are integer expressions. Negative indices
+count backward from an axis's end; out-of-range integer indices fail. Slices
+are half-open, support omitted endpoints, and clip endpoints to the axis bounds.
+`x[start:]`, `x[:end]` and `x[:]` are supported. Endpoints can contain scalar
+arithmetic, for example `sigmas[count - 13:]`.
+
+An explicit slice always returns a tensor, including empty and length-one
+slices. With only integer indices, consuming all axes returns a scalar; otherwise the
+result is a tensor. The compiler retains known rank and element types from
+tensor constructors, shape queries and schedules, so fully consumed indices
+keep their numeric types in lists and user functions. `--check` rejects known
+non-integer indices/endpoints and too many indices for a known rank.
+File-loaded tensor ranks and other unknown properties remain runtime checks. Integral tensor elements become DGL integers;
+floating-point elements become DGL numbers.
+
+Brackets form an indexing expression. The checker selects list or tensor rules
+from the input type, and execution dispatches to the corresponding operation.
+Known list elements and indices also select precise model origins during
+resource planning; dynamic indices conservatively combine candidate origins.
+Step sizes, ellipsis, boolean masks, index tensors and indexed assignment remain
+unsupported.
+
+### Axis insertion and constant padding
+
+`tensor::unsqueeze(input, axis)` inserts one length-one dimension and returns
+a contiguous tensor with the same values and dtype. Negative axes count from
+the end of the insertion positions; `axis=-1` appends a dimension.
+
+`tensor::pad(input, pad, mode="constant", value=0)` takes an even-length list
+or rank-one I64 tensor of padding pairs. The first pair is before/after axis 0,
+the second is axis 1, and so on. Unspecified higher axes receive no padding.
+Only `constant` mode is supported. Lengths must be nonnegative, the number of
+pairs must fit the input rank, and padded shapes retain the 256 MiB limit.
+The fill value follows the input dtype's scalar-conversion rules.
+An empty or all-zero padding list borrows the input without changing it.
+
+Frame assembly and constant padding use tensor operations directly.
+For example, turn one image into a video tensor and append constant frames:
+
+```dgl
+image = image::load("input.png")
+first_frame = tensor::unsqueeze(image::to_f32(image), axis = 2)
+frames = tensor::pad(first_frame, [0, 0, 0, 0, 0, 32], value = 0.5)
+```
+
+Image normalization runs before padding, so `value=0.5` means half-intensity F32
+pixels. The frame axis remains explicit even for a single frame. Use
+`tensor::concat` to assemble multiple frames and `tensor::pad` for constant
+padding. Negative padding lengths fail instead of discarding input frames.
+
+### Tensor values as lists
+
+`tensor::to_list(input, element="auto")` explicitly converts a rank-one tensor
+into a script list for `map`, `fold` or list functions. U8, I32 and I64 produce
+`list<int>`; F32 and F64 produce `list<number>`. Integers are read directly as
+int64 values, preserving I64 values above 2^53. NaN and infinity are rejected.
+An empty tensor produces an empty list with the same element type.
+
+```dgl
+fn add_seed(total, seed) {
+ return total + seed
+}
+
+fn main() {
+ batch_seeds = tensor::to_list(tensor::arange(42, 46))
+ empty = tensor::to_list(tensor::arange(42, 42))
+ return {
+ seeds = batch_seeds,
+ sum = fold(batch_seeds, 0, process = add_seed),
+ empty_sum = fold(empty, 0, process = add_seed)
+ }
+}
+```
+
+The seeds are `[42, 43, 44, 45]`, their sum is `174`, and the empty fold
+returns its initial value `0`. A map callback can also construct an RNG from
+each seed and return an independent noise tensor.
+
+The compiler must know the list element type before checking the map or fold
+function. With `auto`, it uses a known tensor dtype, such as I64 from `arange`
+or the declared dtype of `tensor::vector`. If dtype is unknown, including an
+unconverted `tensor::load()` result or a host tensor parameter, declare it:
+
+```dgl
+items = tensor::to_list(tensor::load("seeds.npy"), element = "int")
+```
+
+`element` must be known at compile time. It declares a type category and does
+not cast values: F32 with `element="int"` and I64 with `element="number"` are
+errors. Use `tensor::cast` before conversion to change dtype explicitly. Known
+rank or dtype mismatches fail compilation; runtime validation checks the actual
+input as well. Multidimensional tensors require an explicit reshape first.
+Each element creates a script value, so the limit is 10,000 items, including
+when the input came from a file or the host.
+
+The former `seeds(seed, count)` function is removed. Use
+`tensor::to_list(tensor::arange(seed, seed + count))`, or a plain integer for a
+single fixed seed. Integer range bounds do not wrap at the int64 limit.
+
+### Scalar arithmetic
+
+Scalars support `+`, `-`, `*` and `/`, parentheses, and unary `+` and `-`.
+Multiplication and division bind before addition and subtraction; binary
+operators of equal precedence associate left to right. Indexing binds before
+unary signs, so `-sigmas[0]` negates the selected scalar.
+
+Integer `+`, `-` and `*` preserve the integer type and reject int64 overflow.
+A floating-point operand produces a floating-point result. `/` always returns
+a floating-point value and rejects division by zero. Use `math::floor`,
+`math::ceil` or `math::ceil_div` when an integer count is required.
+
+```dgl
+missing = count - 1
+area = width * height
+retained = math::floor((steps - 1) * strength)
+sigma = sigmas[count - 13]
+```
+
+Operators accept numeric scalars. Use `tensor::add`, `tensor::mul` and other
+explicit tensor functions for tensor arithmetic. Operators lower to the same
+`math::add`, `math::sub`, `math::mul` and `math::div` implementations as calls.
+The `var` block continues to accept literals, including signed numbers;
+write computed expressions in function bodies.
+
+### Tensor files
+
+Use `tensor::save` to write a tensor from any step to a NumPy `.npy` file.
+The former top-level names `load_tensor` and `save_tensor` have been removed;
+use `tensor::load` and `tensor::save`, with the same parameters and behavior.
+Saving functions take the output path first, followed by the data to write.
+Saving functions have no return value. Call them as standalone statements;
+return tensors or media separately when they should appear in the result.
+`tensor::load` reads the same format, including files exported by `--tensor`.
+
+```dgl
+# version: 0.0.0
+
+fn main() {
+ latent = tensor::load("latent.npy")
+ tensor::save("latent-copy.npy", latent)
+ return latent
+}
+```
+
+`tensor::load` supports `u8`, `i32`, `i64`, `f32` and `f64` tensors up to rank 8
+and 256 MiB. The NumPy shape is the reverse of the DGL shape, so a DGL
+image shaped `[3, width, height]` appears as `(height, width, 3)` in NumPy.
+
+## Independent generation components
+
+A generation script loads the model components, encodes the prompt, builds a
+predictor and a noise schedule, then samples and decodes the latent. See
+[model loading](#model-loading-and-reuse), [prompt formatting](#prompt-encoding),
+[predictors](#predictors-and-guidance) and [sampling](#noise-schedules-and-sampling).
+
+### Model loading and reuse
+
+Use `load_unet` for UNet architectures such as SD 1.x, SD 2.x and SDXL,
+and `load_dit` for diffusion transformers. The names resolve to the same loader.
+`load_text_encoder` loads the text encoder and `load_vae` loads the VAE.
+A single checkpoint can supply all three components, as in
+[SD 1.5 text-to-image](dgl/sd/sd1-5-txt2img.dgl).
+For standalone weights whose tensor names are shared across model families,
+set `family` to a lowercase [model family ID](dgl-builtins.md#model-family-ids)
+such as `"sd1"`, `"flux"` or `"qwen_image_2_1"`. With `family = "auto"`,
+you can pass a diffusion checkpoint as `family_from` to detect the family from
+its metadata. Use `vision_path` when the text encoder's vision weights are in
+a separate file. `path` selects the main text encoder weights; use `clip_l_path`,
+`clip_g_path` and `t5_path` when those components are in separate files.
+
+Component loaders accept `backend`, `params_backend`,
+`wtype`, `tensor_type_rules`, `split_mode`, `flash_attn_mode`, `conv_direct_mode`
+plus `sage_attn`. Pass one setting per loader parameter.
+Use `"auto"`, `"on"` or `"off"` for `flash_attn_mode` and `conv_direct_mode`.
+These options apply only to the model returned by that loader. For example:
+
+```dgl
+unet = load_unet("sdxl.safetensors", flash_attn_mode = "on", conv_direct_mode = "off")
+vae = load_vae("sdxl.safetensors", flash_attn_mode = "off", conv_direct_mode = "on", conv_scale = 0.03125)
+```
+
+To place two components separately, set `backend = "cpu"` on the text encoder
+and `backend = "cuda0"` on the DiT. The DiT and text encoder
+also accept device lists such as `"cuda0&cuda1"` for split execution.
+
+Loader option `conv_scale` overrides Conv2D input scaling, including grouped
+Conv2D. Output scaling is compensated after convolution. The default `0`
+keeps model defaults. For an SDXL VAE, set `conv_scale = 0.03125` explicitly,
+including when loading the VAE from a separate file. Use `conv_scale = 1` for unscaled convolution.
+This does not change VAE latent normalization or model weights.
+
+Model-loading functions accept literals and scalar literal bindings from
+`var` or `main()`.
+An execution context caches components by their weight sources and loader
+options. Programs run in the same context can share a component even if they
+give it different variable names. The cache checks each loader's declared
+files, including its configured tokenizer slots. Encoded conditions also track any textual inversion embeddings
+used by the prompt. Each run calls generation and image-processing
+functions with its own inputs and sampling state. A context permits one active run.
+
+The CLI defaults to `--run-once on`, which releases models after their last use.
+Use `--run-once off` to cache models for reuse during the invocation, subject to
+the context's memory limits. Freeing the context releases its cache.
+
+The executor tracks the last use of each binding and dictionary field. A component
+stays loaded while another live value, such as a predictor, still uses it.
+Repeated loads share a live component. With `--run-once on`, loading the same
+component again after its last use reads its weights again.
+
+### LoRA combinations
+
+`load_lora` creates a reusable adapter handle. `apply_lora` registers the
+adapter's weights and derives a model of the same type with its own ordered
+adapter list. Both models can be used independently.
+For example, a low-resolution pass can use `draft` and a later pass can add a
+second adapter for refinement.
+
+```dgl
+base = load_unet("sd15.safetensors")
+style = load_lora("style.safetensors")
+detail = load_lora("detail.safetensors")
+draft = apply_lora(base, style, weight = 0.8, mode = "immediately")
+refine = apply_lora(draft, detail, weight = 0.4)
+```
+
+The optional `mode` applies to the entire returned combination. Omitting it
+inherits the input model's mode. The same adapter handle can be used on any
+weighted model when the adapter contains matching parameters for that model.
+
+`at_runtime` applies adapters in the forward graph. `immediately` builds and
+caches merged buffers for the affected parameters on first use. Compatible
+combinations share the original parameter buffers and the adapter tensors
+across both modes. Changing mode or strength can reuse the original weights
+while they remain resident. Persistent contexts retain idle buffers subject
+to RAM and VRAM pressure. Run-once contexts can release shared buffers after
+their last use.
+Automatic mode defaults to runtime LoRA in persistent contexts. For run-once
+DiTs and text encoders, automatic mode also considers quantization and
+backend placement.
+
+### Prompt encoding
+
+`encode_prompt` returns named conditioning tensors. `predictor` binds them to a
+model.
+Wan and LingBot encoders zero UMT5 embeddings at padding positions before
+returning the condition, as required by their fixed-length diffusion text input.
+Embeddings for actual prompt tokens are unchanged.
+
+`encode_prompt` uses the text encoder's model default for an omitted or
+nonpositive `clip_skip`, including the penultimate CLIP layer for SD2 and SDXL.
+Explicit positive values override that default. Requesting a layer whose weights
+are absent reports an encoding error.
+
+LLM-based encoders accept optional `system` and `raw` arguments. This includes
+the Qwen/Mistral/GPT-OSS/Gemma paths, Anima, Fibo, Ming-Image, LLaDA-Image,
+LTX, HiDream-O1 and SenseNova U1.5.
+
+- Omit `system` to keep the model's default prompt formatting.
+- Set `system="custom instruction"` to replace only the system text, retaining
+ the encoder's role markers and the rest of its template. Encoders without a
+ default system segment insert one using their role format.
+- Set `system=""` to omit the whole system segment, including its role markers.
+ An encoder without a default system segment keeps its existing template.
+- `raw=false` is the default. With `raw=true`, `text` is the complete tokenizer
+ input: no chat template, caption rewriting, prompt-weight parsing or inline
+ image/embedding resolution. Model-required padding, hidden-layer selection,
+ learned queries and projections still apply.
+
+Passing `system` together with `raw=true` is an error, even for `system=""`.
+Raw text mode also rejects `media` and `image_bindings` arguments;
+use normal template mode for multimodal input. System changes update template
+crop offsets and prefix-dependent padding, and image token positions are
+computed from the resulting prefix. Default formatting remains unchanged.
+CLIP and T5-only encoders reject these formatting overrides.
+
+For an already loaded LLM text encoder:
+
+```dgl
+custom = encode_prompt(text, prompt, system = "Describe the requested image.")
+without_system = encode_prompt(text, prompt, system = "")
+raw = encode_prompt(text, tokenizer_input, raw = true)
+```
+
+The former `unconditional` parameter has been removed. SenseNova's legacy
+negative format also omits `<think>`, so `system=""` does not reproduce it.
+Use the full negative tokenizer input instead:
+
+```dgl
+negative = encode_prompt(text,
+ "<|im_start|>user\nblurry<|im_end|>\n<|im_start|>assistant\n<img>",
+ raw = true)
+```
+
+Native CLI/C API generation constructs that complete negative input automatically,
+for empty and nonempty negative prompts. Ordinary CLIP negative prompts use the
+same encoding as positive prompts and need neither `system` nor `raw`.
+
+#### Media inputs
+
+`media` supplies ordered inputs to the encoder's existing model-specific prompt
+template. An image tensor is shorthand for `{image = tensor}`. Image-only
+lists can use either form; mixed media lists use dictionaries so every list
+element has the same DGL type:
+
+```dgl
+clip = video::load("inputs/clip.webm")
+sound = audio::load("inputs/voice.wav")
+condition = encode_prompt(text, prompt, media = [
+ {image = source_image},
+ {video = clip.video, stride = 12},
+ {audio = sound}
+])
+```
+
+Each dictionary has exactly one `image`, `video` or `audio` field. Video uses
+`{frames, fps_num, fps_den}`; audio uses `{samples, sample_rate}`. Only video
+accepts `stride`, an integer frame interval in `1..65536`, defaulting to `1`.
+Sampling retains source-frame timestamps. A video does not implicitly include
+its soundtrack: add `{audio = clip.audio}` separately when needed. Unsupported
+media types or image counts are errors. MiniMax H3's audio prompt input adds
+its audio marker; audio encoding and predictor references remain separate.
+
+`image_bindings = {subject = tensor}` supplies images referenced by
+`<image:subject>` in the text. Explicit bindings take precedence over file
+paths. An invalid referenced binding is an error; an absent name uses the
+existing path reader and file-access callback. Unreferenced bindings do not
+enter conditioning or its cache key. `raw=true` rejects `media` and
+`image_bindings`, including explicitly empty containers.
+
+Multimodal prompt encoding can consume reference-video frames independently:
+
+```dgl
+condition = encode_prompt(text, prompt, media = [{ video = {frames = frames, fps_num = 24, fps_den = 1}, stride = 12 }])
+```
+
+A `video` media entry takes the same video dict. Its `frames` can be a
+U8 `[3,w,h,t]` or F32 `[w,h,t,3,1]` tensor. `stride` defaults to 1 frame. The encoder receives selected frames with timestamps
+`frame_index * fps_den / fps_num`. At 24 FPS, stride 12 selects timestamps
+0, 0.5, 1.0 seconds and so on. Only selected frames are converted to the
+encoder's float pixel layout. Audio media uses `{audio = sound}`, where
+`sound` is the same samples/sample-rate dict used by audio I/O.
+
+#### Textual inversion
+
+For textual inversion, use an inline file path with an SD 1.x, SD 2.x or SDXL
+CLIP encoder:
+
+```dgl
+text = load_text_encoder(model_path)
+negative = encode_prompt(text, "<embedding:embeddings/badhandV4.pt>, blurry")
+```
+
+`<embedding:...>` follows the same path and escaping rules as `<image:...>`.
+Paths resolve from the process working directory. An escaped marker is ordinary
+text, and a bare filename or former trigger word is also ordinary text. Other
+encoders reject embedding markers. The vectors replace token embeddings before
+CLIP adds positional embeddings; prompt weights, token chunks and SDXL dual-CLIP
+zero-padding still apply. No tokenizer aliases or loader `embeddings` argument
+are used.
+
+Every encoding checks `SD_DGL_FILE_READ_MODEL` permission before inspecting the
+files or looking up a cached condition. Cached conditions track canonical paths
+and file versions, including referenced shards. Replacing an embedding invalidates
+the condition without rebuilding the text encoder. The server's model directory
+allowlist also applies to these paths. Native CLI/C API textual inversion is
+unchanged.
+
+#### Conditioning fields
+
+`zero_out(condition, fields = ["crossattn"])` returns a new
+condition dict with only the selected tensor fields cleared. Omit `fields` to
+clear every tensor field and every element of tensor lists. Shapes, dtypes and
+non-tensor metadata are preserved; the input dict is unchanged. A selected field
+must exist and contain a tensor or tensor list. Clearing a condition is a tensor
+operation, and does not make it valid unconditional input for every model.
+For SDXL, clear text embeddings while preserving size and crop conditioning:
+
+```dgl
+fn sdxl_zero_text(cond) {
+ zeroed = zero_out(cond, fields = ["crossattn"])
+ pooled_zeros = tensor::zeros_like(cond.vector[:1280])
+ adm = tensor::replace_slice(cond.vector, pooled_zeros, axis = 0, start = 0)
+ return {crossattn = zeroed.crossattn, vector = adm}
+}
+```
+
+The first 1280 values of SDXL's `vector` contain pooled text embeddings; its
+remaining 1536 values encode size and crop information. This helper clears
+`crossattn` and pooled text only. The CLI/C API calls it for an empty negative
+prompt with non-EDM-v prediction. The SDXL examples expose `zero_negative`
+as an explicit choice: set it to `false` to use the encoded negative prompt,
+including nonempty negative text. Both branches return the two SDXL condition
+fields, `crossattn` and `vector`, to keep their static dictionary types identical.
+
+A conditioning dictionary can use tensors in the fields
+consumed by `predictor`, including model-specific token and reference metadata.
+
+### Predictors and guidance
+
+`predictor` binds a condition to a diffusion network, with optional `concat` and `ref_latents` inputs. The
+[Qwen Image 2.1 editing program](dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl) passes the
+reference pixels to the text encoder and their VAE latent to the predictor.
+[Text-to-image](dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl) uses the same
+components with text conditions.
+Reference images are prepared by the script. DGL does not select reference
+presets or apply a reference resize policy based on the model family.
+Use `tensor::interpolate` and `tensor::crop` before encoding. Pass prepared
+pixels to `encode_prompt` and prepared latents to `predictor` explicitly.
+`predictor(ref_index_mode="increase")` selects reference position indexing;
+the default is `"fixed"`. Use `"decrease"` for layered references and set
+`force_ref_timestep_zero=true` when required by the checkpoint.
+
+`prediction` belongs to `predictor`, for example
+`pos = predictor(dit, condition, prediction="eps")`. `load_dit` and its
+`load_unet` alias load the model without selecting a prediction parameterization.
+Omitting `prediction` selects `"auto"`, using the model's declared defaults and,
+when needed, its checkpoint detection. The explicit values are `eps`, `v`,
+`edm_v`, `flow`, `flux_flow`, `sefi_flow`, `minit2i_flow`, `sensenova_u1_flow`,
+`flux3_flow`, `h3_av_flow`, and `ming_image_flow`.
+
+`predictor(transform=step_fn, transform_args={...})` calls a script function
+for each model evaluation. Its two arguments are the step dictionary and
+the supplied `transform_args`. The step contains `sigma`, the model's
+`timestep`, and `input_scale`, `skip_scale`, and `output_scale`. Return a
+dictionary with the timestep or scale fields to override. `denoiser_scalings`
+returns the standard coefficients for a supplied sigma. The
+[NitroFusion example](dgl/sd/nitrosd-realism-nitrofusion.dgl) defines its timestep
+rounding and output scaling in this function.
+
+`cfg(pos, neg, scale=4.5, schedule=[7, 6, 5])` uses a scale per step and
+falls back to `scale` after the list ends. `cfg` and `apg` both accept
+`image_only` and `image_scale` for image editing. APG uses the underlying
+model outputs and requires matching prediction scalings across its branches.
+Each `sample` call starts its own APG momentum.
+
+SLG takes a base prediction, a complete positive branch, and a skipped branch.
+It adds `scale * (positive - skipped)` to the base prediction. Reusing the same
+positive predictor in APG or CFG and SLG shares the forward pass within each
+model evaluation.
+
+```dgl
+pos = predictor(dit, pos_cond)
+neg = predictor(dit, neg_cond)
+skipped = predictor(dit, pos_cond, skip_layers = [7, 8, 9])
+guided = apg(pos, neg, scale = 4.5, eta = 0, momentum = -0.5)
+pred = slg(guided, pos, skipped, scale = 2.8, end_sigma = t_to_sigma(pos, t = 0.8))
+latent = sample(start, sigmas, predict = pred, rng = rng)
+```
+
+`start_sigma` and `end_sigma` define an inclusive sigma window for SLG.
+To use the step-based window, set `start_step_fraction` and `stop_step_fraction`.
+The step window uses the sigma tensor length (including its endpoint) times
+each fraction, rounded down. Only step indices strictly inside the window
+activate SLG. When both windows are provided, SLG applies within their
+intersection.
+
+Guidance schedules, SLG step windows, and `switch_predictor(after_steps=...)`
+use logical steps starting at one. Multiple model evaluations within one solver
+step, such as Heun's predictor and corrector, use the same step index.
+
+CFG++ samplers request the unconditional prediction from the supplied
+predictor. SLG and `switch_predictor` can forward this output from a two-branch
+CFG or APG predictor. Every reachable switch branch must supply it; single-branch
+predictors and CFG/APG with `image_only` are rejected before model execution.
+
+### Random streams
+
+`make_rng("cuda", seed = 42)` creates a seeded RNG object. The algorithm
+can be `"cuda"` (Philox on every backend), `"cpu"` (MT19937), or
+`"std_default"` (also `"std"`). `vae_encode`, `tensor::randn_like` and `sample`
+each require an RNG object. Pass the same object to advance one stream in
+execution order. Each call to `make_rng` creates an independent stream,
+including when a cached execution context runs a script again.
+For example, `tensor::randn_like(base, rng = make_rng("cpu", seed = 42))`
+uses a separate MT19937 stream.
+
+For the same RNG sequence as the generation C API, generate the initial noise with
+`tensor::randn_like(base, rng = rng)`, pass that tensor to `add_noise`, then pass
+the same `rng` to `sample`. For images, start a new stream from the image seed
+before initial noise, and start another before the hires pass, including any
+hires VAE re-encoding. For video, share the request stream from VAE encoding
+through high- and low-noise sampling and hires refinement.
+
+### Noise schedules and sampling
+
+The sampler receives a start tensor, a rank-one F32 sigma tensor and a
+predictor. `make_sigmas` returns a schedule tensor, and
+`tensor::vector([1, 0.5, 0])` builds one from explicit values.
+`sigmas[0]` reads the starting sigma. Return a schedule by name
+to export it as one `.npy` array. A single zero sigma requests zero sampling
+steps for ordinary samplers. `flow_shift` on `make_sigmas` configures that
+table. Sampling uses the table's values and the predictor's inputs.
+
+`make_sigmas`, `add_noise`, `t_to_sigma`, and `denoiser_scalings` take a predictor
+as their first `predict` argument. These queries do not evaluate the predictor.
+Build the required prompt/condition and predictor before constructing its
+schedule or adding noise. Changing prediction does not reload the model or
+its LoRA variant.
+
+`make_sigmas` and `t_to_sigma` accept a composition when its branches share
+one `DenoisingParameterization` object. Predictors from the same model with the
+same resolved `prediction` share that object, so ordinary CFG can use `guided`
+for its schedule, noise addition and sampling:
+
+```dgl
+pos = predictor(unet, pos_cond, prediction = "eps")
+neg = predictor(unet, neg_cond, prediction = "eps")
+guided = cfg(pos, neg, scale = cfg_scale)
+sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+latent = sample(start, sigmas, predict = guided, method = "euler_a", rng = rng)
+```
+
+Different models or different parameterization objects make the schedule source
+ambiguous, even when their noise conventions or scaling rules are compatible.
+Pass the branch that defines the desired schedule in that case. For example,
+Wan 2.2 can use `make_sigmas(lo, ...)` for the low-noise model's schedule and
+`add_noise(hi, ...)` before sampling with the high-noise model.
+`denoiser_scalings` accepts a composition only when every branch has the same
+scaling rules. For example,
+a mixture of `eps` and `v` has no common coefficients, even if a queried sigma
+happens to give equal values; query the desired leaf instead. `v` and `edm_v`
+share scaling rules, but have different time mappings.
+
+`add_noise` and sampling finish conversion require only a common noise
+convention, so sampling still supports CFG or switched predictors with different
+parameterizations such as `eps` and `v`. Compatible CFG branches with different
+parameterizations execute separately. APG additionally requires matching
+raw-output scalings.
+
+`make_sigmas` defaults to the `"discrete"` scheduler and rejects `"auto"`.
+Choose a different scheduler explicitly when the model requires it. Omit
+`flow_shift` when no override is needed; its default is zero.
+
+`sample(method=...)` defaults to `"euler"`. Choose a sampling method explicitly
+when another algorithm is needed; `"auto"` is not accepted. `sefi::sample` also
+defaults to `"euler"` and only supports that method.
+
+`t_to_sigma(predict, t, flow_shift=...)` converts a normalized noise time
+before the shift to a sigma. `t=1` is the noisy end, and `t=0` is the
+low-noise end of the model's time mapping. A model with a discrete training
+grid can have a positive minimum sigma. Pass `latent` or `sequence_length`
+for time mappings that depend on the latent size.
+
+In the [Wan 2.2 image-to-video program](dgl/wan/wan2-2-i2v-a14b.dgl), bracket slices
+include the same boundary sigma in both sampling segments. The second Euler segment
+takes the first segment's noisy output. Its final latent, the handoff and the
+decoded pixel tensor can all be returned. The
+[Wan 2.1 text-to-video program](dgl/wan/wan2-1-t2v-1-3b.dgl) uses one predictor
+and the full schedule. `video::save` accepts a video dict containing decoded RGB
+frame tensors and `fps_num`/`fps_den`, and writes AVI.
+
+### Sampling caches
+
+Pass `cache(mode="ucache", reuse_threshold=0.8)` to `sample(cache=...)` to
+enable a sampling cache. Available modes are `disabled`, `easycache`, `ucache`,
+`dbcache`, `taylorseer`, `cache_dit` and `spectrum`. Support depends on the
+denoiser and sampler. Spectrum requires one denoiser, one sigma schedule and a
+sampler without CFG++. Each sample call owns its cache state. The runtime
+prepares each denoiser and active conditioning component on first use in that
+call and reuses its execution scope for the remaining steps. With
+`switch_predictor`, a later branch enters execution when the solver reaches it.
+Spectrum discards its extrapolation history when an active switch changes
+branches or an active CFG/APG guidance scale changes. The new configuration
+starts with real model evaluations before extrapolation resumes. Guidance
+changes inside unused branches do not invalidate the active history. Active SLG
+nodes are part of the key even when they share every model leaf with the base
+predictor. For CFG++, only the branch that supplies the root unconditional
+output is additionally retained; nested guidance does not inherit that demand.
+With caching disabled, execution skips the activity-key traversal entirely.
+
+Condition-level caches (`easycache`, `ucache`, `dbcache`, `taylorseer` and
+`cache_dit`) store separate histories for model conditions. Only enabled caches
+prevent CFG batching; an unsupported cache request leaves batching available.
+When an active condition leaves, its cache is invalidated. If it was the anchor,
+the cache selects a new active condition and rebuilds its calibration.
+
+DGL's `dbcache` reuses whole-model input/output residuals after checking input
+change against `residual_diff_threshold`. `taylorseer` extrapolates each
+condition's raw model output using `taylorseer_n_derivatives`; after enough real
+samples, `taylorseer_skip_interval` bounds the number of extrapolated steps
+between real evaluations. `cache_dit` combines that extrapolation with the
+residual-difference gate. These are condition-level algorithms, not block-level
+DBCache/TaylorSeer. `Fn_compute_blocks`, `Bn_compute_blocks` and
+`scm_policy_dynamic` are not DGL options. Model-block caching requires a runner
+integration and is not exposed by this path.
+
+`max_warmup_steps`, `max_cached_steps`, `max_continuous_cached_steps`, and a `1`
+in `scm_mask` force real computation as applicable; forced steps still update
+history. Negative values for `max_cached_steps` and `max_continuous_cached_steps`
+leave the corresponding limit unset. Condition-level reuse only runs on
+completed-step evaluations; intermediate solver evaluations execute the model
+normally.
+
+Spectrum owns the composed predictor's output history independently. Its
+`spectrum_warmup_steps` counts logical solver steps, and
+`spectrum_stop_percent` sets the stopping position on that same timeline.
+Heun's predictor and corrector use separate histories at their logical step
+indices; neither evaluation advances the other's warmup. Active branch or
+guidance changes reset both histories. Real Spectrum evaluations can still use
+batched CFG. Logs report reused predictor evaluations, rather than dividing
+Heun's evaluation count by the number of sigma intervals.
+
+### Sampling previews
+
+Load a tiny autoencoder with `load_tae("tae.safetensors", family = "sd1")`, or use
+`family_from` to identify its model family from a diffusion checkpoint.
+Pass it to `vae_encode` or `vae_decode` like any other autoencoder.
+For preview-only use, set `decode_only = true` on `load_tae` and pass it to
+`preview(decoder=...)`. Attach the returned observer to `sample` or
+`sefi::sample`. Load the final decoder separately with `load_vae`.
+
+```dgl
+tae = load_tae("tae.safetensors", family = "sd1", decode_only = true)
+observer = preview(decoder = tae)
+latent = sample(start, sigmas, predict = guided, rng = rng, observer = observer)
+```
+
+`preview()` needs no decoder for latent projection. The context's preview
+callback selects the mode, interval and noisy or denoised previews. A sampling
+call without an observer uses a shared projection preview observer. Without a
+decoder, preview falls back to projection. An explicit observer is only needed
+to supply a decoder or custom preview settings. An observer can be reused
+across sampling calls; each call has its own observation state. The observer
+retains its decoder, and decoding resources are acquired only for a selected
+preview and released afterwards. Without an enabled preview callback, it
+performs no preview decoding.
+
+### Media dimensions
+
+Model functions adapt pixel inputs to their own spatial requirements. Dimensions
+are rounded to the nearest legal multiple, with half-way values rounded up and
+at least one multiple retained. Already compatible inputs pass through without
+resizing. Automatic resizing uses Lanczos filtering; it does not impose a
+resolution or pixel-area budget. Use `tensor::interpolate` or `tensor::crop`
+before encoding when you want to choose the geometry yourself.
+
+`vae_encode` normally aligns to the VAE's spatial compression factor.
+`encode_prompt` independently applies its vision encoder's requirements; the
+two paths can see different sizes of the same reference. Qwen Image 2.1 has an
+additional model constraint: each vision image slot expands into four reference
+latent tokens. Its VAE input therefore aligns to 32 pixels, while the spatial
+compression factor remains 16. Neither path changes the source tensor. Existing
+fixed-size vision preprocessing still performs its required crop and normalization.
+
+`latent_shape` aligns requested pixel dimensions to the model family's diffusion
+and VAE grid before returning a latent shape. Use that shape for the output
+canvas; encoded reference latents can have a different shape. Both
+`latent_shape` and video `vae_encode` round temporal lengths up to the supported
+grid: `1 + 4k` for Wan, LingBot, Hunyuan Video and SeedVR2, `1 + 8k` for LTX,
+and `5 + 17k` for H3 video. Encoding repeats the last frame when padding is
+needed, preserving every input frame. Single-image encoding remains a single
+image. Decoding returns the actual aligned size; there is no automatic crop
+back to the original dimensions. For joint audio/video scripts, derive audio
+lengths from the aligned video duration.
+
+`detections::detect` rounds positive `input_size` values to a multiple of 32
+(minimum 32, maximum 4096); detections remain in original-image coordinates.
+The H3 latent upscaler similarly aligns positive target dimensions to even
+latent units. LTX upscaling still gets its fixed scale from the checkpoint.
+Only actual adaptations emit verbose logs. Invalid ranks, channels, nonpositive
+dimensions and incompatible conditioning relationships remain errors.
+
+### VAE latent spaces
+
+`vae_encode(vae, pixels, rng)` returns a latent in the VAE's own space.
+`vae_to_dit_latents(vae, latent)` applies the model's scaling, shift
+or normalization before sampling. After sampling, use
+`dit_to_vae_latents(vae, latent)` before `vae_decode(vae, latent)`.
+Both spaces use float tensors. A saved tensor needs the matching conversion
+for its space and VAE weights.
+`vae_to_unet_latents` is an alias of `vae_to_dit_latents`, and
+`unet_to_vae_latents` is an alias of `dit_to_vae_latents`; each alias resolves
+to the same function and uses the same VAE-specific conversion.
+Use the UNet names with UNet models and the DiT names with diffusion transformers.
+
+`vae_encode` defaults to `posterior="sample"`. Set `posterior="mode"` to
+take the posterior mode, as InstructPix2Pix does for its reference image.
+The mode leaves the RNG state untouched. The two options give the same result
+for deterministic encoders.
+
+```dgl
+rng = make_rng("cuda", seed = seed)
+reference = vae_encode(vae, pixels, rng = rng, posterior = "mode")
+predict = predictor(unet, condition, concat = reference)
+latent = sample(start, sigmas, predict = predict, rng = rng)
+image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+```
+
+`vae_encode` and `vae_decode` accept `tiling=tiling(tile_size_w=32,
+tile_size_h=32)`. Use `temporal_tiling=true` for supported video VAEs.
+`predictor(circular_x=true, circular_y=true)` enables circular convolution
+padding for that prediction branch and its active ControlNets. Branches can
+share a model while using different settings; each forward call restores the
+previous settings afterwards. Model weights are reused.
+
+Set the same arguments independently on `vae_encode`, `vae_decode`, or
+`preview(decoder=..., circular_x=true, circular_y=true)` for VAE operations
+and preview decoding. Latent projection does not use convolution padding.
+Configure predictor, VAE and preview padding consistently for seamless
+generation. `sample` and `sefi::sample` do not accept circular padding options.
+
+Load an F32 latent with `tensor::load` and use the matching latent space for the
+receiving function.
+
+Use `vae_decode(..., dtype = "f32")` to postprocess normalized float pixels in
+`[width, height, channels, batch]` or `[width, height, time, channels, batch]`
+layout. The default `"u8"` returns packed pixels.
+
+## Image and video files
+
+Media I/O uses `image::load` / `image::save`, `video::load` / `video::save`
+and `audio::load` / `audio::save`. The former global names and the aliases
+`read_image`, `read_video`, `read_audio` and `save` have been removed.
+Model loading, inference and latent operations keep their current names.
+`image::to_f32` and `image::to_u8` convert pixel representations for both images
+and video frames; ordinary tensor dtype changes use `tensor::cast`.
+
+### Image files
+
+`image::load(path)` decodes PNG, JPEG, BMP and other formats supported by
+stb_image into a U8 tensor shaped `[channels, width, height]`. Grayscale with
+alpha is decoded as RGBA. `image::save(path, pixels)` writes a U8 pixel tensor
+to PNG and returns no value. A write failure interrupts execution.
+
+```dgl
+# version: 0.0.0
+fn main() {
+ pixels = image::load("input.png")
+ patch = tensor::crop(pixels, axes = [1, 2], start = [16, 32], size = [256, 256])
+ output = tensor::interpolate(patch, axes = [1, 2], size = [512, 512])
+ image::save("output.png", output)
+ return output
+}
+```
+
+Use `mode="nearest_exact"` for half-pixel nearest-neighbor sampling.
+`mode="nearest_max"` uses nearest-neighbor sampling when enlarging and the
+maximum of each source region when shrinking, preserving active mask pixels.
+
+### Pixel formats
+
+`image::to_f32(image)` converts image/video pixels to the F32
+representation used by VAE and conditioning functions:
+
+| Input | Output | Values |
+|---|---|---|
+| U8 image `[c, w, h]` | F32 `[w, h, c, 1]` | Divided by 255 |
+| U8 video `[c, w, h, t]` | F32 `[w, h, t, c, 1]` | Divided by 255 |
+| F32 pixels in either output layout | Same F32 layout | Preserved without rescaling |
+
+The default preserves the input channel count, including masks and alpha.
+Set `channels=3` to request RGB or `channels=4` for RGBA. `channels=0` explicitly
+selects the default preservation behavior. Supported channel conversions
+are grayscale to RGB/RGBA, RGB to RGBA (opaque alpha), and RGBA to RGB (drop
+alpha). One- or two-channel data can be preserved, but RGB-to-grayscale and
+grayscale-with-alpha conversion are not provided by this function. F32 input
+with preserved or matching channels is borrowed without copying. For example,
+`image::to_f32(mask)` keeps a single-channel mask, while a ControlNet RGB hint
+can use `image::to_f32(hint, channels=3)`.
+
+`image::to_u8(pixels)` reverses the layout conversion. It clamps normalized F32
+pixels to `[0, 1]`, scales by 255 and rounds to U8, producing `[c, w, h]` for an
+image or `[c, w, h, t]` for a video. It also accepts U8 pixels. For example,
+`image::save("output.png", image::to_u8(pixels))` saves a processed F32 image.
+Both functions return tensors; they introduce no separate image value type.
+
+### Encoded media and bytes
+
+`image::encode(image, format="png")` encodes a U8 pixel tensor in memory.
+`video::encode(video, audio?, format="webm")` encodes a complete video using
+the video dict's `fps_num` / `fps_den`. An optional audio dict is supported
+with `format="avi"`. Both return an ordinary `{data: bytes, mime_type: string}` dict.
+
+`bytes` is an immutable binary value, including embedded zero bytes. Assignments,
+arguments and return values share its buffer. It is distinct from strings and
+U8 tensors and is not converted to a flat tensor result.
+
+```dgl
+cover = image::encode(pixels)
+clip = video::encode(video)
+media::save("output/cover.png", cover)
+media::save("output/clip.webm", clip)
+return {cover = cover, clip = clip, latent = latent}
+```
+
+`media::save(path, media)` writes these exact bytes, with no re-encoding or
+metadata changes. It checks file-write permission and the extension against the
+MIME type. Omit the save calls for in-memory results only. The server's DGL
+editor displays returned media from the existing job result; ordinary tensors
+remain available for inspection.
+
+| Encoding | MIME type | Requirements |
+| --- | --- | --- |
+| PNG | `image/png` | Built in |
+| VP8 WebM | `video/webm` | `SD_WEBM=ON`; uses independent VP8 frames |
+| MJPEG + PCM16 AVI | `video/x-msvideo` | `format="avi"`; offered as a download in the editor |
+
+Unavailable encoders fail explicitly. WebM encoding rejects nonempty audio;
+use `video::encode(video, audio=sound, format="avi")` for a PCM16 soundtrack.
+Audio is never silently dropped. See [build options](build.md#webp-and-webm-support).
+`video::load` still supports only PCM16 WebM audio: pass `audio=false` to read
+frames from files with unsupported audio codecs.
+
+Encoding embeds the current program when the context's `embed_metadata` option
+is enabled. `media::save` preserves that decision. The existing `image::save`
+and `video::save` functions remain available for direct file output; AVI file
+saving continues to stream its packets instead of buffering the entire file.
+
+### Reading video
+
+Video frames use U8 tensors shaped `[3,width,height,frame_count]`, with
+interleaved RGB channels. Media I/O groups frames and timing in a normal video
+dict, and samples and sample rate in a normal audio dict. Computation operates
+on the tensors inside these dicts.
+`video::load` detects local files by their contents:
+
+| Input | Supported data | Build option |
+| --- | --- | --- |
+| AVI | 24-bit RGB or MJPEG, PCM or IEEE float audio | Always available |
+| WebP | Static images or one cycle of animation; no audio | `SD_WEBP=ON` |
+| WebM | Independent VP8 keyframes and optional PCM16 audio, as written by this project's CLI | `SD_WEBM=ON` |
+
+WebP animation decoding handles partial frames, blending and disposal before
+returning full RGB canvases. Transparent pixels are composited onto black.
+WebM interframe VP8, VP9, laced packets and delayed audio are unsupported;
+unsupported audio reports an error suggesting `audio=false`. MP4 and segmented
+OpenDML AVI are unsupported. Paths support spaces and Unicode.
+
+```dgl
+# version: 0.0.0
+
+fn main() {
+ input = video::load("input.avi", max_frames = 24)
+ frames = tensor::interpolate(input.video.frames, size = [512, 512], axes = [1, 2])
+ video = {frames = frames, fps_num = input.video.fps_num, fps_den = input.video.fps_den}
+ video::save("output.avi", video, audio = input.audio)
+ return { frames = frames, audio = input.audio }
+}
+```
+
+Save the preceding program as `video.dgl` and run it with:
+
+```sh
+./build-dgl/bin/sd-cli -M dgl --script video.dgl --tensor frames frames.npy
+```
+
+`video::load` returns `{video, audio}`. `video` is a normal dict with `frames`
+(tensor), `fps_num` and `fps_den`; `audio` is a normal dict with `samples`
+(tensor) and `sample_rate`. With no audio stream, or
+with `audio=false`, samples are an empty F32 `[1,0]` tensor and the sample rate
+is 0. Passing this empty audio to `video::save` writes no audio track.
+
+Loading defaults to all frames within the 256 MiB tensor limit. Set `max_frames`
+to a positive value to read only that many initial frames. Exceeding the limit
+fails instead of silently truncating. Frames are decoded as RGB at the first
+video stream's rational frame rate. Setting `fps_num` and `fps_den` resamples
+by repeating or skipping frames: each output timestamp uses the latest source
+frame at or before that timestamp. `max_frames` applies after resampling.
+WebP and WebM use frame timestamps to infer a fixed source frame rate.
+Variable or unknown rates require explicit `fps_num` / `fps_den`, including
+static WebP images without a frame duration. Static WebP returns a single frame
+at the requested rate. For example, `video::load("still.webp", fps_num=24)`.
+Animated WebP is read once regardless of its loop count. Streams must start at
+time zero. AVI streams with dropped frames are unsupported. Audio is limited
+to the selected video duration; AVI accepts PCM or IEEE float samples, and WebM
+accepts PCM16 without codec delay, concatenating packet samples in stream order. Use `audio=false` to read frames
+from files with unsupported audio codecs.
+
+`video::load` also reads an image directory in filename order.
+`image::load_dir(path, max_images=0)` returns a list of individual images in
+filename order, which can have different sizes. A positive `max_images` limits
+the list; zero loads every supported image. Both directory readers accept PNG,
+JPEG and BMP. Use `video::load` when the directory represents a time sequence,
+or `image::load_dir` to process independent images.
+
+`video::split_windows(frames, size, overlap=0)` returns a list of frame tensors.
+`video::join_windows(windows, overlap=0)` joins frame tensors, dropping the specified
+overlap from the start of each window after the first, then concatenates the
+remaining frames without blending. These operations only handle pixels;
+keep frame rate and audio outside the window processing and pass them when saving.
+A single full-length window or a single-window join borrows the input tensor.
+Joined video has a 4 GiB limit. The 256 MiB loading limit
+applies when reading source video.
+
+### Writing video
+
+`video::save(path, video, codec="auto", quality=90)` writes `.avi` only.
+The video dict requires `frames`, `fps_num` and `fps_den`. Both frame-rate
+integers must be in [1,1000000]; timing is explicit.
+Saving accepts already allocated frame tensors larger than the loading limit
+and encodes them one frame at a time. It sets the playback rate without
+resampling frames. Supply `audio` for a soundtrack. `trim_audio=true` truncates
+audio longer than the video duration;
+shorter audio is retained unchanged. The default is `false`.
+
+```dgl
+video = {frames = pixels, fps_num = 24, fps_den = 1}
+video::save("output.avi", video, audio = sound, trim_audio = true)
+processed = {frames = processed_frames, fps_num = input.video.fps_num, fps_den = input.video.fps_den}
+video::save("upscaled.avi", processed, audio = input.audio)
+```
+
+`codec="auto"` selects `mjpeg`; `quality` ranges from 1 to 90, with higher
+values preserving more detail. Use `codec="rgb"` for uncompressed, pixel-exact
+RGB output; it ignores `quality` and produces larger files. Both codecs support
+odd frame dimensions. Audio is saved as float32 by default;
+`audio_format="pcm16"` writes PCM16. AVI output is limited to less than 4 GiB.
+Saving has no return value and does not alter the supplied frames or audio.
+Return frame tensors, frame rate and audio explicitly when needed as graph
+results. Audio trimming applies only to the file. MJPEG decoding need not match
+the input pixels exactly.
+
+### Audio files
+
+`audio::load(path)` reads WAV with PCM8/16/24/32 or float32 samples.
+`audio::save(path, audio, format="f32")` writes WAV, with `pcm16` also available.
+`audio::load` and `audio_vae_decode` return a normal dict with `samples` (F32
+`[channels,samples]` tensor) and `sample_rate` (Hz). Construct the same dict to
+save generated or processed samples. Rates range from 1,000 to 384,000 Hz,
+with 1..32 channels and a 256 MiB sample limit. Samples must be finite.
+
+```dgl
+sound = audio::load("input.wav")
+audio::save("output.wav", sound)
+track = {samples = sound.samples, sample_rate = sound.sample_rate}
+video::save("output.avi", video, audio = track)
+```
+
+`encode_audio(encoder, samples, sample_rate=...)` and
+`audio_vae_encode(vae, samples, sample_rate=...)` accept tensors and an explicit
+sample rate. Their existing model-specific resampling remains in effect.
+`audio_vae_decode` reports the decoder's actual output sample rate.
+
+## Regions and masks
+
+Use detections to select objects, regions to crop rectangles, and masks to
+control which pixels a refinement replaces.
+
+### Rectangular regions
+
+A region collection is an ordinary dict `{width, height, boxes}`. Canvas dimensions
+are integers in `1..16384`; `boxes` is an I64 tensor with shape `[4,N]`, limited to
+10,000 boxes. Each column is `[x,y,width,height]` in source-image pixels and covers
+`[x,x+width)` by `[y,y+height)`. Box sizes must be positive and boxes must lie
+entirely inside the canvas. An empty collection has shape `[4,0]`.
+
+`regions::from_boxes(width, height, boxes)` validates a flat `list<int>` or an I64
+`[4,N]` tensor and constructs the dict. A valid input tensor is shared without a
+copy. `regions::crop(image, regions)` accepts the same dict, checks its canvas
+against the image and returns U8 patches in box order. `regions::paste(background,
+foregrounds, regions)` resizes each patch with bilinear interpolation and copies
+its pixels into the corresponding rectangle. Later boxes overwrite earlier ones,
+including the alpha channel; pasting does not alpha-blend.
+
+```dgl
+r = regions::from_boxes(512, 512, [10, 20, 80, 60, 100, 40, 30, 50])
+selected = {width = r.width, height = r.height, boxes = r.boxes[:, 1:2]}
+patches = regions::crop(image, selected)
+result = regions::paste(image, patches, selected)
+```
+
+The slice `[:,1:2]` preserves the `[4,1]` collection shape. `[:,1]` produces a
+single `[4]` box. Handwritten dicts receive the same validation as constructed
+collections. Boxes can be saved and loaded as ordinary tensors:
+
+```dgl
+tensor::save("boxes.npy", r.boxes)
+restored = regions::from_boxes(r.width, r.height, tensor::load("boxes.npy"))
+selected = {width = restored.width, height = restored.height, boxes = restored.boxes[:, 1:2]}
+```
+
+### Detections
+
+Detection results are ordinary dicts with `xyxy` (F32 `[4,N]`), `scores`
+(F32 `[N]`) and `class_ids` (I64 `[N]`). The fields share the same ordering and
+count, limited to 4,096 detections. Empty results keep shapes `[4,0]`, `[0]` and
+`[0]`. Scores must be finite and in `[0,1]`, class IDs must be nonnegative, and
+coordinates must be finite.
+
+`detections::detect(detector, image, confidence=0.3)` runs the detector once.
+`detections::filter(image, detections, ...)` filters or sorts all three fields by
+the same indices. `detections::box_masks(image, detections)` rasterizes full-canvas
+rectangle masks. Filtering and mask creation require boxes inside the image.
+`detections::to_regions(image, detections)` rounds starts down and ends up,
+clips coordinates to the canvas and skips empty boxes, preserving order.
+
+```dgl
+found = detections::detect(detector, image, confidence = 0.3)
+selected = detections::filter(image, found, top_k = 4, top_k_by = "confidence")
+r = detections::to_regions(image, selected)
+patches = regions::crop(image, r)
+result = regions::paste(image, patches, r)
+```
+
+`regions::plan_mask_crops` returns per-mask `{box, mask, index}` tasks for `fold`,
+not a region collection. Masked compositing can be expressed with a crop,
+`tensor::lerp` and slice replacement:
+
+```dgl
+background = tensor::crop(image, axes = [1, 2], start = box[:2], size = box[2:])
+patch = tensor::lerp(background, foreground, mask)
+rows = tensor::crop(image, axes = [2], start = [box[1]], size = [box[3]])
+band = tensor::replace_slice(rows, patch, axis = 1, start = box[0])
+result = tensor::replace_slice(image, band, axis = 2, start = box[1])
+```
+
+### Detail refinement
+
+`detections::detect(detector, image, confidence=0.3)` returns a dict with F32
+`xyxy` `[4,N]`, F32 `scores` `[N]` and I64 `class_ids` `[N]`. Use `input_size`
+to change the detector input size, `nms` for the overlap threshold, and
+`max_detections` to cap the result at up to 4,096 objects.
+`detections::filter` applies `min_area_ratio` and `max_area_ratio` before
+selecting up to `top_k` boxes. Set `top_k=0` to keep every box between the area
+limits. Use `top_k_by="area"` or `"confidence"` to rank them for selection,
+and `sort_by` to set the output order (`"none"`, `"left_to_right"`,
+`"center_to_edge"`, or `"area"`). Coordinates, scores and class IDs stay aligned.
+Use `detections::box_masks` to rasterize the selected boxes, or
+`detections::to_regions` to crop them without running detection again.
+
+Apply mask transforms before passing full-canvas masks to
+`regions::plan_mask_crops(masks, aspect=[1, 1], padding=32)`. Each nonempty mask produces
+a dictionary containing `box` (I64 `[x, y, width, height]`), the original `mask`,
+and its zero-based `index` among nonempty masks. Set `only_masked=false` to
+use the whole canvas. Choose prompts and seeds in the script. Use `index` to
+look up per-region values or to preserve seed offsets when selecting a subset.
+
+The [ADetailer program](dgl/adetailer/adetailer-existing-image.dgl) uses `fold` to crop and
+resize pixels from the latest canvas before inpainting each region. It
+defines sampler settings in DGL and uses `tensor::gaussian_blur` on the
+full-canvas mask for feathering. It resizes the decoded patch to the box,
+blends with `tensor::lerp` and writes it back with `tensor::replace_slice`.
+Crop resizing uses the selected box as the interpolation boundary.
+
+### Mask composition
+
+Use `tensor::translate` to shift a mask, `tensor::maximum` to merge masks and
+`tensor::gaussian_blur` to soften their boundaries before blending a patch.
+Convert U8 mask weights to floating point and divide by 255 before using them
+with `tensor::lerp`. To create a binary U8 mask from RGB, call
+`tensor::grayscale` and then `tensor::threshold` with `low=0, high=255`.
+Swap those output values to invert the binary mask.
+
+Use maximum pooling for dilation and minimum pooling for erosion. Specify
+`pad_value=0` for zero-filled borders. For a four-pixel window anchored like
+ADetailer, set `padding_before=[2, 2]` and `padding_after=[1, 1]` to preserve
+the mask size. The Wan examples fill selected time indices with
+`tensor::index_fill` to build a frame mask.
+
+## Advanced component programs
+
+Browse the [example index](dgl/README.md) for complete scripts organized by model.
+
+### Image refinement
+
+This example encodes an input image, samples the tail of a noise schedule and returns both the refined image and its latent.
+
+```dgl
+# version: 0.0.0
+
+var {
+ weights = "sd15.safetensors"
+
+ input_path = "input.png"
+ output_path = "output.png"
+
+ prompt = "A small house in a forest clearing"
+
+ seed = 42
+}
+
+fn refine(image, vae, unet, text, prompt, seed) {
+ encode_rng = make_rng("cuda", seed = seed)
+ resized = tensor::interpolate(image, axes = [1, 2], size = [512, 512])
+ base = vae_to_unet_latents(vae, vae_encode(vae, resized, rng = encode_rng))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, ""), prediction = "eps")
+ guided = cfg(pos, neg, scale = 7)
+ full_sigmas = make_sigmas(guided, latent = base, scheduler = "karras", steps = 20)
+ sigmas = full_sigmas[-13:]
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, rng = rng)
+ return { image = vae_decode(vae, unet_to_vae_latents(vae, latent)), latent = latent }
+}
+
+fn main() {
+ vae = load_vae(weights)
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ refined = refine(image::load(input_path), vae, unet, text, prompt, seed)
+ image::save(output_path, refined.image)
+ return { image = refined.image, latent = refined.latent }
+}
+```
+
+### Conditioning, inpainting and adapters
+
+- [SD 1.5 ControlNet](dgl/sd/sd1-5-controlnet.dgl) and
+ [Z-Image ControlNet](dgl/z_image/z-image-turbo-controlnet.dgl) load control
+ weights with `controlnet::load`. Use `controlnet::apply(condition, model, hint)`
+ to attach a control to a condition before passing it to `predictor`. Apply
+ it again on the returned condition to add another control. Set each binding's
+ `strength` and use `start_sigma` and `end_sigma` for its inclusive range
+ `end_sigma <= sigma <= start_sigma`. Both bounds are optional. Use
+ `t_to_sigma` to derive bounds from noise time, for example
+ `end_sigma=t_to_sigma(pos, t=0.2)`. The same bounds apply after slicing
+ the sigma tensor. Specify `type` on each binding of a classic Union
+ checkpoint. Supply RGB float pixels for SD 1.5 or a packed 33-channel
+ diffusion latent for Z-Image. Z-Image runs control blocks on the DiT compute
+ backend as part of its graph.
+- [Inpainting](dgl/sd/sd1-5-inpainting.dgl) separately encodes the original and
+ masked pixels. Its `latent_mask` function resizes and broadcasts the mask.
+ `tensor::max_pool2d` expands it for per-step blending. Pass the clean
+ `base` and the `denoise_mask` to `sample`.
+ Set `mask_input=true` for a program that anchors known pixels or keyframes at the
+ input of each prediction as well.
+- [IP-Adapter](dgl/ip_adapter/ip-adapter-sd1-5.dgl) encodes the image with
+ `encode_vision` and projects both the conditional and unconditional vision
+ embeddings with `ip_adapter::project`. Attach each set of tokens with
+ `ip_adapter::apply(condition, adapter, tokens)` before constructing its
+ predictor. Each call adds an adapter with its own `strength` and optional
+ `start_sigma` and `end_sigma` bounds.
+ [Classic and Plus use different vision inputs](models/ip_adapter.md#dgl).
+- [PuLID](dgl/pulid/pulid-flux1-schnell.dgl) loads the attention weights with
+ `pulid::load` and the precomputed identity with `pulid::load_embedding`.
+ `pulid::apply(condition, pulid, embedding)` binds an identity to one
+ predictor branch. Set its `strength` and optional `start_sigma` and
+ `end_sigma` bounds.
+- [PhotoMaker](dgl/photo_maker/photomaker-v1-sdxl.dgl) precomputes reusable identity
+ tokens with `photomaker::encode_images`. `photomaker::encode_prompt` returns the
+ plain condition, the prompt with identity slots, and a slot mask.
+ `photomaker::apply` fuses the identity into that condition.
+ `switch_predictor` selects when to use it.
+ [PhotoMaker v2](dgl/photo_maker/photomaker-v2-sdxl.dgl) also passes the
+ float32 face embeddings from `tensor::load` to `photomaker::encode_images`.
+- [Inline prompt images](guides/prompt_images.md#dgl) use
+ `<image:path>` in `encode_prompt`. DGL reads the file during execution.
+ `image_bindings` can also supply tensors produced earlier in the graph.
+
+- [ADetailer](dgl/adetailer/adetailer-existing-image.dgl) filters detections, rasterizes
+ boxes and uses tensor pooling and blur to transform the masks. Use
+ `tensor::maximum` in a `fold` to merge selected masks before planning regions.
+ A separate `fold` composites each result before cropping the next one.
+
+### Model-specific sampling
+
+- [SeFi](dgl/sefi_image/sefi-image-5b-rl.dgl) uses `sefi::make_sigmas` for the texture and semantic
+ schedules as F32 tensors and passes both explicitly to `sefi::sample`.
+ The SeFi solver uses Euler and accepts repeated sigma values in either
+ schedule. It shares mask, finish, cache and callback handling with `sample`.
+ Slice out texture channels `[16, 144)` before converting them for the VAE.
+- [AnimateDiff](dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl) loads a motion module with
+ `animatediff::load` and attaches it to a compatible SD 1.5 UNet with
+ `animatediff::apply`. The frame count comes from the batch axis of the UNet
+ latent. [Wan I2V](dgl/wan/wan2-1-i2v-14b.dgl),
+ [VACE](dgl/wan/wan2-1-vace-1-3b-video2video.dgl) and [S2V](dgl/wan/wan2-2-s2v-14b.dgl)
+ show the vision, control and driving-audio ports respectively.
+- [LTX](dgl/ltx2/ltx-2-3-dev-txt2video.dgl) and [MiniMax-H3](dgl/minimax_h3/minimax-h3-text-to-av.dgl)
+ build video and audio latents separately. Their `pack_av` and `unpack_audio`
+ functions pack unbatched latents for joint prediction and recover the audio.
+ Zero-padding the audio tail fills whole video channels. For a packed AV mask, broadcast
+ the video mask to the video latent shape and fill the packed audio channels
+ with ones. The video VAE and `audio_vae_decode` run separately before saving. Pass `audio=audio` and
+ `trim_audio=true` to `video::save` to match the video duration, as in
+ the MiniMax-H3 and Wan S2V examples. A video model can
+ receive the audio latent length and frame rate explicitly on its predictor.
+ The MiniMax-H3 script scales the clean audio by `video_shift / audio_shift`
+ when packing it for sampling. It divides by that ratio when unpacking the
+ final latent. For masked sampling, pack the clean `base` with the same
+ scale. The predictor receives both shifts as explicit inputs.
+ LTX text encoding also runs the video and audio embedding connectors and
+ static caption projections. Pass a separate connector file to
+ `load_text_encoder(connectors_path=...)`; use `projection_path` for static
+ conditioning weights stored in the diffusion checkpoint. `encode_prompt`
+ returns the prepared video condition in `crossattn` and the audio condition
+ in `extra_crossattn[0]`. Both are cached together and passed directly to the
+ DiT. `load_dit` does not accept `connectors_path`.
+
+#### Reference inputs and timing
+
+MiniMax-H3 reference blocks use a native `list<dict>` directly, both in
+`predictor(..., reference_blocks=...)` and in the condition's `reference_blocks`
+field. The former `reference_blocks(...)` packing helper has been removed.
+Each block has a `kind` of `image`, `video`, `audio` or `video_audio`.
+`image` and `video` require a `video` index into `ref_latents`; `audio` requires
+an `audio` index into `ref_audios`; `video_audio` requires both. Indices are
+zero-based nonnegative integers, and list order determines reference order.
+If `ref_latents` is empty, video indices refer to the condition's `ref_images`.
+The explicit argument overrides the condition field, including an empty list.
+Malformed blocks or out-of-range indices fail when constructing the predictor,
+with the block number and field in the diagnostic.
+
+```dgl
+p = predictor(dit, condition,
+ ref_latents = [portrait, clip], ref_audios = [soundtrack],
+ reference_blocks = [
+ {kind = "image", video = 0},
+ {kind = "video_audio", video = 1, audio = 0}
+ ])
+```
+
+`ltx::video_positions` builds F32 `[2,3,N,1]` token bounds in seconds and pixels.
+Pass the target latent before concatenating keyframes. `keyframe_frame_index`
+is the keyframe's zero-based position in the output video;
+`keyframe_pixel_frames` is its span in original frames;
+`keyframe_latent_frames` is the number of appended latent frames. The former
+`keyframe_frame` and `keyframe_frames` parameter names are removed.
+
+`wan::s2v_audio_window(features, latent_frames=...)` aligns audio features to
+Wan S2V's timeline. It returns `[feature_dim, 4 * latent_frames, layers]`,
+resampling 50 Hz features to 30 Hz before selecting positions at 16 fps.
+Samples beyond the audio are zero-filled. The old `frames` argument is removed;
+use `latent_frames = tensor::shape(base)[2]` to follow the generated video.
+
+### Upscaling and hires passes
+
+- Image hires resizes the first-pass latent and chooses a second-pass sigma
+ tail with tensor indexing. Keep the last `floor(steps * strength)` steps
+ and the terminal sigma.
+ [Video Hires](dgl/ltx2/ltx-2-3-spatial-upscale.dgl) loads a separate
+ `load_latent_upscaler` resource, runs `upscale_latent`, then repacks
+ the audio and samples a second time. LTX upscalers take and return VAE
+ latents, with the scale inferred from the weights. MiniMax H3 upscalers
+ take and return diffusion latents and accept target `width` and `height`
+ in latent units. Both adapters accept `[W,H,T,C]` or `[W,H,T,C,1]`,
+ with channels matching the checkpoint. A single frame uses `T=1`; ordinary
+ image latents `[W,H,C,B]` are a different layout and are not interchangeable.
+ The latent must also come from a compatible model/VAE. SeedVR2 uses the
+ separate pixel-resize, VAE encode, DiT sampling and VAE decode workflow;
+ it is not a `load_latent_upscaler` backend. The former `upscale_video_latent`
+ name is removed without an alias.
+- [RelayVSR](dgl/relay_vsr/relayvsr-x2-video-upscale.dgl) selects keyframes with tensor
+ functions and assembles a predictor condition with the checkpoint's
+ `crossattn` embedding and the low-resolution `lq_frames` tensor. A single
+ Euler step produces exportable keyframe latents. `relayvsr::decode` reads
+ those latents, the I64 keyframe indices, and the original U8 frames to
+ reconstruct the video. Use a matching 2x or 4x DiT and Flash VAE.
+- [SeedVR2](dgl/seedvr2/seedvr2-3b-video-upscale.dgl) loads its DiT and VAE with
+ `load_dit` and `load_vae`. The example resizes and pads each pixel window
+ with tensor functions, encodes it with `vae_encode`, and explicitly
+ assembles the predictor's `crossattn` embedding and `concat` condition.
+ A shared RNG feeds the noise and the one-step Euler sampler. The script
+ decodes and crops each result before joining the windows. `video::join_windows` discards the
+ specified overlap from the returned frame tensors. Frame rate and audio are
+ passed separately to `video::save` after joining.
+
+### Function names and namespaces
+
+Generic model components and inference stay at the top level. Family- or
+extension-specific operations use their own namespace, such as
+`controlnet::load` / `controlnet::apply`, `ip_adapter::project` and
+`photomaker::encode_images`. Extension `apply` functions retain their own
+execution and resource-lifetime rules; their names do not imply deferred execution.
+
+Generic `load_lora`, `apply_lora`, `encode_prompt`, `sample`
+and component loaders remain top-level.
+
+The following global names have been replaced without aliases. Parameter names,
+argument order, defaults and result fields are unchanged:
+
+| Previous name | Current name |
+|---|---|
+| `load_controlnet` | `controlnet::load` |
+| `apply_controlnet` | `controlnet::apply` |
+| `load_ip_adapter` | `ip_adapter::load` |
+| `ip_adapter_tokens` | `ip_adapter::project` |
+| `apply_ip_adapter` | `ip_adapter::apply` |
+| `load_photomaker` | `photomaker::load` |
+| `photomaker_tokens` | `photomaker::encode_images` |
+| `encode_photomaker_prompt` | `photomaker::encode_prompt` |
+| `apply_photomaker` | `photomaker::apply` |
+| `load_pulid` | `pulid::load` |
+| `load_pulid_embedding` | `pulid::load_embedding` |
+| `apply_pulid` | `pulid::apply` |
+| `load_animatediff` | `animatediff::load` |
+| `apply_animatediff` | `animatediff::apply` |
+| `ltx_video_positions` | `ltx::video_positions` |
+| `s2v_audio_window` | `wan::s2v_audio_window` |
+| `sefi_sigmas` | `sefi::make_sigmas` |
+
+## C API
+
+Include `stable-diffusion.h` for the DGL and tensor APIs.
+
+Structured results expose `bytes` as `SD_DGL_VALUE_BYTES`.
+`sd_dgl_value_get_bytes(value, &data, &size)` returns a borrowed `const void*`
+and length, with no terminator or text conversion. The pointer remains valid
+until `sd_dgl_result_free`. Existing enum values and getters are unchanged.
+
+`sd_dgl_check` checks source without creating a context or loading models.
+Pass `false` for a self-contained script, as the CLI's `--check` does, or
+`true` to allow named tensor parameters in `fn main(...)`.
+
+```c
+const char* source = "fn main() { return load_unet(\"model.safetensors\") }";
+sd_status_t status = sd_dgl_check(source, false);
+if (status != SD_STATUS_OK) {
+ fprintf(stderr, "%s\n", sd_get_last_error());
+}
+```
+
+Execution requires an explicit context. Create one with `sd_dgl_ctx_create`
+to share loaded components across scripts, or `sd_dgl_ctx_create_once` to
+release components after their last use in each run. Configure it while idle,
+then pass source directly to `sd_dgl_run`. Every run compiles the source once;
+there is no public compiled-program handle or graph cache. A normal execution
+does not need a preceding `sd_dgl_check` call.
+
+This example configures a context, runs a script without host inputs, and reads
+its structured result:
+
+```c
+const char* source = "fn main() { return { pixels = tensor::zeros([2, 3]) } }";
+sd_dgl_ctx_t* ctx = sd_dgl_ctx_create();
+sd_dgl_result_t* result = NULL;
+if (ctx && sd_dgl_ctx_set(ctx, "max_ram", "24")) {
+ sd_status_t status = sd_dgl_run(ctx, source, NULL, 0, &result);
+ if (status == SD_STATUS_OK) {
+ const sd_dgl_value_t* root = sd_dgl_result_root(result);
+ const sd_tensor_t* pixels = sd_dgl_value_tensor(sd_dgl_value_field(root, "pixels"));
+ sd_tensor_save(pixels, "pixels.npy");
+ } else {
+ fprintf(stderr, "%s\n", sd_get_last_error());
+ }
+}
+sd_dgl_result_free(result);
+sd_dgl_ctx_free(ctx);
+```
+
+Keep the context alive until all calls using it return. Free it with
+`sd_dgl_ctx_free` when its cached components are no longer needed.
+
+### Host tensor inputs
+
+Standalone scripts can read tensors with `tensor::load(path)` or embed them
+with `tensor::literal`. A C host can pass source with named tensor parameters
+directly to `sd_dgl_run`. Supply exactly one binding per parameter. Compilation
+always permits these parameters; a missing binding reports a missing input.
+Unknown or duplicate names and null tensors are also rejected.
+Bindings share the existing immutable tensor handles. The same source can be
+run with different bindings on subsequent calls; each call compiles it anew.
+Use `sd_dgl_check(source, true)` when only validation is needed.
+
+```c
+/* ctx is an existing DGL context and pixels is an immutable tensor. */
+const char* source = "fn main(pixels) { return { image = pixels, label = \"input\" } }";
+sd_dgl_result_t* result = NULL;
+const sd_dgl_tensor_input_t input = {"pixels", pixels};
+sd_status_t status = sd_dgl_run(ctx, source, &input, 1, &result);
+if (status == SD_STATUS_OK) {
+ const sd_dgl_value_t* root = sd_dgl_result_root(result);
+ const sd_tensor_t* image = sd_dgl_value_tensor(sd_dgl_value_field(root, "image"));
+ sd_tensor_save(image, "pixels-copy.npy");
+}
+sd_dgl_result_free(result);
+```
+
+### Context execution settings
+
+Set these options with `sd_dgl_ctx_set` while the context is idle or use the
+corresponding `sd-cli` and `sd-server` flags. All models in the context use the
+same settings.
+
+| Context key | CLI and server flag | Default |
+| --- | --- | --- |
+| `n_threads` | `--threads N` or `-t N` | Physical core count |
+| `auto_fit` | `--auto-fit on\|off` | On |
+| `batched_cfg` | `--batched-cfg on\|off` | On |
+| `embed_metadata` | `--disable-image-metadata` to turn off | On |
+| `enable_mmap` | `--mmap` | Off |
+| `eager_load` | `--eager-load` | Off |
+| `disable_prefetch` | `--disable-prefetch` | Off |
+| `disable_segmented_compute` | `--disable-segmented-compute` | Off |
+
+The context accepts a thread count from 1 to 4096 and boolean strings such as
+`"on"` and `"off"`. Changing threading, placement or weight-loading options
+discards cached models so they can be loaded with the new settings on next use.
+Models using graph-cut layer splitting prepare weights lazily even when eager
+loading is enabled.
+
+`image::save` (PNG) and `video::save` (AVI) embed the exact executed DGL source
+by default. Set `sd_dgl_ctx_set(ctx, "embed_metadata", "off")` to disable this
+for all saves in that context. The CLI forwards `--disable-image-metadata` to
+this setting in DGL mode. Changing it leaves loaded models cached. Metadata
+records the program text; host tensor contents and input files are not embedded.
+
+`batched_cfg` combines compatible positive and negative `cfg` branches into one
+UNet forward with batch size two. The branches must use the same model, matching
+conditioning shapes and matching circular settings. Predictors with transforms,
+references, ControlNet, IP-Adapter, PuLID, AnimateDiff or active sampling caches
+retain separate forwards, as do branches using different models or incompatible
+shapes. Branch selection through `switch_predictor` is resolved before checking
+compatibility. CFG with scale one only computes the positive branch unless the
+sampler also needs the unconditional prediction. Batching uses more compute
+memory and can produce small floating-point differences. Set `batched_cfg` to
+`"off"` to force separate forwards. Changing it affects subsequent runs without
+invalidating loaded models.
+
+Register RPC devices in the host with `sd_add_rpc_servers` or use
+`--rpc-servers HOST:PORT` at CLI or server startup. A loader can then select a
+registered RPC device with its `backend` option.
+
+Set `conditioning_cache_size` with `sd_dgl_ctx_set` or use
+`--conditioning-cache-size` with `sd-cli` or `sd-server`. The default is 4 entries
+per text encoder instance, including each LoRA variant. Lowering the capacity
+evicts the least recently used entries immediately. A capacity of 0 clears the
+caches and disables caching. Loaded encoders and their weights are reused when
+the capacity changes.
+
+### File access and memory budgets
+
+Install a file access callback with `sd_dgl_ctx_set_file_access_callback` before
+running scripts in a context. The callback receives a UTF-8 path and an access
+type for input reads, model reads or writes. Returning false ends the run with a
+file access error. Keep its user data valid throughout the run. Model reads
+include checkpoints, referenced shards and external tokenizers. Without a
+callback, file access follows the process's permissions.
+
+#### Memory budgets
+
+```c
+sd_dgl_ctx_t* ctx = sd_dgl_ctx_create();
+sd_dgl_result_t* result_a = NULL;
+sd_dgl_result_t* result_b = NULL;
+if (ctx && sd_dgl_ctx_set(ctx, "max_ram", "24") &&
+ sd_dgl_ctx_set(ctx, "max_vram", "cuda0=12")) {
+ /* source_a and source_b can reuse the same encoder and VAE. */
+ sd_status_t status = sd_dgl_run(ctx, source_a, NULL, 0, &result_a);
+ if (status == SD_STATUS_OK) {
+ status = sd_dgl_run(ctx, source_b, NULL, 0, &result_b);
+ }
+ if (status != SD_STATUS_OK) {
+ fprintf(stderr, "%s\n", sd_get_last_error());
+ }
+}
+sd_dgl_result_free(result_a);
+sd_dgl_result_free(result_b);
+sd_dgl_ctx_free(ctx);
+```
+
+The context's `max_vram` applies to all components on each device. Set the
+context limits from the CLI with `--max-vram` and `--max-ram`. Idle GPU weights may
+stay resident between runs. Under pressure the context reclaims idle buffers
+and reloads weights from their configured parameter backend when needed.
+With `params_backend="disk"`, weights are read on demand and their buffers
+are released after use.
+
+`max_ram` caps allocated host weight buffers across the context, including
+pinned host buffers used by CUDA. Set it in GiB with `sd_dgl_ctx_set`.
+A value of `0` disables the explicit cap and keeps the system memory pressure
+check, which subtracts 1 GiB from the available RAM reported by the OS.
+On Windows, this also accounts for the remaining commit capacity, including
+CUDA weight staging, prefetch, and compute buffer allocations under WDDM. Device
+allocations reserve commit capacity without charging their size to physical RAM. Graph
+metadata allocations request space from the same memory manager and can reclaim
+idle weights before creating a scheduler. Temporary graph memory does not count
+against the host weight limit.
+For CPU execution, automatic placement checks weight storage against this cap
+and separately checks weights plus workspace against available system memory.
+Lowering the cap trims idle weights immediately. Before a new allocation,
+the context can evict old CPU weights together with their idle GPU copies, or
+unload an idle component. Reclaimed weights reload from their source files on
+the next use. A request whose active weights cannot fit reports the required
+and available capacity. Automatic placement also uses the cap when choosing
+where to keep new weights.
+
+The cap covers weight buffers. Allow separate RAM for graph workspaces,
+intermediate tensors and other process allocations. The OS manages pages in
+read-only weight mappings. With a cap enabled, weights that need writable
+mappings for in-place LoRA merges use allocated buffers.
+
+#### Automatic weight placement
+
+Set `backend` and `params_backend` on loaders in the script. The context's
+`auto_fit` setting chooses placement when `params_backend` is empty; it is
+enabled by default and cannot be overridden by individual loaders.
+Each run prepares one weight-placement plan for the graph, including models
+already cached in the context. Compilation tracks actual model use separately
+from metadata queries and model captures. For example, `latent_shape` does not
+need VAE weights, while a VAE preview observer keeps the decoder in the sampling
+working set when VAE/TAE preview callbacks are enabled. Unrelated scalar
+operations do not change the model-use phases.
+
+If a loader omits `backend`, the run compares available GPU budgets before
+constructing runners and uses the selected device consistently for default
+loaders. Explicit backend choices and split layouts remain unchanged.
+
+Before computation, the runtime constructs missing runner metadata without
+loading all weights, then combines registered parameter sizes, phase working
+sets, reclaimable cached buffers, device capacity and the context budgets.
+Automatic placement prioritizes diffusion models (`load_dit` / `load_unet`)
+over other components. It reserves their device storage first when the stage
+working set fits with other components offloaded. Larger models are considered
+first within each priority group.
+Explicit parameter backends and split layouts remain constraints. Automatic
+parameter storage can move between the compute device, CPU and disk on later
+runs without reconstructing a compatible model. Under pressure, weights with
+no remaining use or the farthest next use are reclaimed first. Idle weights
+remain cached when there is no pressure.
+
+GGML sizes compute workspaces from the actual graphs built during execution.
+The first plan reserves conservative workspace headroom; actual workspace
+allocations and changes in external memory use are still checked at runtime.
+Models first encountered inside an iteration join the remaining plan when
+their loader executes. LoRA buffers and merged variants also retain runtime
+allocation checks.
+
+### Progress, preview and cancellation
+
+Register callbacks on an idle context with `sd_dgl_ctx_set_progress_callback`
+and `sd_dgl_ctx_set_preview_callback`. They run synchronously on the executing
+thread and receive events for that context's current run. A null callback
+clears the registration. The sampling pass counter starts at one on each
+run. Progress can report model loading in tensors, sampling in steps, video
+upscaling in frames, or a stage with no known total.
+Without a progress callback, the DGL execution layer displays sampling and
+weight-loading progress bars. With a callback, the host controls their display.
+
+The default observer handles previews. Preview mode `proj` uses latent projection. Modes `tae` and `vae`
+use the decoder supplied to `preview(decoder=...)`, falling back to projection. A positive interval
+previews each matching completed logical step. Intermediate model evaluations,
+such as Heun's first evaluation, do not produce duplicate previews or progress
+events. Zero previews the final step of the first sampling pass.
+A negative interval selects the step at its absolute value. The callback
+receives the pass number, total steps and borrowed frame tensors. Use
+`sd_tensor_retain` if a frame must outlive the callback.
+
+`sd_dgl_ctx_cancel` can be called from a callback or another thread. An
+idle context accepts the call without queuing a cancellation. The current
+run stops at an execution boundary, such as between sampling steps or
+after an in-flight model load completes. Cancellation requested during compilation
+is checked after compilation, before binding inputs and executing. The API returns
+`SD_STATUS_CANCELLED` with a null result. Files already written by the
+script remain on disk. Keep the context alive
+until calls using it have returned. Concurrent or reentrant runs on the
+same context return `SD_STATUS_BUSY`.
+
+### Results and diagnostics
+
+`sd_dgl_result_root` exposes the returned value with typed scalar access,
+tensor handles and nested dict or list children. Audio and video reads return
+ordinary dicts and tensors, using the same result traversal as other values.
+Region collections are ordinary dicts with integer `width` and `height` and an
+I64 `boxes` tensor shaped `[4,N]`, with `[x, y, width, height]` per column.
+A root region dict flattens to `boxes`, `width` and `height`; a field named
+`regions` flattens to `regions.boxes`, `regions.width` and `regions.height`.
+Export root coordinates with `--tensor boxes boxes.npy`, replacing the old
+`--tensor result boxes.npy`. Cast old I32 files to I64 before reuse. Runtime objects such as models and RNGs appear as opaque values.
+
+A function without a return value has result type `void`, no output tensors,
+and an `SD_DGL_VALUE_VOID` root through the structured C API.
+
+A single returned tensor is named `result`. Named returns retain their names;
+lists add numeric path components. Conditioning expands into fields such as
+`positive.crossattn` and `positive.vector`. Use `sd_dgl_result_count/name/tensor`
+to enumerate the actual outputs. `sd_tensor_save` writes `.npy` using the reversed
+shape so NumPy sees normal row-major dimensions; RGB becomes `(height,width,3)`.
+
+In the flat tensor view, strings are UTF-8 U8 arrays without a terminating zero
+byte.
+
+Values and tensors are borrowed from the result. Use `sd_tensor_retain`
+to keep a tensor past the result's lifetime. Results remain valid after
+freeing the context. All destructors accept `NULL`.
+
+Failed runs leave `*result_out` as `NULL`. Both check and run return
+`sd_status_t`; compare it explicitly with `SD_STATUS_OK`. Use
+`sd_get_last_error()` for the thread-local diagnostic for DGL and other C
+API calls. `sd_dgl_error()` reads the same diagnostic. Syntax and
+execution diagnostics include source locations when available.
diff --git a/docs/dgl/README.md b/docs/dgl/README.md
new file mode 100644
index 0000000..eaa998d
--- /dev/null
+++ b/docs/dgl/README.md
@@ -0,0 +1,116 @@
+# DGL examples
+
+Choose a runnable script for [image generation and editing](#image-generation-and-editing),
+[video and audio generation](#video-and-audio-generation), or
+[adapters, refinement and upscaling](#adapters-refinement-and-upscaling).
+Each model name links to its guide for weights and setup.
+
+## Run an example
+
+Open the script's `var` block and set the model paths, input files, prompt and
+output path. Run the following commands from the CMake build directory. Relative
+paths in the script resolve from that directory.
+
+**Windows (PowerShell, Visual Studio Release)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sd\sd1-5-txt2img.dgl --check
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sd\sd1-5-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/sd/sd1-5-txt2img.dgl --check
+./bin/sd-cli -M dgl --script ../docs/dgl/sd/sd1-5-txt2img.dgl
+```
+
+The first command checks the script without loading weights. The second runs it
+and writes to the output path in the script. Set context options such as
+`--max-vram` and `--auto-fit` on the command line. In the server's DGL editor,
+choose model paths under a configured model directory and save results under
+`output/`. See the [server guide](../../examples/server/README.md#dgl-editor).
+
+For a large video input, set `max_frames` on `video::load` to limit how many
+frames it decodes. See [video input formats and limits](../dgl.md#reading-video)
+and [NumPy tensor files](../dgl.md#tensor-files).
+
+## Model examples
+
+### Image generation and editing
+
+| Model guide | Scripts |
+| --- | --- |
+| [Anima](../models/anima.md) | [Anima Preview text to image](anima/anima-preview-txt2img.dgl) |
+| [Boogu Image](../models/boogu_image.md) | [Text to image](boogu_image/boogu-image-base-txt2img.dgl) · [Image editing](boogu_image/boogu-image-edit.dgl) |
+| [Chroma](../models/chroma.md) | [Unlocked v40 text to image](chroma/chroma-unlocked-v40-txt2img.dgl) |
+| [Chroma1-Radiance](../models/chroma_radiance.md) | [Text to image](chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl) |
+| [Distilled Stable Diffusion models](../models/distilled_sd.md) | [SSD1B](distilled_sd/ssd1b-txt2img.dgl) · [SDXS-512-DreamShaper](distilled_sd/sdxs-512-dreamshaper-txt2img.dgl) |
+| [ERNIE-Image](../models/ernie_image.md) | [ERNIE-Image Turbo](ernie_image/ernie-image-turbo-txt2img.dgl) · [ERNIE-Image](ernie_image/ernie-image-txt2img.dgl) |
+| [Fibo 2.0](../models/fibo2.md) | [Dev: text to image](fibo2/fibo2-dev-txt2img.dgl) · [Dev: image editing](fibo2/fibo2-dev-edit.dgl) · [Edit Turbo](fibo2/fibo2-edit-turbo.dgl) |
+| [FLUX.1](../models/flux.md) | [FLUX.1 dev](flux/flux1-dev-txt2img.dgl) · [FLUX.1 schnell](flux/flux1-schnell-txt2img.dgl) · [FLUX.1 dev with LoRA](flux/flux1-dev-lora.dgl) |
+| [FLUX.2 dev](../models/flux2.md) | [Image editing](flux2/flux2-dev-edit.dgl) |
+| [FLUX.2 Klein 4B](../models/flux2.md) | [Text to image](flux2/flux2-klein-4b-txt2img.dgl) · [Image editing](flux2/flux2-klein-4b-edit.dgl) · [Base text to image](flux2/flux2-klein-base-4b-txt2img.dgl) |
+| [FLUX.2 Klein 9B](../models/flux2.md) | [Text to image](flux2/flux2-klein-9b-txt2img.dgl) · [Image editing](flux2/flux2-klein-9b-edit.dgl) · [Base text to image](flux2/flux2-klein-base-9b-txt2img.dgl) |
+| [FLUX.3 Image](../models/flux3.md) | [Klein: text to image](flux3/flux3-image-klein-txt2img.dgl) · [Klein: image editing](flux3/flux3-image-klein-edit.dgl) · [Klein Base: text to image](flux3/flux3-image-klein-base-txt2img.dgl) |
+| [HiDream-O1-Image](../models/hidream_o1_image.md) | [Text to image](hidream_o1_image/hidream-o1-image-dev-txt2img.dgl) |
+| [Ideogram4](../models/ideogram4.md) | [Text to image](ideogram4/ideogram4-txt2img.dgl) |
+| [Iris-3B](../models/iris.md) | [Text to image](iris/iris-3b-txt2img.dgl) |
+| [FLUX.1 Kontext](../models/kontext.md) | [Edit with FLUX.1 Kontext dev](kontext/flux1-kontext-dev-edit.dgl) |
+| [Krea2](../models/krea2.md) | [Raw text to image](krea2/krea2-raw-txt2img.dgl) |
+| [Lens](../models/lens.md) | [Lens](lens/lens-txt2img.dgl) · [Lens Turbo](lens/lens-turbo-txt2img.dgl) |
+| [LLaDA-Image](../models/llada_image.md) | [Text to image](llada_image/llada-image-turbo-txt2img.dgl) · [Image editing](llada_image/llada-image-turbo-edit.dgl) |
+| [LongCat Image](../models/longcat_image.md) | [Text to image](longcat_image/longcat-image-txt2img.dgl) |
+| [Mage-Flow](../models/mage_flow.md) | [Text to image](mage_flow/mage-flow-turbo-txt2img.dgl) · [Image editing](mage_flow/mage-flow-edit.dgl) |
+| [Ming-Image](../models/ming_image.md) | [0.1 Design text to image](ming_image/ming-image-0-1-design-txt2img.dgl) |
+| [MiniMax-H3-Image](../models/minimax_h3_image.md) | [Text to image](minimax_h3_image/minimax-h3-image-txt2img.dgl) · [Image editing](minimax_h3_image/minimax-h3-image-edit.dgl) |
+| [MiniT2I](../models/minit2i.md) | [CUDA](minit2i/minit2i-cuda-txt2img.dgl) · [Metal](minit2i/minit2i-metal-txt2img.dgl) (`--threads 8`) |
+| [MOSS-Image](../models/moss_image.md) | [Base: text to image](moss_image/moss-image-base-txt2img.dgl) · [SFT: text to image](moss_image/moss-image-sft-txt2img.dgl) · [Inline images](moss_image/moss-image-sft-inline-images.dgl) |
+| [Ovis-Image](../models/ovis_image.md) | [Text to image](ovis_image/ovis-image-txt2img.dgl) |
+| [P-H3-Image](../models/p_h3_image.md) | [Text to image](p_h3_image/p-h3-image-txt2img.dgl) · [Image editing](p_h3_image/p-h3-image-edit.dgl) |
+| [PiD](../models/pid.md) | [512 to 2048 pixels](pid/pid-flux1-512-to-2048-txt2img.dgl) |
+| [PixArt](../models/pixart.md) | [Sigma XL2 at 1024 pixels](pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl) |
+| [Qwen Image](../models/qwen_image.md) | [Text to image](qwen_image/qwen-image-txt2img.dgl) |
+| [Qwen Image 2.1](../models/qwen_image_2.1.md) | [Text to image](qwen_image_2.1/qwen-image-2-1-txt2img.dgl) · [Image editing](qwen_image_2.1/qwen-image-2-1-edit.dgl) |
+| [Qwen Image Edit](../models/qwen_image_edit.md) | [Qwen Image Edit](qwen_image_edit/qwen-image-edit.dgl) · [Qwen Image Edit 2509](qwen_image_edit/qwen-image-edit-2509.dgl) · [Qwen Image Edit 2511](qwen_image_edit/qwen-image-edit-2511.dgl) |
+| [Stable Diffusion – generation](../models/sd.md) | [SD 1.4](sd/sd1-4-txt2img.dgl) · [SD 1.5](sd/sd1-5-txt2img.dgl) · [SDXL](sd/sdxl-txt2img.dgl) · [NitroFusion](sd/nitrosd-realism-nitrofusion.dgl) |
+| [Stable Diffusion – image conditioning](../models/sd.md) | [SD 1.4 image to image](sd/sd1-4-img2img.dgl) · [SD 1.5 ControlNet](sd/sd1-5-controlnet.dgl) · [SD 1.5 inpainting](sd/sd1-5-inpainting.dgl) |
+| [Stable Diffusion 3](../models/sd3.md) | [SD 3.5 Large text to image](sd3/sd3-5-large-txt2img.dgl) |
+| [SeFi-Image](../models/sefi_image.md) | [1B Turbo](sefi_image/sefi-1b-turbo-txt2img.dgl) · [1B Base](sefi_image/sefi-1b-base-txt2img.dgl) · [5B Turbo](sefi_image/sefi-5b-turbo-txt2img.dgl) (`--max-vram 8`) · [5B RL](sefi_image/sefi-image-5b-rl.dgl) |
+| [SenseNova U1.5](../models/sensenova_u1.md) | [Text to image on CUDA](sensenova_u1/sensenova-u1-cuda-txt2img.dgl) |
+| [Z-Image](../models/z_image.md) | [Turbo: text to image](z_image/z-image-turbo-txt2img.dgl) · [Base: text to image](z_image/z-image-base-txt2img.dgl) · [Turbo with ControlNet](z_image/z-image-turbo-controlnet.dgl) |
+| [Z-Image L2P](../models/z_image_l2p.md) | [Text to image](z_image_l2p/z-image-l2p-txt2img.dgl) |
+
+### Video and audio generation
+
+| Model guide | Scripts |
+| --- | --- |
+| [AnimateDiff – text to video](../models/animatediff.md) | [Text to video with CPU offload](animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl) · [Text to video, 16 frames](animatediff/animatediff-realisticvision-v6-txt2video-16-frames.dgl) · [Text to video with a VRAM budget](animatediff/animatediff-realisticvision-v6-txt2video-vram-budget.dgl) (`--max-vram 2.0`) · [Domain Adapter LoRA](animatediff/animatediff-realisticvision-v6-domain-lora.dgl) |
+| [AnimateDiff – image to video](../models/animatediff.md) | [Generate the initial image](animatediff/realisticvision-v6-txt2img.dgl) · [Image to video](animatediff/animatediff-realisticvision-v6-img2video.dgl) |
+| [HunyuanVideo 1.5](../models/hunyuan_video.md) | [720p text to video](hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl) |
+| [LingBot-Video](../models/lingbot_video.md) | [Dens 1.3B text to video](lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl) |
+| [LTX-2.3](../models/ltx2.md) | [2.3 text to video](ltx2/ltx-2-3-dev-txt2video.dgl) · [2.3 image to video](ltx2/ltx-2-3-dev-img2video.dgl) · [2.3 first/last frames](ltx2/ltx-2-3-dev-first-last-frame.dgl) · [2.3 spatial latent upscale](ltx2/ltx-2-3-spatial-upscale.dgl) |
+| [LTX-2.5](../models/ltx2.md) | [2.5 text to video](ltx2/ltx-2-5-dev-txt2video.dgl) · [2.5 image to video](ltx2/ltx-2-5-dev-img2video.dgl) · [2.5 spatial latent upscale](ltx2/ltx-2-5-spatial-upscale.dgl) |
+| [MiniMax-H3](../models/minimax_h3.md) | [Text to audio/video](minimax_h3/minimax-h3-text-to-av.dgl) · [First/last-frame conditioning](minimax_h3/minimax-h3-first-last-frame.dgl) · [Reference to audio/video](minimax_h3/minimax-h3-reference-to-av.dgl) · [Interpolated latent hires](minimax_h3/minimax-h3-latent-hires.dgl) · [Learned latent hires](minimax_h3/minimax-h3-learned-hires.dgl) |
+| [Wan 2.1](../models/wan.md) | [Text to video, 1.3B](wan/wan2-1-t2v-1-3b.dgl) · [Text to video, 14B](wan/wan2-1-t2v-14b.dgl) · [Image to video, 14B](wan/wan2-1-i2v-14b.dgl) · [First/last frames, 14B](wan/wan2-1-first-last-frame-14b.dgl) |
+| [Wan 2.1 VACE 1.3B](../models/wan.md) | [Text to video](wan/wan2-1-vace-1-3b-txt2video.dgl) · [Reference to video](wan/wan2-1-vace-1-3b-ref2video.dgl) · [Video to video](wan/wan2-1-vace-1-3b-video2video.dgl) |
+| [Wan 2.1 VACE 14B](../models/wan.md) | [Text to video](wan/wan2-1-vace-14b-txt2video.dgl) · [Reference to video](wan/wan2-1-vace-14b-ref2video.dgl) · [Video to video](wan/wan2-1-vace-14b-video2video.dgl) |
+| [Wan 2.2 A14B](../models/wan.md) | [Text to video](wan/wan2-2-t2v-a14b.dgl) · [Image to video](wan/wan2-2-i2v-a14b.dgl) · [Single image](wan/wan2-2-t2v-a14b-image.dgl) · [Text to video with LoRA](wan/wan2-2-t2v-a14b-lora.dgl) · [First/last frames](wan/wan2-2-first-last-frame-a14b.dgl) |
+| [Wan 2.2 TI2V 5B](../models/wan.md) | [Text to video](wan/wan2-2-ti2v-5b-txt2video.dgl) · [Image to video](wan/wan2-2-ti2v-5b-img2video.dgl) |
+| [Wan 2.2 S2V](../models/wan.md) | [Speech to video, 14B](wan/wan2-2-s2v-14b.dgl) |
+| [Wan 2.2 VACE Fun A14B](../models/wan.md) | [Reference to video](wan/wan2-2-vace-fun-a14b-ref2video.dgl) |
+
+### Adapters, refinement and upscaling
+
+| Model guide | Scripts |
+| --- | --- |
+| [ADetailer](../models/adetailer.md) | [Refine an existing image](adetailer/adetailer-existing-image.dgl) · [Generate and refine](adetailer/adetailer-generated-image.dgl) |
+| [ESRGAN and MoSR](../models/esrgan.md) | [Generate and upscale](esrgan/sd1-5-txt2img-esrgan.dgl) · [Upscale an image](esrgan/realesrgan-x4plus-anime-6b-upscale.dgl) |
+| [IP-Adapter](../models/ip_adapter.md) | [SD 1.5 Classic](ip_adapter/ip-adapter-sd1-5.dgl) · [SDXL Classic](ip_adapter/ip-adapter-sdxl.dgl) · [SD 1.5 Plus](ip_adapter/ip-adapter-plus-sd1-5.dgl) · [SDXL with ControlNet](ip_adapter/ip-adapter-sdxl-controlnet.dgl) |
+| [LCM and LCM-LoRA](../models/lcm.md) | [LCM-LoRA with SD 1.5](lcm/lcm-lora-sd1-5.dgl) |
+| [LoRA](../models/lora.md) | [SD 1.5 UNet and text encoder](lora/sd1-5-lora.dgl) |
+| [PhotoMaker](../models/photo_maker.md) | [PhotoMaker v1](photo_maker/photomaker-v1-sdxl.dgl) · [PhotoMaker v2](photo_maker/photomaker-v2-sdxl.dgl) |
+| [PuLID-Flux](../models/pulid.md) | [Identity-guided generation](pulid/pulid-flux1-schnell.dgl) |
+| [RelayVSR](../models/relay_vsr.md) | [Upscale a video](relay_vsr/relayvsr-x2-video-upscale.dgl) · [Upscale an image](relay_vsr/relayvsr-x2-image-upscale.dgl) |
+| [SeedVR2](../models/seedvr2.md) | [Upscale an image](seedvr2/seedvr2-3b-image-upscale.dgl) · [Upscale a video](seedvr2/seedvr2-3b-video-upscale.dgl) |
+| [TAESD](../models/taesd.md) | [SD 1.5 decoding with TAESD](taesd/sd1-5-taesd-preview.dgl). The CLI writes the final PNG. Intermediate previews require a [host preview callback](../dgl.md#progress-preview-and-cancellation) in `tae` mode. |
diff --git a/docs/dgl/adetailer/adetailer-existing-image.dgl b/docs/dgl/adetailer/adetailer-existing-image.dgl
new file mode 100644
index 0000000..66c5a52
--- /dev/null
+++ b/docs/dgl/adetailer/adetailer-existing-image.dgl
@@ -0,0 +1,107 @@
+# version: 0.0.0
+# docs/models/adetailer.md — Repair an existing image
+
+var {
+ weights = "../models/checkpoints/model.safetensors"
+ model_path = "../models/detectors/face_yolov8n.safetensors"
+
+ input_path = "input.png"
+ output_path = "repaired.png"
+
+ detail_prompt = "detailed portrait photo"
+ negative_prompt = "deformed face"
+
+ seed = 42
+
+ settings = {
+ size = [512, 512],
+ steps = 24,
+ strength = 0.4,
+ cfg_scale = 6.0,
+ method = "dpm++2m",
+ scheduler = "karras",
+ mask_blur = 4
+ }
+}
+
+fn latent_mask(mask, like) {
+ shape = tensor::shape(like)
+ w = shape[0]
+ h = shape[1]
+ batch = shape[3]
+ resized = tensor::interpolate(image::to_f32(mask), size = [w, h], mode = "nearest")
+ return tensor::broadcast_to(resized, [w, h, 1, batch])
+}
+
+fn slice_sigmas_by_strength(sigmas, strength) {
+ steps = tensor::numel(sigmas) - 1
+ retained = math::floor(steps * strength)
+ return sigmas[steps - retained:]
+}
+
+fn dilate(mask) {
+ return tensor::max_pool2d(mask, kernel = [4, 4], axes = [1, 2], padding_before = [2, 2], padding_after = [1, 1], pad_value = 0)
+}
+
+fn inpaint_patch(image, mask, unet, text, vae, prompt, negative_prompt, seed, settings) {
+ pixels = image::to_f32(image)
+ encode_rng = make_rng("cuda", seed = seed)
+ base = vae_to_unet_latents(vae, vae_encode(vae, pixels, rng = encode_rng))
+ latent_region = latent_mask(mask, like = base)
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = settings.cfg_scale)
+ sigmas = slice_sigmas_by_strength(make_sigmas(guided, latent = base, scheduler = settings.scheduler, steps = settings.steps), strength = settings.strength)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ denoise_mask = tensor::max_pool2d(latent_region, kernel = [3, 3], padding_before = [1, 1])
+ latent = sample(start, sigmas, predict = guided, method = settings.method, base = base, denoise_mask = denoise_mask, rng = rng)
+ return vae_decode(vae, unet_to_vae_latents(vae, latent))
+}
+
+fn paste_region(canvas, patch, feather, origin, extent) {
+ background = tensor::crop(canvas, axes = [1, 2], start = origin, size = extent)
+ foreground = tensor::interpolate(patch, size = extent, axes = [1, 2], mode = "bilinear")
+ mask = tensor::crop(feather, axes = [1, 2], start = origin, size = extent)
+ weight = tensor::affine(tensor::cast(mask, "f32"), scale = 0.00392156862745098)
+ blended = tensor::lerp(background, foreground, weight)
+ y = origin[1]
+ rows = tensor::crop(canvas, axes = [2], start = [y], size = [extent[1]])
+ band = tensor::replace_slice(rows, blended, axis = 1, start = origin[0])
+ return tensor::replace_slice(canvas, band, axis = 2, start = y)
+}
+
+fn refine_region(canvas, region, unet, text, vae, prompt, negative_prompt, seed, settings) {
+ origin = region.box[:2]
+ extent = region.box[2:4]
+ channels = tensor::shape(canvas)[0]
+ last = channels - 1
+ rgb = tensor::index_select(canvas, axis = 0, indices = [0, math::min(1, last), math::min(2, last)])
+ image_crop = tensor::crop(rgb, axes = [1, 2], start = origin, size = extent)
+ mask_crop = tensor::crop(region.mask, axes = [1, 2], start = origin, size = extent)
+ image = tensor::interpolate(image_crop, size = settings.size, axes = [1, 2], mode = "bilinear")
+ mask = tensor::interpolate(mask_crop, size = settings.size, axes = [1, 2], mode = "nearest_exact")
+ region_seed = seed + region.index
+ patch = inpaint_patch(image, mask, unet, text, vae, prompt, negative_prompt, region_seed, settings)
+ radius = math::ceil(settings.mask_blur * 2.5)
+ feather = tensor::gaussian_blur(region.mask, sigma = settings.mask_blur, radius = radius, axes = [1, 2])
+ refined = paste_region(rgb, patch, feather, origin, extent)
+ alpha = canvas[math::min(3, channels):]
+ return tensor::concat([refined, alpha], axis = 0)
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ detector = load_detector(model_path)
+ image = image::load(input_path)
+ detections = detections::detect(detector, image, confidence = 0.3)
+ selected = detections::filter(image, detections, top_k = 4)
+ masks = map(detections::box_masks(image, selected), process = dilate)
+ regions = regions::plan_mask_crops(masks, aspect = settings.size, padding = 32)
+ refined = fold(regions, image, process = refine_region, unet = unet, text = text, vae = vae,
+ prompt = detail_prompt, negative_prompt = negative_prompt, seed = seed, settings = settings)
+ image::save(output_path, refined)
+ return { image = refined, detections = selected, regions = regions }
+}
diff --git a/docs/dgl/adetailer/adetailer-generated-image.dgl b/docs/dgl/adetailer/adetailer-generated-image.dgl
new file mode 100644
index 0000000..5ccda8e
--- /dev/null
+++ b/docs/dgl/adetailer/adetailer-generated-image.dgl
@@ -0,0 +1,128 @@
+# version: 0.0.0
+# docs/models/adetailer.md — Repair generated images
+
+var {
+ weights = "../models/checkpoints/model.safetensors"
+ model_path = "../models/detectors/face_yolov8n.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "portrait photo"
+ detail_prompt = "portrait photo, detailed face"
+ negative_prompt = ""
+
+ seed = 42
+
+ settings = {
+ size = [512, 512],
+ steps = 20,
+ strength = 0.4,
+ cfg_scale = 7.0,
+ method = "euler_a",
+ scheduler = "karras",
+ mask_blur = 4
+ }
+}
+
+fn latent_mask(mask, like) {
+ shape = tensor::shape(like)
+ w = shape[0]
+ h = shape[1]
+ batch = shape[3]
+ resized = tensor::interpolate(image::to_f32(mask), size = [w, h], mode = "nearest")
+ return tensor::broadcast_to(resized, [w, h, 1, batch])
+}
+
+fn slice_sigmas_by_strength(sigmas, strength) {
+ steps = tensor::numel(sigmas) - 1
+ retained = math::floor(steps * strength)
+ return sigmas[steps - retained:]
+}
+
+fn dilate(mask) {
+ return tensor::max_pool2d(mask, kernel = [4, 4], axes = [1, 2], padding_before = [2, 2], padding_after = [1, 1], pad_value = 0)
+}
+
+fn inpaint_patch(image, mask, unet, text, vae, prompt, negative_prompt, seed, settings) {
+ pixels = image::to_f32(image)
+ encode_rng = make_rng("cuda", seed = seed)
+ base = vae_to_unet_latents(vae, vae_encode(vae, pixels, rng = encode_rng))
+ latent_region = latent_mask(mask, like = base)
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = settings.cfg_scale)
+ sigmas = slice_sigmas_by_strength(make_sigmas(guided, latent = base, scheduler = settings.scheduler, steps = settings.steps), strength = settings.strength)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ denoise_mask = tensor::max_pool2d(latent_region, kernel = [3, 3], padding_before = [1, 1])
+ latent = sample(start, sigmas, predict = guided, method = settings.method, base = base, denoise_mask = denoise_mask, rng = rng)
+ return vae_decode(vae, unet_to_vae_latents(vae, latent))
+}
+
+fn paste_region(canvas, patch, feather, origin, extent) {
+ background = tensor::crop(canvas, axes = [1, 2], start = origin, size = extent)
+ foreground = tensor::interpolate(patch, size = extent, axes = [1, 2], mode = "bilinear")
+ mask = tensor::crop(feather, axes = [1, 2], start = origin, size = extent)
+ weight = tensor::affine(tensor::cast(mask, "f32"), scale = 0.00392156862745098)
+ blended = tensor::lerp(background, foreground, weight)
+ y = origin[1]
+ rows = tensor::crop(canvas, axes = [2], start = [y], size = [extent[1]])
+ band = tensor::replace_slice(rows, blended, axis = 1, start = origin[0])
+ return tensor::replace_slice(canvas, band, axis = 2, start = y)
+}
+
+fn refine_region(canvas, region, unet, text, vae, prompt, negative_prompt, seed, settings) {
+ origin = region.box[:2]
+ extent = region.box[2:4]
+ channels = tensor::shape(canvas)[0]
+ last = channels - 1
+ rgb = tensor::index_select(canvas, axis = 0, indices = [0, math::min(1, last), math::min(2, last)])
+ image_crop = tensor::crop(rgb, axes = [1, 2], start = origin, size = extent)
+ mask_crop = tensor::crop(region.mask, axes = [1, 2], start = origin, size = extent)
+ image = tensor::interpolate(image_crop, size = settings.size, axes = [1, 2], mode = "bilinear")
+ mask = tensor::interpolate(mask_crop, size = settings.size, axes = [1, 2], mode = "nearest_exact")
+ region_seed = seed + region.index
+ patch = inpaint_patch(image, mask, unet, text, vae, prompt, negative_prompt, region_seed, settings)
+ radius = math::ceil(settings.mask_blur * 2.5)
+ feather = tensor::gaussian_blur(region.mask, sigma = settings.mask_blur, radius = radius, axes = [1, 2])
+ refined = paste_region(rgb, patch, feather, origin, extent)
+ alpha = canvas[math::min(3, channels):]
+ return tensor::concat([refined, alpha], axis = 0)
+}
+
+fn generate_canvas(unet, text, vae, prompt, negative_prompt, seed) {
+ width = 512
+ height = 512
+ scheduler = "discrete"
+ steps = 20
+ cfg_scale = 7
+ method = "euler_a"
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ rng = make_rng("cuda", seed = seed)
+ pos = encode_prompt(text, prompt)
+ neg = encode_prompt(text, negative_prompt)
+ positive_predictor = predictor(unet, pos, prediction = "eps")
+ negative_predictor = predictor(unet, neg, prediction = "eps")
+ guided = cfg(positive_predictor, negative_predictor, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ return vae_decode(vae, unet_to_vae_latents(vae, latent))
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ detector = load_detector(model_path)
+ image = generate_canvas(unet, text, vae, prompt, negative_prompt, seed)
+ detections = detections::detect(detector, image, confidence = 0.3)
+ selected = detections::filter(image, detections, top_k = 4)
+ masks = map(detections::box_masks(image, selected), process = dilate)
+ regions = regions::plan_mask_crops(masks, aspect = settings.size, padding = 32)
+ refined = fold(regions, image, process = refine_region, unet = unet, text = text, vae = vae,
+ prompt = detail_prompt, negative_prompt = negative_prompt, seed = seed, settings = settings)
+ image::save(output_path, refined)
+ return { image = refined, detections = selected, regions = regions }
+}
diff --git a/docs/dgl/anima/anima-preview-txt2img.dgl b/docs/dgl/anima/anima-preview-txt2img.dgl
new file mode 100644
index 0000000..66d2b12
--- /dev/null
+++ b/docs/dgl/anima/anima-preview-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/anima.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/anima-preview.safetensors"
+ text_path = "../models/text_encoders/qwen_3_06b_base.safetensors"
+ vae_path = "../models/vae/qwen_image_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'anima.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "anima", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "anima", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "anima", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/animatediff/animatediff-realisticvision-v6-domain-lora.dgl b/docs/dgl/animatediff/animatediff-realisticvision-v6-domain-lora.dgl
new file mode 100644
index 0000000..51c9380
--- /dev/null
+++ b/docs/dgl/animatediff/animatediff-realisticvision-v6-domain-lora.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/animatediff.md — v3 + Domain Adapter LoRA
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+ model_path = "../models/motion_modules/mm_sd15_v3.safetensors"
+ lora_path = "./loras/v3_sd15_adapter.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "close up photo of a rabbit"
+ negative_prompt = ""
+
+ seed = 42
+ width = 512
+ height = 512
+ frames = 16
+ fps = 8
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ original_unet = load_unet(weights)
+ domain_adapter = load_lora(lora_path)
+ base_unet = apply_lora(original_unet, domain_adapter)
+ motion = animatediff::load(model_path)
+ unet = animatediff::apply(base_unet, motion)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, batch = frames))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = video, fps_num = fps, fps_den = 1})
+ return { video = video, latent = latent }
+}
diff --git a/docs/dgl/animatediff/animatediff-realisticvision-v6-img2video.dgl b/docs/dgl/animatediff/animatediff-realisticvision-v6-img2video.dgl
new file mode 100644
index 0000000..70a3767
--- /dev/null
+++ b/docs/dgl/animatediff/animatediff-realisticvision-v6-img2video.dgl
@@ -0,0 +1,49 @@
+# version: 0.0.0
+# docs/models/animatediff.md — img2video
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+ model_path = "../models/motion_modules/mm_sd15_v3.safetensors"
+
+ input_path = "init.png"
+ output_path = "out.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ width = 512
+ height = 512
+ frames = 8
+ fps = 8
+
+ steps = 25
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "karras"
+ strength = 0.75
+}
+
+fn main() {
+ base_unet = load_unet(weights)
+ motion = animatediff::load(model_path)
+ unet = animatediff::apply(base_unet, motion)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ source = tensor::interpolate(image::load(input_path), size = [width, height], axes = [1, 2])
+ encode_rng = make_rng("cuda", seed = seed)
+ encoded = vae_to_unet_latents(vae, vae_encode(vae, image::to_f32(source), rng = encode_rng))
+ base = tensor::broadcast_to(encoded, latent_shape(vae, width = width, height = height, batch = frames))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ full_sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ retained = math::min(math::floor(steps * strength) + 1, steps)
+ sigmas = full_sigmas[steps - retained:]
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = video, fps_num = fps, fps_den = 1})
+ return { video = video, latent = latent }
+}
diff --git a/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-16-frames.dgl b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-16-frames.dgl
new file mode 100644
index 0000000..c7e38fb
--- /dev/null
+++ b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-16-frames.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/animatediff.md — Text-to-video
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+ model_path = "../models/motion_modules/animatediff/mm_sd15_v3.safetensors"
+
+ output_path = "out.avi"
+
+ prompt = "photo of coastline, rocks, storm weather, wind, waves, lightning"
+ negative_prompt = ""
+
+ seed = 42
+ width = 384
+ height = 384
+ frames = 16
+ fps = 8
+
+ steps = 20
+ cfg_scale = 8.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_unet = load_unet(weights, params_backend = "cpu")
+ motion = animatediff::load(model_path, params_backend = "cpu")
+ unet = animatediff::apply(base_unet, motion)
+ text = load_text_encoder(weights, params_backend = "cpu")
+ vae = load_vae(weights, params_backend = "cpu")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, batch = frames))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = video, fps_num = fps, fps_den = 1})
+ return { video = video, latent = latent }
+}
diff --git a/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl
new file mode 100644
index 0000000..bbc9a71
--- /dev/null
+++ b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/animatediff.md — Text-to-video
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+ model_path = "../models/motion_modules/animatediff/mm_sd15_v3.safetensors"
+
+ output_path = "out.avi"
+
+ prompt = "a red apple on a wooden table"
+ negative_prompt = ""
+
+ seed = 42
+ width = 512
+ height = 512
+ frames = 8
+ fps = 8
+
+ steps = 20
+ cfg_scale = 8.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_unet = load_unet(weights, params_backend = "cpu")
+ motion = animatediff::load(model_path, params_backend = "cpu")
+ unet = animatediff::apply(base_unet, motion)
+ text = load_text_encoder(weights, params_backend = "cpu")
+ vae = load_vae(weights, params_backend = "cpu")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, batch = frames))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = video, fps_num = fps, fps_den = 1})
+ return { video = video, latent = latent }
+}
diff --git a/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-vram-budget.dgl b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-vram-budget.dgl
new file mode 100644
index 0000000..53a90f7
--- /dev/null
+++ b/docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-vram-budget.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# CLI/context: --max-vram 2.0
+# docs/models/animatediff.md — Text-to-video
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+ model_path = "../models/motion_modules/animatediff/mm_sd15_v3.safetensors"
+
+ output_path = "out.avi"
+
+ prompt = "photo of coastline, rocks, storm weather, wind, waves, lightning"
+ negative_prompt = ""
+
+ seed = 42
+ width = 384
+ height = 384
+ frames = 8
+ fps = 8
+
+ steps = 20
+ cfg_scale = 8.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_unet = load_unet(weights)
+ motion = animatediff::load(model_path)
+ unet = animatediff::apply(base_unet, motion)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, batch = frames))
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = video, fps_num = fps, fps_den = 1})
+ return { video = video, latent = latent }
+}
diff --git a/docs/dgl/animatediff/realisticvision-v6-txt2img.dgl b/docs/dgl/animatediff/realisticvision-v6-txt2img.dgl
new file mode 100644
index 0000000..d185f2d
--- /dev/null
+++ b/docs/dgl/animatediff/realisticvision-v6-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/animatediff.md — img2video
+
+var {
+ weights = "../models/checkpoints/realisticVisionV60B1.safetensors"
+
+ output_path = "init.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ unet = load_unet(weights)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ rng = make_rng("cuda", seed = seed)
+ pos = encode_prompt(text, prompt)
+ neg = encode_prompt(text, negative_prompt)
+ positive_predictor = predictor(unet, pos, prediction = "eps")
+ negative_predictor = predictor(unet, neg, prediction = "eps")
+ guided = cfg(positive_predictor, negative_predictor, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent, positive = pos, sigmas = sigmas }
+}
diff --git a/docs/dgl/boogu_image/boogu-image-base-txt2img.dgl b/docs/dgl/boogu_image/boogu-image-base-txt2img.dgl
new file mode 100644
index 0000000..09f716f
--- /dev/null
+++ b/docs/dgl/boogu_image/boogu-image-base-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/boogu_image.md — Boogu Image Base
+
+var {
+ model_path = "../models/diffusion_models/boogu_image_base_bf16.safetensors"
+ text_path = "../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "boogu_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "boogu_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "boogu_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/boogu_image/boogu-image-edit.dgl b/docs/dgl/boogu_image/boogu-image-edit.dgl
new file mode 100644
index 0000000..503274c
--- /dev/null
+++ b/docs/dgl/boogu_image/boogu-image-edit.dgl
@@ -0,0 +1,68 @@
+# version: 0.0.0
+# docs/models/boogu_image.md — Boogu Image Edit
+
+var {
+ model_path = "../models/diffusion_models/boogu_image_edit_bf16.safetensors"
+ text_path = "../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/mmproj-Qwen3VL-8B-Instruct-F16.gguf"
+ vae_path = "../models/vae/ae.sft"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'boogu.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "boogu_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "boogu_image", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "boogu_image", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_original])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0])
+ neg = predictor(model, negative, ref_latents = [reference_latent_0])
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/chroma/chroma-unlocked-v40-txt2img.dgl b/docs/dgl/chroma/chroma-unlocked-v40-txt2img.dgl
new file mode 100644
index 0000000..54fe86d
--- /dev/null
+++ b/docs/dgl/chroma/chroma-unlocked-v40-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/chroma.md — Example
+
+var {
+ model_path = "../models/diffusion_models/chroma-unlocked-v40-q8_0.gguf"
+ text_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'chroma.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "chroma", model_args = "chroma_use_dit_mask=false")
+ text = load_text_encoder(text_path, backend = "cpu", family = "chroma", model_args = "chroma_use_dit_mask=false", t5_path = text_path)
+ vae = load_vae(vae_path, family = "chroma", model_args = "chroma_use_dit_mask=false")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl b/docs/dgl/chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl
new file mode 100644
index 0000000..7ea5178
--- /dev/null
+++ b/docs/dgl/chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/chroma_radiance.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/Chroma1-Radiance-v0.4-Q8_0.gguf"
+ text_path = "../models/text_encoders/clip/t5xxl_fp16.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'chroma radiance cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "chroma_radiance")
+ text = load_text_encoder(text_path, family = "chroma_radiance", t5_path = text_path)
+ vae = load_vae(model_path, family = "chroma_radiance")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/distilled_sd/sdxs-512-dreamshaper-txt2img.dgl b/docs/dgl/distilled_sd/sdxs-512-dreamshaper-txt2img.dgl
new file mode 100644
index 0000000..6caf874
--- /dev/null
+++ b/docs/dgl/distilled_sd/sdxs-512-dreamshaper-txt2img.dgl
@@ -0,0 +1,36 @@
+# version: 0.0.0
+# docs/models/distilled_sd.md — SDXS-512-DreamShaper
+
+var {
+ model_path = "../models/checkpoints/sdxs.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "portrait of a lovely cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 1
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sdxs_512_ds")
+ text = load_text_encoder(model_path, family = "sdxs_512_ds")
+ vae = load_vae(model_path, family = "sdxs_512_ds")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/distilled_sd/ssd1b-txt2img.dgl b/docs/dgl/distilled_sd/ssd1b-txt2img.dgl
new file mode 100644
index 0000000..7913c81
--- /dev/null
+++ b/docs/dgl/distilled_sd/ssd1b-txt2img.dgl
@@ -0,0 +1,54 @@
+# version: 0.0.0
+# docs/models/distilled_sd.md — SSD1B
+
+var {
+ model_path = "../models/checkpoints/SSD-1B-A1111.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+ zero_negative = true
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn sdxl_zero_text(cond) {
+ zeroed = zero_out(cond, fields = ["crossattn"])
+ pooled_zeros = tensor::zeros_like(cond.vector[:1280])
+ adm = tensor::replace_slice(cond.vector, pooled_zeros, axis = 0, start = 0)
+ return {crossattn = zeroed.crossattn, vector = adm}
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sdxl_ssd1b")
+ text = load_text_encoder(model_path, family = "sdxl_ssd1b")
+ vae = load_vae(model_path, family = "sdxl_ssd1b", conv_scale = 0.03125)
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height)
+ negative = if zero_negative {
+ sdxl_zero_text(negative_text)
+ } else {
+ {crossattn = negative_text.crossattn, vector = negative_text.vector}
+ }
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ernie_image/ernie-image-turbo-txt2img.dgl b/docs/dgl/ernie_image/ernie-image-turbo-txt2img.dgl
new file mode 100644
index 0000000..5cdd6eb
--- /dev/null
+++ b/docs/dgl/ernie_image/ernie-image-turbo-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/ernie_image.md — ERNIE-Image-Turbo
+
+var {
+ model_path = "../models/diffusion_models/ernie-image-turbo.safetensors"
+ text_path = "../models/text_encoders/ministral-3-3b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 8
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ernie_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ernie_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "ernie_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ernie_image/ernie-image-txt2img.dgl b/docs/dgl/ernie_image/ernie-image-txt2img.dgl
new file mode 100644
index 0000000..f6980f5
--- /dev/null
+++ b/docs/dgl/ernie_image/ernie-image-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/ernie_image.md — ERNIE-Image
+
+var {
+ model_path = "../models/diffusion_models/ernie-image-UD-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/ministral-3-3b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ernie_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ernie_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "ernie_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/esrgan/realesrgan-x4plus-anime-6b-upscale.dgl b/docs/dgl/esrgan/realesrgan-x4plus-anime-6b-upscale.dgl
new file mode 100644
index 0000000..0c02c09
--- /dev/null
+++ b/docs/dgl/esrgan/realesrgan-x4plus-anime-6b-upscale.dgl
@@ -0,0 +1,17 @@
+# version: 0.0.0
+# docs/models/esrgan.md — standalone image upscaling
+
+var {
+ upscaler_path = "../models/upscale_models/RealESRGAN_x4plus_anime_6B.pth"
+
+ input_path = "input.png"
+ output_path = "upscaled.png"
+}
+
+fn main() {
+ upscaler = load_upscaler(upscaler_path)
+ image = image::load(input_path)
+ output = upscale(upscaler, image)
+ image::save(output_path, output)
+ return output
+}
diff --git a/docs/dgl/esrgan/sd1-5-txt2img-esrgan.dgl b/docs/dgl/esrgan/sd1-5-txt2img-esrgan.dgl
new file mode 100644
index 0000000..080effd
--- /dev/null
+++ b/docs/dgl/esrgan/sd1-5-txt2img-esrgan.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/esrgan.md — CLI
+
+var {
+ model_path = "../models/checkpoints/v1-5-pruned-emaonly.safetensors"
+ upscaler_path = "../models/upscale_models/RealESRGAN_x4plus_anime_6B.pth"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ upscaler = load_upscaler(upscaler_path)
+ enlarged = upscale(upscaler, image)
+ image::save(output_path, enlarged)
+ return {image = enlarged, latent = latent}
+}
diff --git a/docs/dgl/fibo2/fibo2-dev-edit.dgl b/docs/dgl/fibo2/fibo2-dev-edit.dgl
new file mode 100644
index 0000000..ac890f7
--- /dev/null
+++ b/docs/dgl/fibo2/fibo2-dev-edit.dgl
@@ -0,0 +1,61 @@
+# version: 0.0.0
+# docs/models/fibo2.md — Fibo 2.0 Dev
+
+var {
+ model_path = "../models/diffusion_models/fibo2_dev_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'fibo2.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn center_crop_reference(input, width, height) {
+ shape = tensor::shape(input)
+ scale = tensor::maximum(tensor::vector([width / shape[0]]), tensor::vector([height / shape[1]]))[0]
+ resized_width = math::ceil(shape[0] * scale)
+ resized_height = math::ceil(shape[1] * scale)
+ resized = tensor::interpolate(input, size = [resized_width, resized_height], mode = "lanczos")
+ return tensor::crop(resized, axes = [0, 1], start = [math::floor((resized_width - width) / 2), math::floor((resized_height - height) / 2)], size = [width, height])
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "fibo2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "fibo2", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "fibo2", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = center_crop_reference(reference_0_original, width, height)
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_pixels])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/fibo2/fibo2-dev-txt2img.dgl b/docs/dgl/fibo2/fibo2-dev-txt2img.dgl
new file mode 100644
index 0000000..fa8d213
--- /dev/null
+++ b/docs/dgl/fibo2/fibo2-dev-txt2img.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/fibo2.md — Fibo 2.0 Dev
+
+var {
+ model_path = "../models/diffusion_models/fibo2_dev_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'fibo2.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "fibo2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "fibo2", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "fibo2", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/fibo2/fibo2-edit-turbo.dgl b/docs/dgl/fibo2/fibo2-edit-turbo.dgl
new file mode 100644
index 0000000..ac9804b
--- /dev/null
+++ b/docs/dgl/fibo2/fibo2-edit-turbo.dgl
@@ -0,0 +1,56 @@
+# version: 0.0.0
+# docs/models/fibo2.md — Fibo 2.0 Edit Turbo
+
+var {
+ model_path = "../models/diffusion_models/fibo2_edit_turbo_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'fibo2.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn center_crop_reference(input, width, height) {
+ shape = tensor::shape(input)
+ scale = tensor::maximum(tensor::vector([width / shape[0]]), tensor::vector([height / shape[1]]))[0]
+ resized_width = math::ceil(shape[0] * scale)
+ resized_height = math::ceil(shape[1] * scale)
+ resized = tensor::interpolate(input, size = [resized_width, resized_height], mode = "lanczos")
+ return tensor::crop(resized, axes = [0, 1], start = [math::floor((resized_width - width) / 2), math::floor((resized_height - height) / 2)], size = [width, height])
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "fibo2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "fibo2", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "fibo2", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = center_crop_reference(reference_0_original, width, height)
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_pixels])
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux/flux1-dev-lora.dgl b/docs/dgl/flux/flux1-dev-lora.dgl
new file mode 100644
index 0000000..9b4fc41
--- /dev/null
+++ b/docs/dgl/flux/flux1-dev-lora.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/flux.md — Flux-dev q8_0 with LoRA
+
+var {
+ model_path = "../models/diffusion_models/flux1-dev-q8_0.gguf"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ t5_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ vae_path = "../models/vae/ae.sft"
+ lora_0_path = "../models/loras/realism_lora_comfy_converted.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'flux.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ base_model = load_dit(model_path, family = "flux")
+ text = load_text_encoder(clip_l_path, backend = "cpu", clip_l_path = clip_l_path, family = "flux", t5_path = t5_path)
+ vae = load_vae(vae_path, family = "flux")
+
+ lora_0 = load_lora(lora_0_path)
+ model = apply_lora(base_model, lora_0)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux/flux1-dev-txt2img.dgl b/docs/dgl/flux/flux1-dev-txt2img.dgl
new file mode 100644
index 0000000..a5e3a5b
--- /dev/null
+++ b/docs/dgl/flux/flux1-dev-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/flux.md — Flux-dev
+
+var {
+ model_path = "../models/diffusion_models/flux1-dev-q8_0.gguf"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ t5_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'flux.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux")
+ text = load_text_encoder(clip_l_path, backend = "cpu", clip_l_path = clip_l_path, family = "flux", t5_path = t5_path)
+ vae = load_vae(vae_path, family = "flux")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux/flux1-schnell-txt2img.dgl b/docs/dgl/flux/flux1-schnell-txt2img.dgl
new file mode 100644
index 0000000..0667f10
--- /dev/null
+++ b/docs/dgl/flux/flux1-schnell-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/flux.md — Flux-schnell
+
+var {
+ model_path = "../models/diffusion_models/flux1-schnell-q8_0.gguf"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ t5_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'flux.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux")
+ text = load_text_encoder(clip_l_path, backend = "cpu", clip_l_path = clip_l_path, family = "flux", t5_path = t5_path)
+ vae = load_vae(vae_path, family = "flux")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-dev-edit.dgl b/docs/dgl/flux2/flux2-dev-edit.dgl
new file mode 100644
index 0000000..70abada
--- /dev/null
+++ b/docs/dgl/flux2/flux2-dev-edit.dgl
@@ -0,0 +1,62 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2-dev
+
+var {
+ model_path = "../models/diffusion_models/flux2-dev-Q4_K_S.gguf"
+ text_path = "../models/text_encoders/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "./kontext_input.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'flux2-dev.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "flux2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-4b-edit.dgl b/docs/dgl/flux2/flux2-klein-4b-edit.dgl
new file mode 100644
index 0000000..930faf1
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-4b-edit.dgl
@@ -0,0 +1,62 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 4B / Flux.2 klein base 4B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-4b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_4b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "./kontext_input.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'klein.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-4b-txt2img.dgl b/docs/dgl/flux2/flux2-klein-4b-txt2img.dgl
new file mode 100644
index 0000000..a33edfa
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-4b-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 4B / Flux.2 klein base 4B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-4b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_4b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-9b-edit.dgl b/docs/dgl/flux2/flux2-klein-9b-edit.dgl
new file mode 100644
index 0000000..ea14525
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-9b-edit.dgl
@@ -0,0 +1,62 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 9B / Flux.2 klein base 9B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-9b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_8b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "./kontext_input.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'klein.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-9b-txt2img.dgl b/docs/dgl/flux2/flux2-klein-9b-txt2img.dgl
new file mode 100644
index 0000000..a0ea095
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-9b-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 9B / Flux.2 klein base 9B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-9b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_8b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-base-4b-txt2img.dgl b/docs/dgl/flux2/flux2-klein-base-4b-txt2img.dgl
new file mode 100644
index 0000000..549fbf0
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-base-4b-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 4B / Flux.2 klein base 4B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-base-4b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_4b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux2/flux2-klein-base-9b-txt2img.dgl b/docs/dgl/flux2/flux2-klein-base-9b-txt2img.dgl
new file mode 100644
index 0000000..17ac175
--- /dev/null
+++ b/docs/dgl/flux2/flux2-klein-base-9b-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/flux2.md — Flux.2 klein 9B / Flux.2 klein base 9B
+
+var {
+ model_path = "../models/diffusion_models/flux-2-klein-base-9b.safetensors"
+ text_path = "../models/text_encoders/qwen_3_8b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "flux2"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux2_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux2_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux2_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux3/flux3-image-klein-base-txt2img.dgl b/docs/dgl/flux3/flux3-image-klein-base-txt2img.dgl
new file mode 100644
index 0000000..ebdd495
--- /dev/null
+++ b/docs/dgl/flux3/flux3-image-klein-base-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/flux3.md — FLUX.3 Image Klein Base
+
+var {
+ model_path = "../models/diffusion_models/flux-3-image-klein-base.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'flux3.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux3_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux3_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux3_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux3/flux3-image-klein-edit.dgl b/docs/dgl/flux3/flux3-image-klein-edit.dgl
new file mode 100644
index 0000000..25f519a
--- /dev/null
+++ b/docs/dgl/flux3/flux3-image-klein-edit.dgl
@@ -0,0 +1,62 @@
+# version: 0.0.0
+# docs/models/flux3.md — FLUX.3 Image Klein
+
+var {
+ model_path = "../models/diffusion_models/flux-3-image-klein.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ reference_0_path = "../assets/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'flux3-image-klein.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 16
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "flux3_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux3_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux3_klein", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/flux3/flux3-image-klein-txt2img.dgl b/docs/dgl/flux3/flux3-image-klein-txt2img.dgl
new file mode 100644
index 0000000..633e6ce
--- /dev/null
+++ b/docs/dgl/flux3/flux3-image-klein-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/flux3.md — FLUX.3 Image Klein
+
+var {
+ model_path = "../models/diffusion_models/flux-3-image-klein.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'flux3.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 16
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "flux3_klein", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "flux3_klein", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "flux3_klein", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/hidream_o1_image/hidream-o1-image-dev-txt2img.dgl b/docs/dgl/hidream_o1_image/hidream-o1-image-dev-txt2img.dgl
new file mode 100644
index 0000000..cd4e5d5
--- /dev/null
+++ b/docs/dgl/hidream_o1_image/hidream-o1-image-dev-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/hidream_o1_image.md — HiDream-O1-Image-Dev
+
+var {
+ model_path = "../models/checkpoints/hidream_o1_image_dev_bf16.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'hidream o1 cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ noise_scale = 8.0
+}
+
+fn main() {
+ model = load_dit(model_path, family = "hidream_o1")
+ text = load_text_encoder(model_path, family = "hidream_o1")
+ vae = load_vae(model_path, family = "hidream_o1")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ noise = tensor::affine(tensor::randn_like(base, rng = rng), scale = noise_scale)
+ start = add_noise(predict, base, noise, sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl b/docs/dgl/hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl
new file mode 100644
index 0000000..e18ac53
--- /dev/null
+++ b/docs/dgl/hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl
@@ -0,0 +1,46 @@
+# version: 0.0.0
+# docs/models/hunyuan_video.md — Text-to-video example
+
+var {
+ model_path = "../models/diffusion_models/hunyuanvideo1.5_720p_t2v_fp16.safetensors"
+ text_path = "../models/text_encoders/qwen_2.5_vl_7b.safetensors"
+ t5_path = "../models/text_encoders/byt5_small_glyphxl_fp16.safetensors"
+ vae_path = "../models/vae/hunyuanvideo15_vae_fp16.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 720
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "hunyuan_video", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "hunyuan_video", params_backend = "cpu", t5_path = t5_path)
+ vae = load_vae(vae_path, family = "hunyuan_video", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent), tiling = tiling())
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/ideogram4/ideogram4-txt2img.dgl b/docs/dgl/ideogram4/ideogram4-txt2img.dgl
new file mode 100644
index 0000000..9eb7644
--- /dev/null
+++ b/docs/dgl/ideogram4/ideogram4-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/ideogram4.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/ideogram4-Q8_0.gguf"
+ negative_model_path = "../models/diffusion_models/ideogram4_uncond-Q8_0.gguf"
+ text_path = "../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "{\"high_level_description\":\"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.\",\"style_description\":{\"aesthetics\":\"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished\",\"lighting\":\"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background\",\"photo\":\"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field\",\"medium\":\"mixed media fashion photography and premium editorial graphic design\",\"color_palette\":[\"#F4EFE7\",\"#111111\",\"#D8B56D\",\"#B73A3A\",\"#FFFFFF\",\"#8A7A6A\"]},\"compositional_deconstruction\":{\"canvas\":\"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.\",\"background\":\"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.\",\"layout\":\"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.\",\"elements\":[{\"type\":\"text\",\"desc\":\"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text.\"},{\"type\":\"obj\",\"desc\":\"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated.\"},{\"type\":\"obj\",\"desc\":\"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude.\"},{\"type\":\"obj\",\"desc\":\"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate.\"},{\"type\":\"obj\",\"desc\":\"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail.\"},{\"type\":\"obj\",\"desc\":\"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing.\"},{\"type\":\"text\",\"desc\":\"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right.\"},{\"type\":\"obj\",\"desc\":\"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations.\"},{\"type\":\"text\",\"desc\":\"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text.\"}]}}"
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "logit_normal"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ideogram4", params_backend = "cpu")
+ negative_model = load_dit(negative_model_path, family = "ideogram4", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ideogram4", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "ideogram4", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = {vector = tensor::vector([1])}
+ pos = predictor(model, positive)
+ neg = predictor(negative_model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(pos, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ip_adapter/ip-adapter-plus-sd1-5.dgl b/docs/dgl/ip_adapter/ip-adapter-plus-sd1-5.dgl
new file mode 100644
index 0000000..8eb9db4
--- /dev/null
+++ b/docs/dgl/ip_adapter/ip-adapter-plus-sd1-5.dgl
@@ -0,0 +1,49 @@
+# version: 0.0.0
+# docs/models/ip_adapter.md — Plus variants
+
+var {
+ weights = "../models/checkpoints/sd_v1.5.safetensors"
+ vision_encoder_path = "../models/clip_vision/clip_vision_h.safetensors"
+ ip_adapter_path = "../models/ipadapter/ip-adapter-plus_sd15.safetensors"
+
+ input_path = "../inputs/reference.png"
+ output_path = "output.png"
+
+ prompt = "a woman, best quality"
+ negative_prompt = "lowres, bad anatomy"
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 30
+ cfg_scale = 7.0
+ method = "dpm++2m"
+ scheduler = "karras"
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ vision = load_vision_encoder(vision_encoder_path)
+ adapter = ip_adapter::load(ip_adapter_path, family = "sd1")
+ reference = image::load(input_path)
+ embedding = encode_vision(vision, reference)
+ positive_image = ip_adapter::project(adapter, embedding)
+ negative_embedding = encode_vision(vision, reference, zero_input = true)
+ negative_image = ip_adapter::project(adapter, negative_embedding)
+ positive = ip_adapter::apply(encode_prompt(text, prompt, width = width, height = height), adapter, positive_image, strength = 0.8)
+ negative = ip_adapter::apply(encode_prompt(text, negative_prompt, width = width, height = height), adapter, negative_image, strength = 0.8)
+ pos = predictor(unet, positive, prediction = "eps")
+ neg = predictor(unet, negative, prediction = "eps")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent, image_tokens = positive_image }
+}
diff --git a/docs/dgl/ip_adapter/ip-adapter-sd1-5.dgl b/docs/dgl/ip_adapter/ip-adapter-sd1-5.dgl
new file mode 100644
index 0000000..145e90c
--- /dev/null
+++ b/docs/dgl/ip_adapter/ip-adapter-sd1-5.dgl
@@ -0,0 +1,47 @@
+# version: 0.0.0
+# docs/models/ip_adapter.md — Example (SD 1.5)
+
+var {
+ weights = "../models/checkpoints/sd_v1.5.safetensors"
+ vision_encoder_path = "../models/clip_vision/clip_vision_h.safetensors"
+ ip_adapter_path = "../models/ipadapter/ip-adapter_sd15.safetensors"
+
+ input_path = "../inputs/reference.png"
+ output_path = "output.png"
+
+ prompt = "a woman, best quality"
+ negative_prompt = "lowres, bad anatomy"
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 30
+ cfg_scale = 7.0
+ method = "dpm++2m"
+ scheduler = "karras"
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ vision = load_vision_encoder(vision_encoder_path)
+ adapter = ip_adapter::load(ip_adapter_path, family = "sd1")
+ embedding = encode_vision(vision, image::load(input_path), pooled = true)
+ positive_image = ip_adapter::project(adapter, embedding)
+ negative_image = ip_adapter::project(adapter, tensor::zeros_like(embedding))
+ positive = ip_adapter::apply(encode_prompt(text, prompt, width = width, height = height), adapter, positive_image, strength = 0.8)
+ negative = ip_adapter::apply(encode_prompt(text, negative_prompt, width = width, height = height), adapter, negative_image, strength = 0.8)
+ pos = predictor(unet, positive, prediction = "eps")
+ neg = predictor(unet, negative, prediction = "eps")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent, image_tokens = positive_image }
+}
diff --git a/docs/dgl/ip_adapter/ip-adapter-sdxl-controlnet.dgl b/docs/dgl/ip_adapter/ip-adapter-sdxl-controlnet.dgl
new file mode 100644
index 0000000..64a4f7f
--- /dev/null
+++ b/docs/dgl/ip_adapter/ip-adapter-sdxl-controlnet.dgl
@@ -0,0 +1,76 @@
+# version: 0.0.0
+# docs/models/ip_adapter.md — Combining with Control Net
+
+var {
+ weights = "../models/checkpoints/sdxl.safetensors"
+ vision_encoder_path = "../models/clip_vision/clip_vision_h.safetensors"
+ ip_adapter_path = "../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors"
+ control_path = "../models/controlnet/OpenPoseXL2.safetensors"
+
+ input_path = "../inputs/character.png"
+ control_image_path = "../inputs/pose.png"
+ output_path = "output.png"
+
+ prompt = "a character, side view"
+ negative_prompt = ""
+ zero_negative = true
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ steps = 25
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn sdxl_zero_text(cond) {
+ zeroed = zero_out(cond, fields = ["crossattn"])
+ pooled_zeros = tensor::zeros_like(cond.vector[:1280])
+ adm = tensor::replace_slice(cond.vector, pooled_zeros, axis = 0, start = 0)
+ return {crossattn = zeroed.crossattn, vector = adm}
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights, conv_scale = 0.03125)
+ vision = load_vision_encoder(vision_encoder_path)
+ adapter = ip_adapter::load(ip_adapter_path, family = "sdxl")
+ embedding = encode_vision(vision, image::load(input_path), pooled = true)
+ positive_image = ip_adapter::project(adapter, embedding)
+ negative_image = ip_adapter::project(adapter, tensor::zeros_like(embedding))
+ positive = ip_adapter::apply(encode_prompt(text, prompt, width = width, height = height), adapter, positive_image, strength = 0.9)
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height)
+ negative_condition = if zero_negative {
+ sdxl_zero_text(negative_text)
+ } else {
+ {crossattn = negative_text.crossattn, vector = negative_text.vector}
+ }
+ negative = ip_adapter::apply(negative_condition, adapter, negative_image, strength = 0.9)
+ control = controlnet::load(control_path, family = "sdxl")
+ hint = fit_canvas(image::load(control_image_path), width, height, mode = "nearest")
+ controlled_positive = controlnet::apply(positive, control, hint, strength = 0.8)
+ controlled_negative = controlnet::apply(negative, control, hint, strength = 0.8)
+ pos = predictor(unet, controlled_positive, prediction = "eps")
+ neg = predictor(unet, controlled_negative, prediction = "eps")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent), tiling = tiling())
+ image::save(output_path, image)
+ return { image = image, latent = latent, image_tokens = positive_image }
+}
diff --git a/docs/dgl/ip_adapter/ip-adapter-sdxl.dgl b/docs/dgl/ip_adapter/ip-adapter-sdxl.dgl
new file mode 100644
index 0000000..86079dd
--- /dev/null
+++ b/docs/dgl/ip_adapter/ip-adapter-sdxl.dgl
@@ -0,0 +1,47 @@
+# version: 0.0.0
+# docs/models/ip_adapter.md — Example (SDXL)
+
+var {
+ weights = "../models/checkpoints/sdxl.safetensors"
+ vision_encoder_path = "../models/clip_vision/clip_vision_h.safetensors"
+ ip_adapter_path = "../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors"
+
+ input_path = "../inputs/reference.png"
+ output_path = "output.png"
+
+ prompt = "a woman, best quality"
+ negative_prompt = "lowres, bad anatomy"
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ steps = 25
+ cfg_scale = 6.0
+ method = "dpm++2m"
+ scheduler = "karras"
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights, conv_scale = 0.03125)
+ vision = load_vision_encoder(vision_encoder_path)
+ adapter = ip_adapter::load(ip_adapter_path, family = "sdxl")
+ embedding = encode_vision(vision, image::load(input_path), pooled = true)
+ positive_image = ip_adapter::project(adapter, embedding)
+ negative_image = ip_adapter::project(adapter, tensor::zeros_like(embedding))
+ positive = ip_adapter::apply(encode_prompt(text, prompt, width = width, height = height), adapter, positive_image, strength = 0.8)
+ negative = ip_adapter::apply(encode_prompt(text, negative_prompt, width = width, height = height), adapter, negative_image, strength = 0.8)
+ pos = predictor(unet, positive, prediction = "eps")
+ neg = predictor(unet, negative, prediction = "eps")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent), tiling = tiling())
+ image::save(output_path, image)
+ return { image = image, latent = latent, image_tokens = positive_image }
+}
diff --git a/docs/dgl/iris/iris-3b-txt2img.dgl b/docs/dgl/iris/iris-3b-txt2img.dgl
new file mode 100644
index 0000000..8f48442
--- /dev/null
+++ b/docs/dgl/iris/iris-3b-txt2img.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/iris.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/iris-3b.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "iris_3b")
+ text = load_text_encoder(text_path, family = "iris_3b")
+ base = tensor::zeros([width, height, 3, 1])
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ pixels = tensor::clamp(tensor::affine(latent, scale = 0.5, offset = 0.5), 0, 1)
+ image = image::to_u8(pixels)
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/kontext/flux1-kontext-dev-edit.dgl b/docs/dgl/kontext/flux1-kontext-dev-edit.dgl
new file mode 100644
index 0000000..a13cb8b
--- /dev/null
+++ b/docs/dgl/kontext/flux1-kontext-dev-edit.dgl
@@ -0,0 +1,63 @@
+# version: 0.0.0
+# docs/models/kontext.md — Example
+
+var {
+ model_path = "../models/diffusion_models/flux1-kontext-dev-q8_0.gguf"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ t5_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ reference_0_path = "./flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'kontext.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "flux")
+ text = load_text_encoder(clip_l_path, backend = "cpu", clip_l_path = clip_l_path, family = "flux", t5_path = t5_path)
+ vae = load_vae(vae_path, family = "flux")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 16, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/krea2/krea2-raw-txt2img.dgl b/docs/dgl/krea2/krea2-raw-txt2img.dgl
new file mode 100644
index 0000000..621374e
--- /dev/null
+++ b/docs/dgl/krea2/krea2-raw-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/krea2.md — Krea2
+
+var {
+ model_path = "../models/diffusion_models/Krea-2-Raw-Q8_0.gguf"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'krea2.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "krea2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "krea2", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "krea2", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/lcm/lcm-lora-sd1-5.dgl b/docs/dgl/lcm/lcm-lora-sd1-5.dgl
new file mode 100644
index 0000000..16895dd
--- /dev/null
+++ b/docs/dgl/lcm/lcm-lora-sd1-5.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/lcm.md — LCM-LoRA with SD 1.5
+
+var {
+ model_path = "../models/checkpoints/v1-5-pruned-emaonly.safetensors"
+ lora_0_path = "../models/loras/lcm-lora-sdv1-5.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ lora_0 = load_lora(lora_0_path)
+ model = apply_lora(base_model, lora_0)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/lens/lens-turbo-txt2img.dgl b/docs/dgl/lens/lens-turbo-txt2img.dgl
new file mode 100644
index 0000000..71299ed
--- /dev/null
+++ b/docs/dgl/lens/lens-turbo-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/lens.md — Lens Turbo
+
+var {
+ model_path = "../models/diffusion_models/lens_turbo_bf16.safetensors"
+ text_path = "../models/text_encoders/gpt-oss-20b-UD-Q8_K_XL.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_gpt_oss.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "lens")
+ text = load_text_encoder(text_path, family = "lens", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "lens")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/lens/lens-txt2img.dgl b/docs/dgl/lens/lens-txt2img.dgl
new file mode 100644
index 0000000..9b39ea9
--- /dev/null
+++ b/docs/dgl/lens/lens-txt2img.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/lens.md — Lens
+
+var {
+ model_path = "../models/diffusion_models/lens_bf16.safetensors"
+ text_path = "../models/text_encoders/gpt-oss-20b-UD-Q8_K_XL.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_gpt_oss.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "lens")
+ text = load_text_encoder(text_path, family = "lens", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "lens")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl b/docs/dgl/lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl
new file mode 100644
index 0000000..197df27
--- /dev/null
+++ b/docs/dgl/lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/lingbot_video.md — LingBot-Video T2V
+
+var {
+ model_path = "../models/diffusion_models/lingbot-video-dens-1.3b.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ negative_prompt = "{\"universal_negative\":{\"visual_quality\":[\"low quality\",\"worst quality\",\"blurry\",\"pixelated\",\"jpeg artifacts\",\"low resolution\",\"unstable color\",\"color flicker\",\"underexposed\",\"overexposed\",\"invisible subject\",\"subject hidden in darkness\"],\"artistic_style\":[\"painting\",\"illustration\",\"drawing\",\"cartoon\",\"3d render\",\"cgi\",\"sketch\",\"digital art\"],\"composition_and_content\":[\"text\",\"watermark\",\"signature\",\"logo\",\"subtitles\",\"pillarboxed\",\"side bars\",\"portrait image in landscape frame\"],\"temporal_and_motion_stability\":[\"flickering\",\"jittery\",\"motion blur\",\"temporal inconsistency\",\"warping\",\"morphing\",\"incoherent motion\",\"unnatural movement\",\"static object with sudden jump\",\"frame-to-frame inconsistency\"],\"material_and_structure\":[\"plastic-like glass\",\"unrealistic texture\",\"deformed bottle\",\"liquid freezing improperly\",\"distorted reflections\"]}}"
+ prompt = "{\"caption\":{\"comprehensive_description\":\"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cats charming appearance, soft texture, and affectionate presence.\",\"camera_info\":{\"color\":\"Warm\",\"frame_size\":\"Close Up\",\"shot_type_angle\":\"Eye level\",\"lens_size\":\"Medium Lens\",\"composition\":\"Centered balanced\",\"lighting\":\"Soft light\",\"lighting_type\":\"Daylight\"},\"world_knowledge\":[],\"prominent_elements\":[{\"name\":\"lovely cat\",\"description\":\"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.\",\"location\":\"center of the frame\",\"relative_size\":\"large\",\"shape_and_color\":\"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones\",\"texture\":\"soft, fluffy, silky\",\"appearance_details\":\"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.\",\"relationship\":\"Acts as the main subject and emotional focal point of the scene.\",\"orientation\":\"facing the camera\",\"pose\":\"sitting comfortably with a relaxed posture\",\"expression\":\"gentle, adorable, calm, slightly curious\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"cat eyes\",\"description\":\"Bright expressive eyes that give the cat a sweet and affectionate appearance.\",\"location\":\"upper center of the cat face\",\"relative_size\":\"small\",\"shape_and_color\":\"Round almond-like eyes with glossy reflections\",\"texture\":\"clear, glossy, reflective\",\"appearance_details\":\"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.\",\"relationship\":\"Enhance the emotional charm and cuteness of the cat.\",\"orientation\":\"looking toward the camera\",\"pose\":\"\",\"expression\":\"soft and attentive\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"soft cushion\",\"description\":\"A comfortable cushion or blanket where the cat is resting.\",\"location\":\"bottom portion of the frame\",\"relative_size\":\"medium\",\"shape_and_color\":\"Soft rounded fabric surface in light warm neutral tones\",\"texture\":\"plush, fabric, cozy\",\"appearance_details\":\"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.\",\"relationship\":\"Provides a cozy resting place for the cat.\",\"orientation\":\"horizontal\",\"pose\":\"\",\"expression\":\"\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"sunlit indoor background\",\"description\":\"A softly blurred indoor background with warm daylight and peaceful home atmosphere.\",\"location\":\"behind the cat, filling the upper and side areas of the frame\",\"relative_size\":\"large\",\"shape_and_color\":\"Soft abstract shapes in warm beige, cream, and pale golden tones\",\"texture\":\"soft, blurry, bokeh-like\",\"appearance_details\":\"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.\",\"relationship\":\"Provides a warm and gentle environment that supports the cute domestic scene.\",\"orientation\":\"upright\",\"pose\":\"\",\"expression\":\"\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\",\"is_cluster\":true,\"number_of_objects\":\"numerous\"}]}}"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "lingbot_video", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "lingbot_video", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "lingbot_video", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/llada_image/llada-image-turbo-edit.dgl b/docs/dgl/llada_image/llada-image-turbo-edit.dgl
new file mode 100644
index 0000000..41037af
--- /dev/null
+++ b/docs/dgl/llada_image/llada-image-turbo-edit.dgl
@@ -0,0 +1,48 @@
+# version: 0.0.0
+# docs/models/llada_image.md — Image editing
+
+var {
+ model_path = "../models/diffusion_models/llada-image-turbo-f16.gguf"
+ text_path = "../models/text_encoders/llada-image-turbo-text_encoder-q8_0.gguf"
+ connectors_path = "../models/text_encoders/llada-image-turbo-connectors-edit.safetensors"
+ tokenizer_path = "../models/tokenizers/tokenizer.json"
+ vae_path = "../models/vae/llada_vae.safetensors"
+
+ reference_0_path = "../inputs/input.png"
+ output_path = "output.png"
+
+ prompt = "change the sign text to 'sd.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 4
+ method = "euler"
+ scheduler = "llada_image"
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+
+ model = load_dit(model_path, family = "llada_image")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "llada_image", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "llada_image")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = tensor::interpolate(reference_0_original, size = [width, height], mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_pixels])
+ predict = predictor(model, condition, ref_latents = [reference_latent_0], force_ref_timestep_zero = true)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/llada_image/llada-image-turbo-txt2img.dgl b/docs/dgl/llada_image/llada-image-turbo-txt2img.dgl
new file mode 100644
index 0000000..8a93fd6
--- /dev/null
+++ b/docs/dgl/llada_image/llada-image-turbo-txt2img.dgl
@@ -0,0 +1,40 @@
+# version: 0.0.0
+# docs/models/llada_image.md — Text to image
+
+var {
+ model_path = "../models/diffusion_models/llada-image-turbo-f16.gguf"
+ text_path = "../models/text_encoders/llada-image-turbo-text_encoder-q8_0.gguf"
+ connectors_path = "../models/text_encoders/llada-image-turbo-connectors.safetensors"
+ tokenizer_path = "../models/tokenizers/tokenizer.json"
+ vae_path = "../models/vae/llada_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'llada.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 4
+ method = "euler"
+ scheduler = "llada_image"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "llada_image")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "llada_image", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "llada_image")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/longcat_image/longcat-image-txt2img.dgl b/docs/dgl/longcat_image/longcat-image-txt2img.dgl
new file mode 100644
index 0000000..1f43801
--- /dev/null
+++ b/docs/dgl/longcat_image/longcat-image-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/longcat_image.md — LongCat Image
+
+var {
+ model_path = "../models/diffusion_models/LongCat-Image-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'longcat.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "longcat_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "longcat_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "longcat_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/lora/sd1-5-lora.dgl b/docs/dgl/lora/sd1-5-lora.dgl
new file mode 100644
index 0000000..a8f3379
--- /dev/null
+++ b/docs/dgl/lora/sd1-5-lora.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/lora.md — CLI
+
+var {
+ model_path = "../models/checkpoints/v1-5-pruned-emaonly.safetensors"
+ lora_0_path = "../models/loras/marblesh.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_model = load_unet(model_path, family = "sd1")
+ base_text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ lora_0 = load_lora(lora_0_path)
+ model = apply_lora(base_model, lora_0)
+ text = apply_lora(base_text, lora_0)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ltx2/ltx-2-3-dev-first-last-frame.dgl b/docs/dgl/ltx2/ltx-2-3-dev-first-last-frame.dgl
new file mode 100644
index 0000000..2ab96e3
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-3-dev-first-last-frame.dgl
@@ -0,0 +1,100 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.3 dev FLF2V
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf"
+ connectors_path = "../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors"
+ vae_path = "../models/vae/ltx-2.3-22b-dev_video_vae.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors"
+
+ first_frame_path = "../inputs/start_image.png"
+ last_frame_path = "../inputs/end_image.png"
+ output_path = "flf2v.avi"
+
+ prompt = "glass flower blossom"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 736
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "ltx2"
+ strength = 0.75
+ audio_scale = 1
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+ last_frame = image::load(last_frame_path)
+
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ video_shape = tensor::shape(video_base)
+ video_mask = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ video_first = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_first = tensor::index_fill(video_mask, axis = 2, indices = [0], value = 1 - strength)
+ last_latent = vae_to_dit_latents(vae, vae_encode(vae, last_frame_pixels, rng = rng))
+ tail_mask = tensor::full([video_shape[0], video_shape[1], tensor::shape(last_latent)[2], 1, 1], 1 - strength)
+ video_conditioned = tensor::concat([video_first, last_latent], axis = 2)
+ mask_conditioned = tensor::concat([mask_first, tail_mask], axis = 2)
+ positions = ltx::video_positions(video_first, keyframe_latent_frames = tensor::shape(last_latent)[2], keyframe_frame_index = frames - 1, keyframe_pixel_frames = 1, fps = fps)
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_conditioned, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ packed_shape = tensor::shape(base)
+ full_video_mask = tensor::broadcast_to(mask_conditioned, tensor::shape(video_conditioned))
+ packed_mask = tensor::concat([full_video_mask, tensor::full([packed_shape[0], packed_shape[1], packed_shape[2], packed_shape[3] - video_shape[3], 1], 1)], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps, video_positions = positions)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps, video_positions = positions)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, base = base, denoise_mask = packed_mask, mask_input = true)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-3-dev-img2video.dgl b/docs/dgl/ltx2/ltx-2-3-dev-img2video.dgl
new file mode 100644
index 0000000..13793a8
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-3-dev-img2video.dgl
@@ -0,0 +1,92 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.3 dev I2V
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf"
+ connectors_path = "../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors"
+ vae_path = "../models/vae/ltx-2.3-22b-dev_video_vae.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors"
+
+ first_frame_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "i2v.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 736
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "ltx2"
+ strength = 0.75
+ audio_scale = 1
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ video_shape = tensor::shape(video_base)
+ video_mask = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ video_conditioned = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_conditioned = tensor::index_fill(video_mask, axis = 2, indices = [0], value = 1 - strength)
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_conditioned, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ packed_shape = tensor::shape(base)
+ full_video_mask = tensor::broadcast_to(mask_conditioned, tensor::shape(video_conditioned))
+ packed_mask = tensor::concat([full_video_mask, tensor::full([packed_shape[0], packed_shape[1], packed_shape[2], packed_shape[3] - video_shape[3], 1], 1)], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, base = base, denoise_mask = packed_mask, mask_input = true)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-3-dev-txt2video.dgl b/docs/dgl/ltx2/ltx-2-3-dev-txt2video.dgl
new file mode 100644
index 0000000..2712de4
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-3-dev-txt2video.dgl
@@ -0,0 +1,70 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.3 dev T2V
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf"
+ connectors_path = "../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors"
+ vae_path = "../models/vae/ltx-2.3-22b-dev_video_vae.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors"
+
+ output_path = "t2v.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "worst quality, low quality, blurry, distorted, artifacts"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 736
+ frames = 33
+ fps = 24
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "ltx2"
+ audio_scale = 1
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-3-spatial-upscale.dgl b/docs/dgl/ltx2/ltx-2-3-spatial-upscale.dgl
new file mode 100644
index 0000000..6bb5a1e
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-3-spatial-upscale.dgl
@@ -0,0 +1,125 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.3 spatial latent upscale
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf"
+ connectors_path = "../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors"
+ vae_path = "../models/vae/ltx-2.3-22b-dev_video_vae.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors"
+ hires_model_path = "../models/latent_upscale_models/ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
+
+ first_frame_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "hires_i2v.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 640
+ height = 352
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "ltx2"
+ strength = 0.75
+ audio_scale = 1
+
+ hires_steps = 4
+ hires_scheduler_steps = 5
+ hires_vae_scale = 32
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ video_shape = tensor::shape(video_base)
+ video_mask = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ video_conditioned = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_conditioned = tensor::index_fill(video_mask, axis = 2, indices = [0], value = 1 - strength)
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_conditioned, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ packed_shape = tensor::shape(base)
+ full_video_mask = tensor::broadcast_to(mask_conditioned, tensor::shape(video_conditioned))
+ packed_mask = tensor::concat([full_video_mask, tensor::full([packed_shape[0], packed_shape[1], packed_shape[2], packed_shape[3] - video_shape[3], 1], 1)], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, base = base, denoise_mask = packed_mask, mask_input = true)
+ hires_source = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ hires_upscaler = load_latent_upscaler(hires_model_path, family = "ltxav", params_backend = "cpu")
+ hires_upscaled = upscale_latent(hires_upscaler, dit_to_vae_latents(vae, hires_source))
+ hires_upscaled_shape = tensor::shape(hires_upscaled)
+ hires_video = vae_to_dit_latents(vae, tensor::reshape(hires_upscaled, [hires_upscaled_shape[0], hires_upscaled_shape[1], hires_upscaled_shape[2], hires_upscaled_shape[3], 1]))
+ hires_video_shape = tensor::shape(hires_video)
+ hires_original_audio = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ hires_width = hires_video_shape[0] * hires_vae_scale
+ hires_height = hires_video_shape[1] * hires_vae_scale
+ hires_frames = (hires_video_shape[2] - 1) * 8 + 1
+ hires_audio_shape = audio_latent_shape(audio_vae, frames = hires_frames, fps = fps)
+ hires_audio_retained = math::min(audio_shape[1], hires_audio_shape[1])
+ hires_audio = tensor::pad(hires_original_audio[:, :hires_audio_retained], [0, 0, 0, hires_audio_shape[1] - hires_audio_retained])
+ hires_video_mask = tensor::full([hires_video_shape[0], hires_video_shape[1], hires_video_shape[2], 1, 1], 1)
+ hires_first_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, hires_width, hires_height, mode = "nearest"), channels = 3), axis = 2)
+ hires_first_latent = vae_to_dit_latents(vae, vae_encode(vae, hires_first_pixels, rng = rng))
+ hires_conditioned = tensor::replace_slice(hires_video, hires_first_latent, axis = 2, start = 0)
+ hires_condition_mask = tensor::index_fill(hires_video_mask, axis = 2, indices = [0], value = 0)
+ hires_base = pack_av(hires_conditioned, hires_audio, audio_scale = audio_scale)
+ hires_packed_shape = tensor::shape(hires_base)
+ hires_packed_mask = tensor::concat([tensor::broadcast_to(hires_condition_mask, tensor::shape(hires_conditioned)), tensor::full([hires_packed_shape[0], hires_packed_shape[1], hires_packed_shape[2], hires_packed_shape[3] - hires_video_shape[3], 1], 1)], axis = 3)
+ hires_pos = predictor(model, positive, audio_length = hires_audio_shape[1], fps = fps)
+ hires_neg = predictor(model, negative, audio_length = hires_audio_shape[1], fps = fps)
+ hires_guided = cfg(hires_pos, hires_neg, scale = cfg_scale)
+ hires_schedule = make_sigmas(hires_guided, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler)
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+ hires_start = add_noise(hires_guided, hires_base, tensor::randn_like(hires_base, rng = rng), sigma = hires_sigmas[0])
+ hires_latent = sample(hires_start, hires_sigmas, predict = hires_guided, method = method, rng = rng, base = hires_base, denoise_mask = hires_packed_mask, mask_input = true)
+ video_latent = hires_latent[:, :, :hires_video_shape[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(hires_latent, channels = tensor::shape(video_base)[3], audio_shape = hires_audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = hires_latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-5-dev-img2video.dgl b/docs/dgl/ltx2/ltx-2-5-dev-img2video.dgl
new file mode 100644
index 0000000..912af8a
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-5-dev-img2video.dgl
@@ -0,0 +1,91 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.5 dev I2V
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf"
+ text_path = "../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf"
+ vae_path = "../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.5-audio-vae-bf16.safetensors"
+
+ first_frame_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "i2v.avi"
+
+ prompt = "a lovely cat blinking slowly, gentle camera push in"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 736
+ frames = 121
+ fps = 16
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "ltx2"
+ strength = 0.75
+ audio_scale = 1
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ video_shape = tensor::shape(video_base)
+ video_mask = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ video_conditioned = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_conditioned = tensor::index_fill(video_mask, axis = 2, indices = [0], value = 1 - strength)
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_conditioned, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ packed_shape = tensor::shape(base)
+ full_video_mask = tensor::broadcast_to(mask_conditioned, tensor::shape(video_conditioned))
+ packed_mask = tensor::concat([full_video_mask, tensor::full([packed_shape[0], packed_shape[1], packed_shape[2], packed_shape[3] - video_shape[3], 1], 1)], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, base = base, denoise_mask = packed_mask, mask_input = true)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-5-dev-txt2video.dgl b/docs/dgl/ltx2/ltx-2-5-dev-txt2video.dgl
new file mode 100644
index 0000000..55e4fdc
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-5-dev-txt2video.dgl
@@ -0,0 +1,69 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.5 dev T2V
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf"
+ text_path = "../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf"
+ vae_path = "../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.5-audio-vae-bf16.safetensors"
+
+ output_path = "t2v.avi"
+
+ prompt = "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below"
+ negative_prompt = "worst quality, low quality, blurry, distorted, artifacts"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1280
+ height = 736
+ frames = 121
+ fps = 24
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "ltx2"
+ audio_scale = 1
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/ltx2/ltx-2-5-spatial-upscale.dgl b/docs/dgl/ltx2/ltx-2-5-spatial-upscale.dgl
new file mode 100644
index 0000000..8babae4
--- /dev/null
+++ b/docs/dgl/ltx2/ltx-2-5-spatial-upscale.dgl
@@ -0,0 +1,92 @@
+# version: 0.0.0
+# docs/models/ltx2.md — LTX-2.5 spatial latent upscale
+
+var {
+ model_path = "../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf"
+ text_path = "../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf"
+ vae_path = "../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors"
+ audio_vae_path = "../models/vae/ltx-2.5-audio-vae-bf16.safetensors"
+ hires_model_path = "../models/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors"
+
+ output_path = "hires_t2v.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 640
+ height = 352
+ frames = 121
+ fps = 16
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "ltx2"
+ audio_scale = 1
+
+ hires_steps = 6
+ hires_scheduler_steps = 8
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ltxav", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ltxav", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "ltxav", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "ltxav", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, audio_length = audio_shape[1], fps = fps)
+ neg = predictor(model, negative, audio_length = audio_shape[1], fps = fps)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ hires_source = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ hires_upscaler = load_latent_upscaler(hires_model_path, family = "ltxav", params_backend = "cpu")
+ hires_upscaled = upscale_latent(hires_upscaler, dit_to_vae_latents(vae, hires_source))
+ hires_upscaled_shape = tensor::shape(hires_upscaled)
+ hires_video = vae_to_dit_latents(vae, tensor::reshape(hires_upscaled, [hires_upscaled_shape[0], hires_upscaled_shape[1], hires_upscaled_shape[2], hires_upscaled_shape[3], 1]))
+ hires_video_shape = tensor::shape(hires_video)
+ hires_original_audio = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ hires_frames = (hires_video_shape[2] - 1) * 8 + 1
+ hires_audio_shape = audio_latent_shape(audio_vae, frames = hires_frames, fps = fps)
+ hires_audio_retained = math::min(audio_shape[1], hires_audio_shape[1])
+ hires_audio = tensor::pad(hires_original_audio[:, :hires_audio_retained], [0, 0, 0, hires_audio_shape[1] - hires_audio_retained])
+ hires_base = pack_av(hires_video, hires_audio, audio_scale = audio_scale)
+ hires_pos = predictor(model, positive, audio_length = hires_audio_shape[1], fps = fps)
+ hires_neg = predictor(model, negative, audio_length = hires_audio_shape[1], fps = fps)
+ hires_guided = cfg(hires_pos, hires_neg, scale = cfg_scale)
+ hires_schedule = make_sigmas(hires_guided, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler)
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+ hires_start = add_noise(hires_guided, hires_base, tensor::randn_like(hires_base, rng = rng), sigma = hires_sigmas[0])
+ hires_latent = sample(hires_start, hires_sigmas, predict = hires_guided, method = method, rng = rng)
+ video_latent = hires_latent[:, :, :hires_video_shape[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(hires_latent, channels = tensor::shape(video_base)[3], audio_shape = hires_audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = hires_latent, audio = audio}
+}
diff --git a/docs/dgl/mage_flow/mage-flow-edit.dgl b/docs/dgl/mage_flow/mage-flow-edit.dgl
new file mode 100644
index 0000000..4b36682
--- /dev/null
+++ b/docs/dgl/mage_flow/mage-flow-edit.dgl
@@ -0,0 +1,52 @@
+# version: 0.0.0
+# docs/models/mage_flow.md — Image editing
+
+var {
+ model_path = "../models/diffusion_models/Mage-Flow-Edit.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf"
+ vae_path = "../models/vae/mage_vae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'mage.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "mage_flow", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "mage_flow", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "mage_flow", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = tensor::interpolate(reference_0_original, size = [width, height], mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_original])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/mage_flow/mage-flow-turbo-txt2img.dgl b/docs/dgl/mage_flow/mage-flow-turbo-txt2img.dgl
new file mode 100644
index 0000000..d9bcdc5
--- /dev/null
+++ b/docs/dgl/mage_flow/mage-flow-turbo-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/mage_flow.md — Text-to-image
+
+var {
+ model_path = "../models/diffusion_models/Mage-Flow-Turbo.safetensors"
+ text_path = "../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/mage_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'mage.cpp'"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "mage_flow", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "mage_flow", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "mage_flow", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ming_image/ming-image-0-1-design-txt2img.dgl b/docs/dgl/ming_image/ming-image-0-1-design-txt2img.dgl
new file mode 100644
index 0000000..44814ec
--- /dev/null
+++ b/docs/dgl/ming_image/ming-image-0-1-design-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/ming_image.md — Text-to-image
+
+var {
+ model_path = "../models/diffusion_models/ming_image_0.1_design_bf16.safetensors"
+ text_path = "../models/text_encoders/ming_image_0.1_ling_mini_2.0_bf16.safetensors"
+ tokenizer_path = "../models/tokenizers/tokenizer.json"
+ vae_path = "../models/vae/ming_image_vae_bf16.safetensors"
+
+ output_path = "ming_image.png"
+
+ prompt = "A cheerful orange cat sticker, transparent background"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 12
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ming_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ming_image", params_backend = "cpu", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "ming_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/minimax_h3/minimax-h3-first-last-frame.dgl b/docs/dgl/minimax_h3/minimax-h3-first-last-frame.dgl
new file mode 100644
index 0000000..ab0e844
--- /dev/null
+++ b/docs/dgl/minimax_h3/minimax-h3-first-last-frame.dgl
@@ -0,0 +1,85 @@
+# version: 0.0.0
+# docs/models/minimax_h3.md — First/last-frame conditioning
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_video_vae_fp16.safetensors"
+ audio_vae_path = "../models/vae/minimax_h3_audio_vae_fp32.safetensors"
+
+ first_frame_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 864
+ height = 480
+ frames = 56
+ fps = 24
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ audio_scale = 4
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn h3_reference_noise(latent, seed) {
+ rng = make_rng("cuda", seed = seed)
+ return tensor::lerp(latent, tensor::randn_like(latent, rng = rng), 0.001)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "minimax_h3", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ audio_vae = load_audio_vae(audio_vae_path, family = "minimax_h3", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ h3_visual_0_raw = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ h3_visual_0 = h3_reference_noise(h3_visual_0_raw, seed)
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [first_frame_pixels[:, :, 0]])
+ predict = predictor(model, condition, audio_length = audio_shape[0], ref_latents = [h3_visual_0], keyframe_indices = [0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/minimax_h3/minimax-h3-latent-hires.dgl b/docs/dgl/minimax_h3/minimax-h3-latent-hires.dgl
new file mode 100644
index 0000000..f1f3e82
--- /dev/null
+++ b/docs/dgl/minimax_h3/minimax-h3-latent-hires.dgl
@@ -0,0 +1,81 @@
+# version: 0.0.0
+# docs/models/minimax_h3.md — Latent hires second pass (two-stage sampling)
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_video_vae_fp16.safetensors"
+ audio_vae_path = "../models/vae/minimax_h3_audio_vae_fp32.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 864
+ height = 480
+ frames = 56
+ fps = 24
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ audio_scale = 4
+
+ hires_target_width = 1216
+ hires_target_height = 672
+ hires_vae_scale = 16
+ hires_steps = 4
+ hires_scheduler_steps = 9
+ hires_mode = "bicubic"
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "minimax_h3", flash_attn_mode = "on", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "minimax_h3", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, audio_length = audio_shape[0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ hires_source = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ hires_video = tensor::interpolate(hires_source, size = [math::floor(hires_target_width / hires_vae_scale), math::floor(hires_target_height / hires_vae_scale)], mode = hires_mode)
+ hires_video_shape = tensor::shape(hires_video)
+ hires_original_audio = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ hires_base = pack_av(hires_video, hires_original_audio, audio_scale = audio_scale)
+ hires_schedule = make_sigmas(predict, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler)
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+ hires_start = add_noise(predict, hires_base, tensor::randn_like(hires_base, rng = rng), sigma = hires_sigmas[0])
+ hires_latent = sample(hires_start, hires_sigmas, predict = predict, method = method, rng = rng)
+ video_latent = hires_latent[:, :, :hires_video_shape[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(hires_latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = hires_latent, audio = audio}
+}
diff --git a/docs/dgl/minimax_h3/minimax-h3-learned-hires.dgl b/docs/dgl/minimax_h3/minimax-h3-learned-hires.dgl
new file mode 100644
index 0000000..82af9cb
--- /dev/null
+++ b/docs/dgl/minimax_h3/minimax-h3-learned-hires.dgl
@@ -0,0 +1,84 @@
+# version: 0.0.0
+# docs/models/minimax_h3.md — Latent hires second pass (two-stage sampling)
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_video_vae_fp16.safetensors"
+ audio_vae_path = "../models/vae/minimax_h3_audio_vae_fp32.safetensors"
+ hires_model_path = "../models/latent_upscale_models/h3-latent-upscaler-3d.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 864
+ height = 480
+ frames = 56
+ fps = 24
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ audio_scale = 4
+
+ hires_target_width = 1216
+ hires_target_height = 672
+ hires_vae_scale = 16
+ hires_steps = 4
+ hires_scheduler_steps = 9
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "minimax_h3", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "minimax_h3", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, audio_length = audio_shape[0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ hires_source = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ hires_upscaler = load_latent_upscaler(hires_model_path, family = "minimax_h3", params_backend = "cpu")
+ hires_upscaled = upscale_latent(hires_upscaler, hires_source, width = math::floor(hires_target_width / hires_vae_scale), height = math::floor(hires_target_height / hires_vae_scale))
+ hires_upscaled_shape = tensor::shape(hires_upscaled)
+ hires_video = tensor::reshape(hires_upscaled, [hires_upscaled_shape[0], hires_upscaled_shape[1], hires_upscaled_shape[2], hires_upscaled_shape[3], 1])
+ hires_video_shape = tensor::shape(hires_video)
+ hires_original_audio = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ hires_base = pack_av(hires_video, hires_original_audio, audio_scale = audio_scale)
+ hires_schedule = make_sigmas(predict, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler)
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+ hires_start = add_noise(predict, hires_base, tensor::randn_like(hires_base, rng = rng), sigma = hires_sigmas[0])
+ hires_latent = sample(hires_start, hires_sigmas, predict = predict, method = method, rng = rng)
+ video_latent = hires_latent[:, :, :hires_video_shape[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(hires_latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = hires_latent, audio = audio}
+}
diff --git a/docs/dgl/minimax_h3/minimax-h3-reference-to-av.dgl b/docs/dgl/minimax_h3/minimax-h3-reference-to-av.dgl
new file mode 100644
index 0000000..9601aad
--- /dev/null
+++ b/docs/dgl/minimax_h3/minimax-h3-reference-to-av.dgl
@@ -0,0 +1,94 @@
+# version: 0.0.0
+# docs/models/minimax_h3.md — Reference-to-audio-video conditioning
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_ref2va_pruned-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_video_vae_fp16.safetensors"
+ audio_vae_path = "../models/vae/minimax_h3_audio_vae_fp32.safetensors"
+
+ h3_image_0_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "output.avi"
+
+ prompt = "Use the cat from <Picture 1> as the main character. Keep the cat鈥檚 appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat鈥檚 face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat鈥檚 head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat鈥檚 fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat鈥檚 identity unchanged."
+
+ seed = 42
+ rng_type = "cpu"
+ width = 864
+ height = 480
+ frames = 56
+ fps = 24
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ audio_scale = 4
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn h3_reference_noise(latent, seed) {
+ rng = make_rng("cuda", seed = seed)
+ return tensor::lerp(latent, tensor::randn_like(latent, rng = rng), 0.001)
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ h3_image_0 = image::load(h3_image_0_path)
+
+ model = load_dit(model_path, family = "minimax_h3", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "minimax_h3", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ h3_image_0_original = image::to_f32(h3_image_0, channels = 3)
+ h3_image_0_area = math::min(width * height, tensor::shape(h3_image_0_original)[0] * tensor::shape(h3_image_0_original)[1])
+ h3_image_0_pixels = fit_reference_area(h3_image_0_original, h3_image_0_area, 32, mode = "bilinear")
+ h3_visual_0_raw = vae_to_dit_latents(vae, vae_encode(vae, h3_image_0_pixels, rng = rng))
+ h3_visual_0 = h3_reference_noise(h3_visual_0_raw, seed)
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [{image = h3_image_0_pixels}])
+ predict = predictor(model, condition, audio_length = audio_shape[0], ref_latents = [h3_visual_0], reference_blocks = [{kind = "image", video = 0}])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/minimax_h3/minimax-h3-text-to-av.dgl b/docs/dgl/minimax_h3/minimax-h3-text-to-av.dgl
new file mode 100644
index 0000000..57f6942
--- /dev/null
+++ b/docs/dgl/minimax_h3/minimax-h3-text-to-av.dgl
@@ -0,0 +1,65 @@
+# version: 0.0.0
+# docs/models/minimax_h3.md — Text-to-audio-video
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_video_vae_fp16.safetensors"
+ audio_vae_path = "../models/vae/minimax_h3_audio_vae_fp32.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten鈥檚 energetic surfing."
+
+ seed = 42
+ rng_type = "cpu"
+ width = 864
+ height = 480
+ frames = 56
+ fps = 24
+
+ steps = 20
+ method = "euler"
+ scheduler = "discrete"
+ audio_scale = 4
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn main() {
+ model = load_dit(model_path, family = "minimax_h3", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3", params_backend = "cpu")
+
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ audio_vae = load_audio_vae(audio_vae_path, family = "minimax_h3", params_backend = "cpu")
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, audio_length = audio_shape[0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ video_latent = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent))
+ sound_latent = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/minimax_h3_image/minimax-h3-image-edit.dgl b/docs/dgl/minimax_h3_image/minimax-h3-image-edit.dgl
new file mode 100644
index 0000000..88b17fa
--- /dev/null
+++ b/docs/dgl/minimax_h3_image/minimax-h3-image-edit.dgl
@@ -0,0 +1,73 @@
+# version: 0.0.0
+# docs/models/minimax_h3_image.md — Reference-image editing
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_image_pruned_bf16.safetensors"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_image_vae_fp16.safetensors"
+
+ reference_0_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "output.png"
+
+ prompt = "Edit <Picture 1>. Put a red scarf on the cat, preserving its face and the background."
+
+ seed = 42
+ rng_type = "cpu"
+ width = 1024
+ height = 1024
+
+ method = "euler"
+ custom_sigmas = [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, 0.23200887441635132, 0.0]
+}
+
+fn h3_image_reference_noise(latent, seed) {
+ shape = tensor::shape(latent)
+ patches = tensor::zeros([2, 2, shape[-2], math::floor(shape[0] / 2), math::floor(shape[1] / 2)])
+ rng = make_rng("cpu", seed = seed)
+ noise = tensor::reshape(tensor::permute(tensor::randn_like(patches, rng = rng), [0, 3, 1, 4, 2]), shape)
+ return tensor::lerp(latent, noise, 0.001)
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+
+ model = load_dit(model_path, family = "minimax_h3_image", flash_attn_mode = "on", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3_image", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ base = tensor::unsqueeze(image_base, axis = 2)
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_short = math::min(tensor::shape(reference_0_original)[0], tensor::shape(reference_0_original)[1])
+ reference_0_side = math::min(2048, -math::min(-1024, -reference_0_short))
+ reference_0_pixels = fit_reference_area(reference_0_original, tensor::shape(reference_0_original)[0] * tensor::shape(reference_0_original)[1] * reference_0_side * reference_0_side / (reference_0_short * reference_0_short), 32, mode = "lanczos")
+ reference_latent_0_raw = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng, tiling = tiling()))
+ reference_latent_0 = h3_image_reference_noise(reference_latent_0_raw, seed)
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ predict = predictor(model, condition, video_sigma_shift = 1, reference_blocks = [{kind = "image", video = 0}], ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent), tiling = tiling())
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/minimax_h3_image/minimax-h3-image-txt2img.dgl b/docs/dgl/minimax_h3_image/minimax-h3-image-txt2img.dgl
new file mode 100644
index 0000000..3e338d8
--- /dev/null
+++ b/docs/dgl/minimax_h3_image/minimax-h3-image-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/minimax_h3_image.md — Text-to-image
+
+var {
+ model_path = "../models/diffusion_models/minimax_h3_image_pruned_bf16.safetensors"
+ text_path = "../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ vae_path = "../models/vae/minimax_h3_image_vae_fp16.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "A silver tabby kitten holding a sign reading 'sd.cpp', natural daylight"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 1024
+ height = 1024
+
+ method = "euler"
+ custom_sigmas = [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, 0.23200887441635132, 0.0]
+}
+
+fn main() {
+ model = load_dit(model_path, family = "minimax_h3_image", flash_attn_mode = "on", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "minimax_h3_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "minimax_h3_image", params_backend = "cpu")
+
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ base = tensor::unsqueeze(image_base, axis = 2)
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, video_sigma_shift = 1)
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent), tiling = tiling())
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/minit2i/minit2i-cuda-txt2img.dgl b/docs/dgl/minit2i/minit2i-cuda-txt2img.dgl
new file mode 100644
index 0000000..a218b85
--- /dev/null
+++ b/docs/dgl/minit2i/minit2i-cuda-txt2img.dgl
@@ -0,0 +1,41 @@
+# version: 0.0.0
+# docs/models/minit2i.md — CUDA
+
+var {
+ model_path = "../models/diffusion_models/minit2i/diffusion_pytorch_model.safetensors"
+ text_path = "../models/text_encoders/flan-t5-large/model.safetensors"
+
+ output_path = "minit2i_cuda.png"
+
+ prompt = "a cat"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 100
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "minit2i")
+ text = load_text_encoder(text_path, family = "minit2i", t5_path = text_path)
+ base = tensor::zeros([width, height, 3, 1])
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = zero_out(positive, fields = ["vector"])
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ pixels = tensor::clamp(tensor::affine(latent, scale = 0.5, offset = 0.5), 0, 1)
+ image = image::to_u8(pixels)
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/minit2i/minit2i-metal-txt2img.dgl b/docs/dgl/minit2i/minit2i-metal-txt2img.dgl
new file mode 100644
index 0000000..4d6bc26
--- /dev/null
+++ b/docs/dgl/minit2i/minit2i-metal-txt2img.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# CLI/context: --threads 8
+# docs/models/minit2i.md — macOS with Metal
+
+var {
+ model_path = "../models/diffusion_models/minit2i/diffusion_pytorch_model.safetensors"
+ text_path = "../models/text_encoders/flan-t5-large/model.safetensors"
+
+ output_path = "minit2i_metal.png"
+
+ prompt = "a cat"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 512
+ height = 512
+
+ steps = 100
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, backend = "metal", family = "minit2i")
+ text = load_text_encoder(text_path, backend = "metal", family = "minit2i", t5_path = text_path)
+ base = tensor::zeros([width, height, 3, 1])
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = zero_out(positive, fields = ["vector"])
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ pixels = tensor::clamp(tensor::affine(latent, scale = 0.5, offset = 0.5), 0, 1)
+ image = image::to_u8(pixels)
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/moss_image/moss-image-base-txt2img.dgl b/docs/dgl/moss_image/moss-image-base-txt2img.dgl
new file mode 100644
index 0000000..9da3a8c
--- /dev/null
+++ b/docs/dgl/moss_image/moss-image-base-txt2img.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/moss_image.md — MOSS-Image Base
+
+var {
+ model_path = "../models/diffusion_models/moss_image_base_int8_convrot_attn.safetensors"
+ text_path = "../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_qwen3.5.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'moss-image.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "moss_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "moss_image", params_backend = "cpu", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "moss_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative, skip_layers = [22])
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/moss_image/moss-image-sft-inline-images.dgl b/docs/dgl/moss_image/moss-image-sft-inline-images.dgl
new file mode 100644
index 0000000..40dbe5d
--- /dev/null
+++ b/docs/dgl/moss_image/moss-image-sft-inline-images.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+
+var {
+ weights = "MOSS-Image-SFT-Q8_0.gguf"
+ encoder_path = "Qwen3.5-9B-Q4_K_M.gguf"
+ vision_path = "Qwen3.5-9B-mmproj-BF16.gguf"
+ tokenizer_path = "tokenizer_qwen3.5.json"
+ vae_path = "flux2_ae.safetensors"
+
+ image_path = "subject.png"
+ output_path = "moss-image-prompt.png"
+
+ prompt = "Draw the subject from <image:subject> in watercolor."
+ negative_prompt = ""
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 3
+}
+
+fn main() {
+ dit = load_dit(weights)
+ encoder = load_text_encoder(encoder_path, family_from = weights,
+ tokenizer = tokenizer_path, vision_path = vision_path)
+ vae = load_vae(vae_path, family_from = weights)
+ subject = image::load(image_path)
+ positive = encode_prompt(encoder, prompt,
+ image_bindings = {subject = subject}, width = width, height = height)
+ negative = encode_prompt(encoder, negative_prompt, width = width, height = height)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ rng = make_rng("cuda", seed = seed)
+ noise = tensor::randn_like(base, rng = rng)
+ positive_predictor = predictor(dit, positive)
+ negative_predictor = predictor(dit, negative, skip_layers = [22])
+ guided = cfg(positive_predictor, negative_predictor, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps)
+ start = add_noise(guided, base, noise, sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, rng = rng)
+ output = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, output)
+ return output
+}
diff --git a/docs/dgl/moss_image/moss-image-sft-txt2img.dgl b/docs/dgl/moss_image/moss-image-sft-txt2img.dgl
new file mode 100644
index 0000000..6ef4d7a
--- /dev/null
+++ b/docs/dgl/moss_image/moss-image-sft-txt2img.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/moss_image.md — MOSS-Image SFT
+
+var {
+ model_path = "../models/diffusion_models/moss_image_sft_int8_convrot_attn.safetensors"
+ text_path = "../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf"
+ tokenizer_path = "../models/tokenizers/tokenizer_qwen3.5.json"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'moss-image.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 3.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "moss_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "moss_image", params_backend = "cpu", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "moss_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative, skip_layers = [22])
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/ovis_image/ovis-image-txt2img.dgl b/docs/dgl/ovis_image/ovis-image-txt2img.dgl
new file mode 100644
index 0000000..9b844a5
--- /dev/null
+++ b/docs/dgl/ovis_image/ovis-image-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/ovis_image.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/ovis_image-Q4_0.gguf"
+ text_path = "../models/text_encoders/ovis_2.5.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "ovis_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "ovis_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "ovis_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/p_h3_image/p-h3-image-edit.dgl b/docs/dgl/p_h3_image/p-h3-image-edit.dgl
new file mode 100644
index 0000000..fdcc65b
--- /dev/null
+++ b/docs/dgl/p_h3_image/p-h3-image-edit.dgl
@@ -0,0 +1,71 @@
+# version: 0.0.0
+# docs/models/p_h3_image.md — Image editing
+
+var {
+ model_path = "../models/diffusion_models/p-h3-image_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/qwen3vl_4b_bf16.safetensors"
+ connectors_path = "../models/text_encoders/p-h3-image_text_connector.safetensors"
+ vae_path = "../models/vae/p-h3-image_vae_bf16.safetensors"
+
+ reference_0_path = "source.png"
+ reference_1_path = "reference.png"
+ output_path = "output.png"
+
+ prompt = "Edit <Picture 1>. Give the cat the scarf from <Picture 2>, preserving its face and the background."
+
+ seed = 42
+ rng_type = "cpu"
+ width = 2912
+ height = 2912
+
+ method = "euler"
+ custom_sigmas = [1.0, 0.9847530126571655, 0.9679049849510193, 0.9516879916191101, 0.9253720045089722, 0.8819230198860168, 0.8100829720497131, 0.5287179946899414, 0.0]
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ reference_1 = image::load(reference_1_path)
+
+ model = load_dit(model_path, family = "p_h3_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "p_h3_image", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "p_h3_image", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ image_shape = tensor::shape(image_base)
+ image_video = tensor::unsqueeze(image_base, axis = 2)
+ audio_channels = math::ceil_div(128, image_shape[0] * image_shape[1])
+ base = tensor::concat([image_video, tensor::zeros([image_shape[0], image_shape[1], 1, audio_channels, 1])], axis = 3)
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(width * height, tensor::shape(reference_0_original)[0] * tensor::shape(reference_0_original)[1]), 32, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+ reference_1_original = image::to_f32(reference_1, channels = 3)
+ reference_1_pixels = fit_reference_area(reference_1_original, math::min(width * height, tensor::shape(reference_1_original)[0] * tensor::shape(reference_1_original)[1]), 32, mode = "bilinear")
+ reference_latent_1 = vae_to_dit_latents(vae, vae_encode(vae, reference_1_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original, reference_1_original])
+ predict = predictor(model, condition, video_sigma_shift = 1, audio_length = 2, audio_sigma_shift = 1, reference_blocks = [{kind = "image", video = 0}, {kind = "image", video = 1}], ref_latents = [reference_latent_0, reference_latent_1], ref_index_mode = "increase")
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, :, :24]))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/p_h3_image/p-h3-image-txt2img.dgl b/docs/dgl/p_h3_image/p-h3-image-txt2img.dgl
new file mode 100644
index 0000000..e8090c3
--- /dev/null
+++ b/docs/dgl/p_h3_image/p-h3-image-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/p_h3_image.md — Text-to-image
+
+var {
+ model_path = "../models/diffusion_models/p-h3-image_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/qwen3vl_4b_bf16.safetensors"
+ connectors_path = "../models/text_encoders/p-h3-image_text_connector.safetensors"
+ vae_path = "../models/vae/p-h3-image_vae_bf16.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat holding a sign says 'p-h3-image.cpp'"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 2912
+ height = 2912
+
+ method = "euler"
+ custom_sigmas = [1.0, 0.9847530126571655, 0.9679049849510193, 0.9516879916191101, 0.9253720045089722, 0.8819230198860168, 0.8100829720497131, 0.5287179946899414, 0.0]
+}
+
+fn main() {
+ model = load_dit(model_path, family = "p_h3_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, connectors_path = connectors_path, family = "p_h3_image", params_backend = "cpu", projection_path = model_path)
+ vae = load_vae(vae_path, family = "p_h3_image", params_backend = "cpu")
+
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ image_shape = tensor::shape(image_base)
+ image_video = tensor::unsqueeze(image_base, axis = 2)
+ audio_channels = math::ceil_div(128, image_shape[0] * image_shape[1])
+ base = tensor::concat([image_video, tensor::zeros([image_shape[0], image_shape[1], 1, audio_channels, 1])], axis = 3)
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, video_sigma_shift = 1, audio_length = 2, audio_sigma_shift = 1)
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, :, :24]))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/photo_maker/photomaker-v1-sdxl.dgl b/docs/dgl/photo_maker/photomaker-v1-sdxl.dgl
new file mode 100644
index 0000000..88336dc
--- /dev/null
+++ b/docs/dgl/photo_maker/photomaker-v1-sdxl.dgl
@@ -0,0 +1,49 @@
+# version: 0.0.0
+# docs/models/photo_maker.md — PhotoMaker Version 1
+
+var {
+ weights = "../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors"
+ model_path = "../models/photomaker/photomaker-v1.safetensors"
+ vae_path = "../models/vae/sdxl_vae.safetensors"
+
+ image_dir = "../inputs/photomaker_examples/scarletthead_woman"
+ output_path = "output.png"
+
+ prompt = "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed"
+ negative_prompt = "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text"
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ steps = 50
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+ style_strength = 10.0
+}
+
+fn main() {
+ base_unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(vae_path, conv_scale = 0.03125, backend = "cpu")
+ photomaker = photomaker::load(model_path, family = "sdxl")
+ photomaker_lora = load_lora(model_path)
+ unet = apply_lora(base_unet, photomaker_lora)
+ tokens = photomaker::encode_images(photomaker, image::load_dir(image_dir))
+ conditions = photomaker::encode_prompt(text, prompt, tokens, width = width, height = height)
+ identity = photomaker::apply(conditions.condition, photomaker, tokens, conditions.mask)
+ plain_predictor = predictor(unet, conditions.plain)
+ identity_predictor = predictor(unet, identity)
+ pos = switch_predictor(plain_predictor, identity_predictor, after_steps = math::floor(steps * style_strength / 100))
+ neg = predictor(unet, encode_prompt(text, negative_prompt, width = width, height = height))
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent, identity = identity }
+}
diff --git a/docs/dgl/photo_maker/photomaker-v2-sdxl.dgl b/docs/dgl/photo_maker/photomaker-v2-sdxl.dgl
new file mode 100644
index 0000000..73a4599
--- /dev/null
+++ b/docs/dgl/photo_maker/photomaker-v2-sdxl.dgl
@@ -0,0 +1,52 @@
+# version: 0.0.0
+# docs/models/photo_maker.md — PhotoMaker Version 2
+
+var {
+ weights = "../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors"
+ model_path = "../models/photomaker/photomaker-v2.safetensors"
+ vae_path = "../models/vae/sdxl_vae.safetensors"
+
+ image_dir = "../inputs/photomaker_examples/scarletthead_woman"
+ embedding_path = "../inputs/photomaker_examples/scarletthead_woman/id_embeds.npy"
+ output_path = "output.png"
+
+ prompt = "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed"
+ negative_prompt = "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text"
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ steps = 50
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+ style_strength = 10.0
+}
+
+fn main() {
+ base_unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(vae_path, conv_scale = 0.03125, backend = "cpu")
+ photomaker = photomaker::load(model_path, family = "sdxl")
+ images = image::load_dir(image_dir)
+ identity = tensor::load(embedding_path)
+ photomaker_lora = load_lora(model_path)
+ unet = apply_lora(base_unet, photomaker_lora)
+ tokens = photomaker::encode_images(photomaker, images, id_embedding = identity)
+ conditions = photomaker::encode_prompt(text, prompt, tokens, width = width, height = height)
+ fused = photomaker::apply(conditions.condition, photomaker, tokens, conditions.mask)
+ plain_predictor = predictor(unet, conditions.plain)
+ identity_predictor = predictor(unet, fused)
+ pos = switch_predictor(plain_predictor, identity_predictor, after_steps = math::floor(steps * style_strength / 100))
+ neg = predictor(unet, encode_prompt(text, negative_prompt, width = width, height = height))
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, output)
+ return output
+}
diff --git a/docs/dgl/pid/pid-flux1-512-to-2048-txt2img.dgl b/docs/dgl/pid/pid-flux1-512-to-2048-txt2img.dgl
new file mode 100644
index 0000000..2718743
--- /dev/null
+++ b/docs/dgl/pid/pid-flux1-512-to-2048-txt2img.dgl
@@ -0,0 +1,47 @@
+# version: 0.0.0
+# docs/models/pid.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/pid_flux1_512_to_2048_4step_bf16.safetensors"
+ text_path = "../models/text_encoders/gemma_2_2b_it_elm_bf16.safetensors"
+ tokenizer_path = "../models/tokenizers/tokenizer_gemma2.json"
+ vae_path = "../models/vae/ae.sft"
+
+ reference_0_path = "../inputs/ernie_image/turbo_example.png"
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+
+ seed = 42
+ rng_type = "cpu"
+ width = 2048
+ height = 2048
+
+ steps = 4
+ method = "lcm"
+ scheduler = "lcm"
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+
+ model = load_dit(model_path, family = "pid")
+ text = load_text_encoder(text_path, family = "pid", tokenizer = tokenizer_path)
+ vae = load_vae(vae_path, family = "pid", vae_format = "flux")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros([width, height, 3, 1])
+ reference_0_pixels = image::to_f32(reference_0, channels = 3)
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition, ref_latents = [reference_latent_0])
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ pixels = tensor::clamp(tensor::affine(latent, scale = 0.5, offset = 0.5), 0, 1)
+ image = image::to_u8(pixels)
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl b/docs/dgl/pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl
new file mode 100644
index 0000000..63796d0
--- /dev/null
+++ b/docs/dgl/pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/pixart.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/pixart_sigma_xl2_1024_ms.safetensors"
+ text_path = "../models/text_encoders/t5xxl.safetensors"
+ vae_path = "../models/vae/pixart_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 4.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "pixart")
+ text = load_text_encoder(text_path, family = "pixart", t5_path = text_path)
+ vae = load_vae(vae_path, family = "pixart")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/pulid/pulid-flux1-schnell.dgl b/docs/dgl/pulid/pulid-flux1-schnell.dgl
new file mode 100644
index 0000000..2e9d949
--- /dev/null
+++ b/docs/dgl/pulid/pulid-flux1-schnell.dgl
@@ -0,0 +1,41 @@
+# version: 0.0.0
+# docs/models/pulid.md — CLI
+
+var {
+ dit_path = "../models/diffusion_models/flux1-schnell-Q4_K_S.gguf"
+ pulid_path = "../models/pulid/pulid_flux_v0.9.1.safetensors"
+ encoder_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ vae_path = "../models/vae/ae.safetensors"
+
+ identity_path = "../inputs/source.pulidembd"
+ output_path = "out.png"
+
+ prompt = "candid photograph of a young woman on a beach at sunset"
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 4
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ pulid = pulid::load(pulid_path)
+ text = load_text_encoder(encoder_path, family = "flux", clip_l_path = clip_l_path, backend = "cpu")
+ vae = load_vae(vae_path, family = "flux")
+ identity = pulid::load_embedding(identity_path)
+ condition = pulid::apply(encode_prompt(text, prompt), pulid, identity, strength = 1.0)
+ predict = predictor(dit, condition)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ sigmas = make_sigmas(predict, latent = base, scheduler = scheduler, steps = steps)
+ rng = make_rng("cuda", seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent }
+}
diff --git a/docs/dgl/qwen_image/qwen-image-txt2img.dgl b/docs/dgl/qwen_image/qwen-image-txt2img.dgl
new file mode 100644
index 0000000..6cafcb0
--- /dev/null
+++ b/docs/dgl/qwen_image/qwen-image-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/qwen_image.md — CLI
+
+var {
+ model_path = "../models/diffusion_models/qwen-image-Q8_0.gguf"
+ text_path = "../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf"
+ vae_path = "../models/vae/qwen_image_vae.safetensors"
+
+ output_path = "output.png"
+
+ negative_prompt = ""
+ prompt = "一个穿着\"QWEN\"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线: 探索视觉生成基础模型的极限,开创理解与生成一体化的未来。二、Qwen-Image的模型特色:1、复杂文字渲染。支持中英渲染、自动布局; 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景:赋能专业内容创作、助力生成式AI发展。”"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 2.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "qwen_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "qwen_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl b/docs/dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl
new file mode 100644
index 0000000..7bf6162
--- /dev/null
+++ b/docs/dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl
@@ -0,0 +1,68 @@
+# version: 0.0.0
+# docs/models/qwen_image_2.1.md — Image editing
+
+var {
+ model_path = "../models/diffusion_models/qwen_image_2.1_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf"
+ vision_path = "../models/text_encoders/Qwen3VL-8B-Instruct-mmproj-BF16.gguf"
+ vae_path = "../models/vae/qwen_image_2.1_vae_bf16.safetensors"
+
+ reference_0_path = "../inputs/qwen/qwen_image_2.1.png"
+ output_path = "qwen_image_2.1_edit.png"
+
+ prompt = "change 'qwen2.1.cpp' to 'sd.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "qwen_image_2_1", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image_2_1", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "qwen_image_2_1", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 4)
+ reference_0_pixels = fit_reference_area(reference_0_original, width * height, 32, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_pixels])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_pixels])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl b/docs/dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl
new file mode 100644
index 0000000..169ba1c
--- /dev/null
+++ b/docs/dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/qwen_image_2.1.md — Text to image
+
+var {
+ model_path = "../models/diffusion_models/qwen_image_2.1_int8_convrot.safetensors"
+ text_path = "../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf"
+ vae_path = "../models/vae/qwen_image_2.1_vae_bf16.safetensors"
+
+ output_path = "qwen_image_2.1.png"
+
+ prompt = "a lovely cat holding a sign says 'qwen2.1.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "flux"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "qwen_image_2_1", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image_2_1", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "qwen_image_2_1", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/qwen_image_edit/qwen-image-edit-2509.dgl b/docs/dgl/qwen_image_edit/qwen-image-edit-2509.dgl
new file mode 100644
index 0000000..99876a3
--- /dev/null
+++ b/docs/dgl/qwen_image_edit/qwen-image-edit-2509.dgl
@@ -0,0 +1,68 @@
+# version: 0.0.0
+# docs/models/qwen_image_edit.md — Qwen Image Edit 2509
+
+var {
+ model_path = "../models/diffusion_models/Qwen-Image-Edit-2509-Q4_K_S.gguf"
+ text_path = "../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf"
+ vision_path = "../models/text_encoders/Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf"
+ vae_path = "../models/vae/qwen_image_vae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'Qwen Image Edit 2509'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 2.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "qwen_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image", params_backend = "cpu", vision_path = vision_path)
+ vae = load_vae(vae_path, family = "qwen_image", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 32, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_original])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/qwen_image_edit/qwen-image-edit-2511.dgl b/docs/dgl/qwen_image_edit/qwen-image-edit-2511.dgl
new file mode 100644
index 0000000..426372e
--- /dev/null
+++ b/docs/dgl/qwen_image_edit/qwen-image-edit-2511.dgl
@@ -0,0 +1,67 @@
+# version: 0.0.0
+# docs/models/qwen_image_edit.md — Qwen Image Edit 2511
+
+var {
+ model_path = "../models/diffusion_models/qwen-image-edit-2511-Q4_K_M.gguf"
+ text_path = "../models/text_encoders/qwen_2.5_vl_7b.safetensors"
+ vae_path = "../models/vae/qwen_image_vae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'edit.cpp'"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 2.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "qwen_image", model_args = "qwen_image_zero_cond_t=true", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image", model_args = "qwen_image_zero_cond_t=true", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "qwen_image", model_args = "qwen_image_zero_cond_t=true", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 32, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_original])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/qwen_image_edit/qwen-image-edit.dgl b/docs/dgl/qwen_image_edit/qwen-image-edit.dgl
new file mode 100644
index 0000000..f25aa37
--- /dev/null
+++ b/docs/dgl/qwen_image_edit/qwen-image-edit.dgl
@@ -0,0 +1,67 @@
+# version: 0.0.0
+# docs/models/qwen_image_edit.md — Qwen Image Edit
+
+var {
+ model_path = "../models/diffusion_models/Qwen_Image_Edit-Q8_0.gguf"
+ text_path = "../models/text_encoders/qwen_2.5_vl_7b.safetensors"
+ vae_path = "../models/vae/qwen_image_vae.safetensors"
+
+ reference_0_path = "../inputs/flux/flux1-dev-q8_0.png"
+ output_path = "output.png"
+
+ prompt = "change 'flux.cpp' to 'edit.cpp'"
+ negative_prompt = ""
+
+ seed = 1118877715456453
+ rng_type = "cuda"
+
+ steps = 20
+ cfg_scale = 2.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn sqrt_step(estimate, index, value) {
+ return (estimate + value / estimate) / 2
+}
+
+fn positive_sqrt(value) {
+ return fold(tensor::to_list(tensor::arange(0, 32)), value + 1.0, process = sqrt_step, value = value)
+}
+
+fn fit_reference_area(input, area, factor, mode) {
+ shape = tensor::shape(input)
+ width = positive_sqrt(area * shape[0] / shape[1])
+ rounded = [math::round(width / factor) * factor, math::round(width * shape[1] / shape[0] / factor) * factor]
+ size = tensor::maximum(tensor::vector(rounded, dtype = "i64"), tensor::full([2], factor, dtype = "i64"))
+ return tensor::interpolate(input, size = size, mode = mode)
+}
+
+fn main() {
+ reference_0 = image::load(reference_0_path)
+ width = tensor::shape(reference_0)[1]
+ height = tensor::shape(reference_0)[2]
+
+ model = load_dit(model_path, family = "qwen_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "qwen_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "qwen_image", params_backend = "cpu")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ reference_0_original = image::to_f32(reference_0, channels = 3)
+ reference_0_pixels = fit_reference_area(reference_0_original, math::min(1048576, width * height), 32, mode = "bilinear")
+ reference_latent_0 = vae_to_dit_latents(vae, vae_encode(vae, reference_0_pixels, rng = encode_rng))
+
+ positive = encode_prompt(text, prompt, width = width, height = height, media = [reference_0_original])
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, media = [reference_0_original])
+ pos = predictor(model, positive, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ neg = predictor(model, negative, ref_latents = [reference_latent_0], ref_index_mode = "increase")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/relay_vsr/relayvsr-x2-image-upscale.dgl b/docs/dgl/relay_vsr/relayvsr-x2-image-upscale.dgl
new file mode 100644
index 0000000..cc790d1
--- /dev/null
+++ b/docs/dgl/relay_vsr/relayvsr-x2-image-upscale.dgl
@@ -0,0 +1,47 @@
+# version: 0.0.0
+# docs/models/relay_vsr.md — CLI
+
+var {
+ dit_path = "../models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors"
+ vae_path = "../models/vae/relayvsr_flash_vae_x2_fp32.safetensors"
+
+ input_path = "input.png"
+ output_path = "output.png"
+
+ seed = 42
+ window_size = 16
+
+ sampler = "euler"
+}
+
+fn keyframe_indices(frames, stride) {
+ count = tensor::shape(frames)[3]
+ last = count - 1
+ regular = tensor::arange(0, last, step = stride)
+ return tensor::concat([regular, tensor::vector([last], dtype = "i64")], axis = 0)
+}
+
+fn upscale_window(frames, dit, flash, rng, window_size, sampler) {
+ indices = keyframe_indices(frames, stride = window_size - 1)
+ keyframes = tensor::index_select(frames, axis = 3, indices = indices)
+ positive = { crossattn = relayvsr::positive_embedding(dit), lq_frames = keyframes }
+ base = tensor::zeros(relayvsr::latent_shape(dit, keyframes))
+ noise = tensor::randn_like(base, rng = rng)
+ sigmas = tensor::vector([1.0, 0.0])
+ positive_predictor = predictor(dit, positive)
+ latent = sample(noise, sigmas, predict = positive_predictor, method = sampler, rng = rng)
+ return relayvsr::decode(flash, latent, frames, keyframe_indices = indices)
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ flash = load_vae(vae_path)
+ image = image::load(input_path)
+ frames = tensor::unsqueeze(image, axis = 3)
+ rng = make_rng("cuda", seed = seed)
+ windows = video::split_windows(frames, size = window_size, overlap = 1)
+ enhanced = map(windows, process = upscale_window, dit = dit, flash = flash, rng = rng, window_size = window_size, sampler = sampler)
+ pixels = video::join_windows(enhanced, overlap = 1)
+ image::save(output_path, pixels[:, :, :, 0])
+ return { video = pixels }
+}
diff --git a/docs/dgl/relay_vsr/relayvsr-x2-video-upscale.dgl b/docs/dgl/relay_vsr/relayvsr-x2-video-upscale.dgl
new file mode 100644
index 0000000..c678a13
--- /dev/null
+++ b/docs/dgl/relay_vsr/relayvsr-x2-video-upscale.dgl
@@ -0,0 +1,48 @@
+# version: 0.0.0
+# docs/models/relay_vsr.md — CLI
+
+var {
+ dit_path = "../models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors"
+ vae_path = "../models/vae/relayvsr_flash_vae_x2_fp32.safetensors"
+
+ input_path = "input_frames"
+ output_path = "output.avi"
+
+ seed = 42
+ fps = 24
+ window_size = 16
+
+ sampler = "euler"
+}
+
+fn keyframe_indices(frames, stride) {
+ count = tensor::shape(frames)[3]
+ last = count - 1
+ regular = tensor::arange(0, last, step = stride)
+ return tensor::concat([regular, tensor::vector([last], dtype = "i64")], axis = 0)
+}
+
+fn upscale_window(frames, dit, flash, rng, window_size, sampler) {
+ indices = keyframe_indices(frames, stride = window_size - 1)
+ keyframes = tensor::index_select(frames, axis = 3, indices = indices)
+ positive = { crossattn = relayvsr::positive_embedding(dit), lq_frames = keyframes }
+ base = tensor::zeros(relayvsr::latent_shape(dit, keyframes))
+ noise = tensor::randn_like(base, rng = rng)
+ sigmas = tensor::vector([1.0, 0.0])
+ positive_predictor = predictor(dit, positive)
+ latent = sample(noise, sigmas, predict = positive_predictor, method = sampler, rng = rng)
+ return relayvsr::decode(flash, latent, frames, keyframe_indices = indices)
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ flash = load_vae(vae_path)
+ clip = video::load(input_path, fps_num = fps)
+ frames = clip.video.frames
+ rng = make_rng("cuda", seed = seed)
+ windows = video::split_windows(frames, size = window_size, overlap = 1)
+ enhanced = map(windows, process = upscale_window, dit = dit, flash = flash, rng = rng, window_size = window_size, sampler = sampler)
+ pixels = video::join_windows(enhanced, overlap = 1)
+ video::save(output_path, {frames = pixels, fps_num = clip.video.fps_num, fps_den = clip.video.fps_den}, audio = clip.audio)
+ return { video = pixels }
+}
diff --git a/docs/dgl/sd/nitrosd-realism-nitrofusion.dgl b/docs/dgl/sd/nitrosd-realism-nitrofusion.dgl
new file mode 100644
index 0000000..75ab934
--- /dev/null
+++ b/docs/dgl/sd/nitrosd-realism-nitrofusion.dgl
@@ -0,0 +1,46 @@
+# version: 0.0.0
+
+var {
+ model_path = "nitrosd-realism_f16.gguf"
+
+ output_path = "nitrofusion.png"
+
+ prompt = "cute cat"
+
+ seed = 2024
+ width = 1024
+ height = 1024
+
+ steps = 1
+ timestep_limit = 250
+ scheduler = "sgm_uniform"
+}
+
+fn nitrofusion_step(step, options) {
+ time_scale = options.timestep_limit / 1000
+ timestep = math::min(math::round(step.timestep * time_scale), 999)
+ t = timestep / 999
+ sigma = t_to_sigma(options.predict, t)
+ scales = denoiser_scalings(options.predict, sigma)
+ skip = scales.skip_scale * step.input_scale / scales.input_scale
+ return { timestep = timestep, skip_scale = skip, output_scale = scales.output_scale }
+}
+
+fn main() {
+ unet = load_unet(model_path)
+ text = load_text_encoder(model_path)
+ vae = load_vae(model_path, conv_scale = 0.03125)
+ shape = latent_shape(vae, width = width, height = height)
+ base = tensor::zeros(shape)
+ rng = make_rng("cuda", seed = seed)
+ pos_text = encode_prompt(text, prompt, width = width, height = height)
+ sampling_predict = predictor(unet, pos_text)
+ sigmas = make_sigmas(sampling_predict, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(sampling_predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ options = { predict = sampling_predict, timestep_limit = timestep_limit }
+ predict = predictor(unet, pos_text, transform = nitrofusion_step, transform_args = options)
+ latent = sample(start, sigmas, predict = predict, method = "euler_a", rng = rng)
+ pixels = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, pixels)
+ return { image = pixels, latent = latent }
+}
diff --git a/docs/dgl/sd/sd1-4-img2img.dgl b/docs/dgl/sd/sd1-4-img2img.dgl
new file mode 100644
index 0000000..b6d218b
--- /dev/null
+++ b/docs/dgl/sd/sd1-4-img2img.dgl
@@ -0,0 +1,63 @@
+# version: 0.0.0
+# docs/models/sd.md — img2img example
+
+var {
+ model_path = "../models/checkpoints/sd-v1-4.ckpt"
+
+ input_image_path = "./output.png"
+ output_path = "img2img_output.png"
+
+ prompt = "cat with blue eyes"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+
+ steps = 20
+ strength = 0.4000000059604645
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn slice_sigmas_by_strength(sigmas, strength) {
+ steps = tensor::numel(sigmas) - 1
+ retained = math::min(math::floor(steps * strength) + 1, steps)
+ return sigmas[steps - retained:]
+}
+
+fn main() {
+ input_image = image::load(input_image_path)
+ width = tensor::shape(input_image)[1]
+ height = tensor::shape(input_image)[2]
+
+ model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ encode_rng = make_rng(rng_type, seed = seed)
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+ base = vae_to_unet_latents(vae, vae_encode(vae, image::to_f32(source, channels = 3), rng = encode_rng))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ full_sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sd/sd1-4-txt2img.dgl b/docs/dgl/sd/sd1-4-txt2img.dgl
new file mode 100644
index 0000000..bd74c2a
--- /dev/null
+++ b/docs/dgl/sd/sd1-4-txt2img.dgl
@@ -0,0 +1,41 @@
+# version: 0.0.0
+# docs/models/sd.md — SD 1.4
+
+var {
+ model_path = "../models/checkpoints/sd-v1-4.ckpt"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sd/sd1-5-controlnet.dgl b/docs/dgl/sd/sd1-5-controlnet.dgl
new file mode 100644
index 0000000..a465cd0
--- /dev/null
+++ b/docs/dgl/sd/sd1-5-controlnet.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+
+var {
+ weights = "sd15.safetensors"
+ model_path = "control_openpose-fp16.safetensors"
+
+ input_path = "pose.png"
+ output_path = "controlnet.png"
+
+ prompt = "A dancer in a bright studio"
+ negative_prompt = ""
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7
+ scheduler = "karras"
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ control = controlnet::load(model_path, family = "sd1")
+ pose = tensor::interpolate(image::load(input_path), size = [width, height], axes = [1, 2])
+ pixels = image::to_f32(pose, channels = 3)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ rng = make_rng("cuda", seed = seed)
+ positive = controlnet::apply(encode_prompt(text, prompt), control, pixels)
+ negative = controlnet::apply(encode_prompt(text, negative_prompt), control, pixels)
+ pos = predictor(unet, positive, prediction = "eps")
+ neg = predictor(unet, negative, prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent }
+}
diff --git a/docs/dgl/sd/sd1-5-inpainting.dgl b/docs/dgl/sd/sd1-5-inpainting.dgl
new file mode 100644
index 0000000..bb91ad6
--- /dev/null
+++ b/docs/dgl/sd/sd1-5-inpainting.dgl
@@ -0,0 +1,62 @@
+# version: 0.0.0
+
+var {
+ weights = "sd-v1-5-inpainting.safetensors"
+
+ input_path = "input.png"
+ mask_path = "mask.png"
+ output_path = "inpaint.png"
+
+ prompt = "A red flower in the masked area"
+ negative_prompt = ""
+
+ seed = 42
+
+ strength = 0.8
+ steps = 20
+ cfg_scale = 7
+ scheduler = "karras"
+}
+
+fn latent_mask(mask, like) {
+ shape = tensor::shape(like)
+ w = shape[0]
+ h = shape[1]
+ batch = shape[3]
+ resized = tensor::interpolate(image::to_f32(mask), size = [w, h], mode = "nearest")
+ return tensor::broadcast_to(resized, [w, h, 1, batch])
+}
+
+fn slice_sigmas_by_strength(sigmas, strength) {
+ steps = tensor::numel(sigmas) - 1
+ retained = math::floor(steps * strength)
+ return sigmas[steps - retained:]
+}
+
+fn main() {
+ unet = load_unet(weights)
+ text = load_text_encoder(weights)
+ vae = load_vae(weights)
+ source = tensor::interpolate(image::load(input_path), size = [512, 512], axes = [1, 2])
+ gray = tensor::grayscale(image::load(mask_path), axis = 0)
+ binary_mask = tensor::threshold(gray, threshold = 128, low = 0, high = 255)
+ region = tensor::interpolate(binary_mask, size = [512, 512], axes = [1, 2], mode = "nearest_exact")
+ pixels = image::to_f32(source)
+ masked = tensor::lerp(pixels, 0.5, image::to_f32(region))
+ encode_rng = make_rng("cuda", seed = seed)
+ base = vae_to_unet_latents(vae, vae_encode(vae, pixels, rng = encode_rng))
+ cond_image = vae_to_unet_latents(vae, vae_encode(vae, masked, rng = encode_rng))
+ region_latent = latent_mask(region, like = base)
+ condition = tensor::concat([region_latent, cond_image], axis = 2)
+ denoise_mask = tensor::max_pool2d(region_latent, kernel = [3, 3], padding_before = [1, 1])
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(unet, encode_prompt(text, prompt), concat = condition, prediction = "eps")
+ neg = predictor(unet, encode_prompt(text, negative_prompt), concat = condition, prediction = "eps")
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = slice_sigmas_by_strength(make_sigmas(guided, latent = base, scheduler = scheduler, steps = steps), strength = strength)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, base = base, denoise_mask = denoise_mask, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent, mask = denoise_mask }
+}
diff --git a/docs/dgl/sd/sd1-5-txt2img.dgl b/docs/dgl/sd/sd1-5-txt2img.dgl
new file mode 100644
index 0000000..5057b2a
--- /dev/null
+++ b/docs/dgl/sd/sd1-5-txt2img.dgl
@@ -0,0 +1,41 @@
+# version: 0.0.0
+# docs/models/sd.md — SD 1.5
+
+var {
+ model_path = "../models/checkpoints/v1-5-pruned-emaonly.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_vae(model_path, family = "sd1")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sd/sdxl-txt2img.dgl b/docs/dgl/sd/sdxl-txt2img.dgl
new file mode 100644
index 0000000..bb875a4
--- /dev/null
+++ b/docs/dgl/sd/sdxl-txt2img.dgl
@@ -0,0 +1,55 @@
+# version: 0.0.0
+# docs/models/sd.md — SDXL
+
+var {
+ model_path = "../models/checkpoints/sd_xl_base_1.0.safetensors"
+ vae_path = "../models/vae/sdxl_vae-fp16-fix.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+ zero_negative = true
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn sdxl_zero_text(cond) {
+ zeroed = zero_out(cond, fields = ["crossattn"])
+ pooled_zeros = tensor::zeros_like(cond.vector[:1280])
+ adm = tensor::replace_slice(cond.vector, pooled_zeros, axis = 0, start = 0)
+ return {crossattn = zeroed.crossattn, vector = adm}
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sdxl")
+ text = load_text_encoder(model_path, family = "sdxl")
+ vae = load_vae(vae_path, family = "sdxl", conv_scale = 0.03125)
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height)
+ negative = if zero_negative {
+ sdxl_zero_text(negative_text)
+ } else {
+ {crossattn = negative_text.crossattn, vector = negative_text.vector}
+ }
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sd3/sd3-5-large-txt2img.dgl b/docs/dgl/sd3/sd3-5-large-txt2img.dgl
new file mode 100644
index 0000000..b5594ae
--- /dev/null
+++ b/docs/dgl/sd3/sd3-5-large-txt2img.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/sd3.md — SD3.5 Large
+
+var {
+ model_path = "../models/checkpoints/sd3.5_large.safetensors"
+ clip_l_path = "../models/text_encoders/clip_l.safetensors"
+ clip_g_path = "../models/text_encoders/clip_g.safetensors"
+ t5_path = "../models/text_encoders/t5xxl_fp16.safetensors"
+
+ output_path = "output.png"
+ negative_prompt = ""
+ prompt = "a lovely cat holding a sign says \\\"Stable diffusion 3.5 Large\\\""
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 20
+ cfg_scale = 4.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "sd3")
+ text = load_text_encoder(model_path, backend = "cpu", clip_g_path = clip_g_path, clip_l_path = clip_l_path, family = "sd3", t5_path = t5_path)
+ vae = load_vae(model_path, family = "sd3")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/seedvr2/seedvr2-3b-image-upscale.dgl b/docs/dgl/seedvr2/seedvr2-3b-image-upscale.dgl
new file mode 100644
index 0000000..6d1d67d
--- /dev/null
+++ b/docs/dgl/seedvr2/seedvr2-3b-image-upscale.dgl
@@ -0,0 +1,65 @@
+# version: 0.0.0
+# docs/models/seedvr2.md — Upscale an image
+
+var {
+ dit_path = "../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors"
+ vae_path = "../models/vae/seedvr2_ema_vae_fp16.safetensors"
+
+ input_path = "input.png"
+ output_path = "restored.png"
+
+ seed = 42
+ window_size = 5
+ factor = 2
+}
+
+fn prepare_seedvr2_frames(frames, width, height, count) {
+ aligned_width = math::ceil_div(width, 16) * 16
+ aligned_height = math::ceil_div(height, 16) * 16
+ padded_count = math::ceil_div(count - 1, 4) * 4 + 1
+ planar = tensor::permute(tensor::cast(frames, "f32"), axes = [1, 2, 3, 0])
+ pixels = tensor::div(planar, 255.0)
+ scaled = tensor::clamp(tensor::interpolate(pixels, [width, height], mode = "bicubic"), min = 0, max = 1)
+ padded = tensor::pad(scaled, [0, aligned_width - width, 0, aligned_height - height])
+ tail = tensor::full([padded_count - count], count - 1, dtype = "i64")
+ indices = tensor::concat([tensor::arange(0, count), tail], axis = 0)
+ extended = tensor::index_select(padded, axis = 2, indices = indices)
+ return tensor::reshape(extended, [aligned_width, aligned_height, padded_count, 3, 1])
+}
+
+fn upscale_window(frames, dit, vae, embedding, factor, rng) {
+ shape = tensor::shape(frames)
+ width = shape[1] * factor
+ height = shape[2] * factor
+ count = shape[3]
+ pixels = prepare_seedvr2_frames(frames, width, height, count)
+ encoded = vae_encode(vae, pixels, rng = rng, posterior = "mode")
+ condition_latent = vae_to_dit_latents(vae, encoded)
+ encoded_shape = tensor::shape(encoded)
+ mask_shape = [encoded_shape[0], encoded_shape[1], encoded_shape[2], 1, encoded_shape[4]]
+ mask = tensor::full(mask_shape, 1)
+ condition = { crossattn = embedding, concat = tensor::concat([condition_latent, mask], axis = 3) }
+ noise = tensor::randn_like(condition_latent, rng = rng)
+ sigmas = tensor::vector([1.0, 0.0])
+ sampling_predictor = predictor(dit, condition)
+ latent = sample(noise, sigmas, predict = sampling_predictor, method = "euler", rng = rng)
+ decoded = vae_decode(vae, dit_to_vae_latents(vae, latent), dtype = "f32")
+ cropped = tensor::crop(decoded, axes = [0, 1, 2], start = [0, 0, 0], size = [width, height, count])
+ planar = tensor::reshape(cropped, [width, height, count, 3])
+ output = tensor::cast(tensor::mul(tensor::permute(planar, axes = [3, 0, 1, 2]), 255.0), "u8")
+ return output
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ vae = load_vae(vae_path)
+ image = image::load(input_path)
+ source_frames = tensor::unsqueeze(image, axis = 3)
+ rng = make_rng("cuda", seed = seed)
+ embedding = seedvr2::positive_embedding(dit)
+ windows = video::split_windows(source_frames, size = window_size)
+ upscaled = map(windows, process = upscale_window, dit = dit, vae = vae, embedding = embedding, factor = factor, rng = rng)
+ output = video::join_windows(upscaled)
+ image::save(output_path, output[:, :, :, 0])
+ return output
+}
diff --git a/docs/dgl/seedvr2/seedvr2-3b-video-upscale.dgl b/docs/dgl/seedvr2/seedvr2-3b-video-upscale.dgl
new file mode 100644
index 0000000..c5ca43c
--- /dev/null
+++ b/docs/dgl/seedvr2/seedvr2-3b-video-upscale.dgl
@@ -0,0 +1,65 @@
+# version: 0.0.0
+# docs/models/seedvr2.md — Upscale a video
+
+var {
+ dit_path = "../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors"
+ vae_path = "../models/vae/seedvr2_ema_vae_fp16.safetensors"
+
+ input_path = "frames"
+ output_path = "restored.avi"
+
+ seed = 42
+ fps = 24
+ window_size = 1
+ factor = 2
+}
+
+fn prepare_seedvr2_frames(frames, width, height, count) {
+ aligned_width = math::ceil_div(width, 16) * 16
+ aligned_height = math::ceil_div(height, 16) * 16
+ padded_count = math::ceil_div(count - 1, 4) * 4 + 1
+ planar = tensor::permute(tensor::cast(frames, "f32"), axes = [1, 2, 3, 0])
+ pixels = tensor::div(planar, 255.0)
+ scaled = tensor::clamp(tensor::interpolate(pixels, [width, height], mode = "bicubic"), min = 0, max = 1)
+ padded = tensor::pad(scaled, [0, aligned_width - width, 0, aligned_height - height])
+ tail = tensor::full([padded_count - count], count - 1, dtype = "i64")
+ indices = tensor::concat([tensor::arange(0, count), tail], axis = 0)
+ extended = tensor::index_select(padded, axis = 2, indices = indices)
+ return tensor::reshape(extended, [aligned_width, aligned_height, padded_count, 3, 1])
+}
+
+fn upscale_window(frames, dit, vae, embedding, factor, rng) {
+ shape = tensor::shape(frames)
+ width = shape[1] * factor
+ height = shape[2] * factor
+ count = shape[3]
+ pixels = prepare_seedvr2_frames(frames, width, height, count)
+ encoded = vae_encode(vae, pixels, rng = rng, posterior = "mode")
+ condition_latent = vae_to_dit_latents(vae, encoded)
+ encoded_shape = tensor::shape(encoded)
+ mask_shape = [encoded_shape[0], encoded_shape[1], encoded_shape[2], 1, encoded_shape[4]]
+ mask = tensor::full(mask_shape, 1)
+ condition = { crossattn = embedding, concat = tensor::concat([condition_latent, mask], axis = 3) }
+ noise = tensor::randn_like(condition_latent, rng = rng)
+ sigmas = tensor::vector([1.0, 0.0])
+ sampling_predictor = predictor(dit, condition)
+ latent = sample(noise, sigmas, predict = sampling_predictor, method = "euler", rng = rng)
+ decoded = vae_decode(vae, dit_to_vae_latents(vae, latent), dtype = "f32")
+ cropped = tensor::crop(decoded, axes = [0, 1, 2], start = [0, 0, 0], size = [width, height, count])
+ planar = tensor::reshape(cropped, [width, height, count, 3])
+ output = tensor::cast(tensor::mul(tensor::permute(planar, axes = [3, 0, 1, 2]), 255.0), "u8")
+ return output
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ vae = load_vae(vae_path)
+ clip = video::load(input_path, fps_num = fps)
+ rng = make_rng("cuda", seed = seed)
+ embedding = seedvr2::positive_embedding(dit)
+ windows = video::split_windows(clip.video.frames, size = window_size)
+ upscaled = map(windows, process = upscale_window, dit = dit, vae = vae, embedding = embedding, factor = factor, rng = rng)
+ output = video::join_windows(upscaled)
+ video::save(output_path, {frames = output, fps_num = clip.video.fps_num, fps_den = clip.video.fps_den}, audio = clip.audio)
+ return output
+}
diff --git a/docs/dgl/sefi_image/sefi-1b-base-txt2img.dgl b/docs/dgl/sefi_image/sefi-1b-base-txt2img.dgl
new file mode 100644
index 0000000..c1dc15e
--- /dev/null
+++ b/docs/dgl/sefi_image/sefi-1b-base-txt2img.dgl
@@ -0,0 +1,44 @@
+# version: 0.0.0
+# docs/models/sefi_image.md — 1B / 2B base
+
+var {
+ model_path = "../models/diffusion_models/sefi_1b_base.safetensors"
+ text_path = "../models/text_encoders/qwen3_vl_2b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "out.png"
+
+ prompt = "a photograph of an orange tabby cat sitting on a couch"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 50
+ cfg_scale = 4.0
+ method = "euler"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "sefi_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "sefi_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "sefi_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ dual_schedule = sefi::make_sigmas(steps = steps)
+ sigmas = dual_schedule.sigmas
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sefi::sample(start, sigmas, predict = guided, semantic_sigmas = dual_schedule.semantic_sigmas, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 16:144]))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sefi_image/sefi-1b-turbo-txt2img.dgl b/docs/dgl/sefi_image/sefi-1b-turbo-txt2img.dgl
new file mode 100644
index 0000000..c8bbde2
--- /dev/null
+++ b/docs/dgl/sefi_image/sefi-1b-turbo-txt2img.dgl
@@ -0,0 +1,39 @@
+# version: 0.0.0
+# docs/models/sefi_image.md — 1B / 2B turbo
+
+var {
+ model_path = "../models/diffusion_models/sefi_1b_turbo.safetensors"
+ text_path = "../models/text_encoders/qwen3_vl_2b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "out.png"
+
+ prompt = "a photograph of an orange tabby cat sitting on a couch"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 4
+ method = "euler"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "sefi_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "sefi_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "sefi_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ dual_schedule = sefi::make_sigmas(steps = steps)
+ sigmas = dual_schedule.sigmas
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sefi::sample(start, sigmas, predict = predict, semantic_sigmas = dual_schedule.semantic_sigmas, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 16:144]))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sefi_image/sefi-5b-turbo-txt2img.dgl b/docs/dgl/sefi_image/sefi-5b-turbo-txt2img.dgl
new file mode 100644
index 0000000..07571de
--- /dev/null
+++ b/docs/dgl/sefi_image/sefi-5b-turbo-txt2img.dgl
@@ -0,0 +1,40 @@
+# version: 0.0.0
+# CLI/context: --max-vram 8
+# docs/models/sefi_image.md — 5B (needs streaming on 12 GiB VRAM)
+
+var {
+ model_path = "../models/diffusion_models/sefi_5b_turbo.safetensors"
+ text_path = "../models/text_encoders/qwen3_vl_4b.safetensors"
+ vae_path = "../models/vae/flux2_ae.safetensors"
+
+ output_path = "out.png"
+
+ prompt = "a photograph of an orange tabby cat sitting on a couch"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 4
+ method = "euler"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "sefi_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "sefi_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "sefi_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ dual_schedule = sefi::make_sigmas(steps = steps)
+ sigmas = dual_schedule.sigmas
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sefi::sample(start, sigmas, predict = predict, semantic_sigmas = dual_schedule.semantic_sigmas, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 16:144]))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/sefi_image/sefi-image-5b-rl.dgl b/docs/dgl/sefi_image/sefi-image-5b-rl.dgl
new file mode 100644
index 0000000..08b69fc
--- /dev/null
+++ b/docs/dgl/sefi_image/sefi-image-5b-rl.dgl
@@ -0,0 +1,36 @@
+# version: 0.0.0
+
+var {
+ dit_path = "SeFi-Image-5B-RL.safetensors"
+ encoder_path = "Qwen3-VL-4B-Instruct.gguf"
+ vae_path = "flux2_ae.safetensors"
+
+ output_path = "sefi.png"
+
+ prompt = "A small house in a forest clearing, warm afternoon light"
+ negative_prompt = ""
+
+ seed = 42
+ width = 1024
+ height = 1024
+
+ cfg_scale = 4
+}
+
+fn main() {
+ dit = load_dit(dit_path)
+ text = load_text_encoder(encoder_path, family = "sefi_image")
+ vae = load_vae(vae_path, family = "sefi_image")
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ schedule = sefi::make_sigmas(steps = 20)
+ rng = make_rng("cuda", seed = seed)
+ pos = predictor(dit, encode_prompt(text, prompt))
+ neg = predictor(dit, encode_prompt(text, negative_prompt))
+ guided = cfg(pos, neg, scale = cfg_scale)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = schedule.sigmas[0])
+ latent = sefi::sample(start, schedule.sigmas, predict = guided, semantic_sigmas = schedule.semantic_sigmas, finish = true, rng = rng)
+ texture = latent[:, :, 16:144]
+ image = vae_decode(vae, dit_to_vae_latents(vae, texture))
+ image::save(output_path, image)
+ return { image = image, latent = latent }
+}
diff --git a/docs/dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl b/docs/dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl
new file mode 100644
index 0000000..bd26996
--- /dev/null
+++ b/docs/dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl
@@ -0,0 +1,41 @@
+# version: 0.0.0
+# docs/models/sensenova_u1.md — CUDA
+
+var {
+ model_path = "../models/checkpoints/SenseNova-U1.5-8B-MoT"
+
+ output_path = "output.png"
+
+ negative_prompt = "<|im_start|>user\n<|im_end|>\n<|im_start|>assistant\n<img>"
+ prompt = "a red cube on a white background"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 2048
+ height = 2048
+
+ steps = 50
+ cfg_scale = 4.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "sensenova_u1_5")
+ text = load_text_encoder(model_path, family = "sensenova_u1_5")
+ vae = load_vae(model_path, family = "sensenova_u1_5")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height, raw = true)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/taesd/sd1-5-taesd-preview.dgl b/docs/dgl/taesd/sd1-5-taesd-preview.dgl
new file mode 100644
index 0000000..ef09c17
--- /dev/null
+++ b/docs/dgl/taesd/sd1-5-taesd-preview.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/taesd.md — CLI
+
+var {
+ model_path = "../models/checkpoints/v1-5-pruned-emaonly.safetensors"
+ vae_path = "../models/vae/diffusion_pytorch_model.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 512
+
+ steps = 20
+ cfg_scale = 7.0
+ method = "euler_a"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_unet(model_path, family = "sd1")
+ text = load_text_encoder(model_path, family = "sd1")
+ vae = load_tae(vae_path, family = "sd1")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, observer = preview(decoder = vae))
+ image = vae_decode(vae, unet_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/wan/wan2-1-first-last-frame-14b.dgl b/docs/dgl/wan/wan2-1-first-last-frame-14b.dgl
new file mode 100644
index 0000000..e87960b
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-first-last-frame-14b.dgl
@@ -0,0 +1,78 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.1 FLF2V 14B
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-flf2v-14b-720p-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+ vision_path = "../models/clip_vision/clip_vision_h.safetensors"
+
+ first_frame_path = "../inputs/start_image.png"
+ last_frame_path = "../inputs/end_image.png"
+ output_path = "output.avi"
+
+ prompt = "glass flower blossom"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+ last_frame = image::load(last_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ condition_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ condition_first = tensor::replace_slice(condition_blank, first_frame_pixels, axis = 2, start = 0)
+ condition_pixels = tensor::replace_slice(condition_first, last_frame_pixels, axis = 2, start = frames - 1)
+ condition_latent = vae_to_dit_latents(vae, vae_encode(vae, condition_pixels, rng = rng))
+ condition_shape = tensor::shape(condition_latent)
+ mask_blank = tensor::zeros([condition_shape[0], condition_shape[1], condition_shape[2], 4, 1])
+ first_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+ last_mask = tensor::index_fill(first_mask[:, :, :, 3:4], axis = 2, indices = [condition_shape[2] - 1], value = 1)
+ image_mask = tensor::replace_slice(first_mask, last_mask, axis = 3, start = 3)
+ video_concat = tensor::concat([image_mask, condition_latent], axis = 3)
+ vision = load_vision_encoder(vision_path, family = "wan2", params_backend = "cpu")
+ vision_first = encode_vision(vision, image::to_u8(first_frame_pixels[:, :, 0]))
+ vision_end = encode_vision(vision, image::to_u8(last_frame_pixels[:, :, 0]))
+ vision_condition = tensor::concat([vision_first, vision_end], axis = 1)
+ positive_text = encode_prompt(text, prompt, width = width, height = height)
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height)
+ positive = {crossattn = positive_text.crossattn, vector = vision_condition}
+ negative = {crossattn = negative_text.crossattn, vector = vision_condition}
+ pos = predictor(model, positive, concat = video_concat)
+ neg = predictor(model, negative, concat = video_concat)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-i2v-14b.dgl b/docs/dgl/wan/wan2-1-i2v-14b.dgl
new file mode 100644
index 0000000..5b227a9
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-i2v-14b.dgl
@@ -0,0 +1,70 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.1 I2V 14B
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-i2v-14b-480p-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+ vision_path = "../models/clip_vision/clip_vision_h.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ condition_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ condition_pixels = tensor::replace_slice(condition_blank, first_frame_pixels, axis = 2, start = 0)
+ condition_latent = vae_to_dit_latents(vae, vae_encode(vae, condition_pixels, rng = rng))
+ condition_shape = tensor::shape(condition_latent)
+ mask_blank = tensor::zeros([condition_shape[0], condition_shape[1], condition_shape[2], 4, 1])
+ image_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+ video_concat = tensor::concat([image_mask, condition_latent], axis = 3)
+ vision = load_vision_encoder(vision_path, family = "wan2", params_backend = "cpu")
+ vision_condition = encode_vision(vision, image::to_u8(first_frame_pixels[:, :, 0]))
+ positive_text = encode_prompt(text, prompt, width = width, height = height)
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height)
+ positive = {crossattn = positive_text.crossattn, vector = vision_condition}
+ negative = {crossattn = negative_text.crossattn, vector = vision_condition}
+ pos = predictor(model, positive, concat = video_concat)
+ neg = predictor(model, negative, concat = video_concat)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-t2v-1-3b.dgl b/docs/dgl/wan/wan2-1-t2v-1-3b.dgl
new file mode 100644
index 0000000..756bcc6
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-t2v-1-3b.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.1 T2V 1.3B
+
+var {
+ model_path = "../models/diffusion_models/wan2.1_t2v_1.3B_fp16.safetensors"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2")
+ text = load_text_encoder(text_path, family = "wan2", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-t2v-14b.dgl b/docs/dgl/wan/wan2-1-t2v-14b.dgl
new file mode 100644
index 0000000..848790a
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-t2v-14b.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.1 T2V 14B
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-t2v-14b-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-1-3b-ref2video.dgl b/docs/dgl/wan/wan2-1-vace-1-3b-ref2video.dgl
new file mode 100644
index 0000000..b554924
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-1-3b-ref2video.dgl
@@ -0,0 +1,71 @@
+# version: 0.0.0
+# docs/models/wan.md — R2V
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-1-3b-txt2video.dgl b/docs/dgl/wan/wan2-1-vace-1-3b-txt2video.dgl
new file mode 100644
index 0000000..e9adb73
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-1-3b-txt2video.dgl
@@ -0,0 +1,53 @@
+# version: 0.0.0
+# docs/models/wan.md — T2V
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.png"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 1
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_context = tensor::concat([vace_video, vace_mask], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, output_frames[:, :, :, 0])
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-1-3b-video2video.dgl b/docs/dgl/wan/wan2-1-vace-1-3b-video2video.dgl
new file mode 100644
index 0000000..2032bf3
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-1-3b-video2video.dgl
@@ -0,0 +1,84 @@
+# version: 0.0.0
+# docs/models/wan.md — V2V
+
+var {
+ model_path = "../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/dance_girl.jpg"
+ control_video_path = "./post+depth"
+ output_path = "output.avi"
+
+ prompt = "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere."
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn control_frame_pixels(index, args) {
+ pixels = image::to_f32(fit_canvas(args.frames[:, :, :, index], args.width, args.height, mode = "nearest"), channels = 3)
+ return tensor::unsqueeze(pixels, axis = 2)
+}
+
+fn main() {
+ control_video_media = video::load(control_video_path, max_frames = frames, fps_num = fps, audio = false)
+ control_video = control_video_media.video.frames
+ control_count = tensor::shape(control_video)[3]
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ control_indices = tensor::to_list(tensor::arange(0, control_count))
+ control_parts = map(control_indices, process = control_frame_pixels, args = {frames = control_video, width = width, height = height})
+ control_short = tensor::concat(control_parts, axis = 2)
+ control_pixels = tensor::pad(control_short, [0, 0, 0, 0, 0, frames - tensor::shape(control_short)[2]], value = 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_pixels, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-14b-ref2video.dgl b/docs/dgl/wan/wan2-1-vace-14b-ref2video.dgl
new file mode 100644
index 0000000..392cd97
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-14b-ref2video.dgl
@@ -0,0 +1,71 @@
+# version: 0.0.0
+# docs/models/wan.md — R2V
+
+var {
+ model_path = "../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-14b-txt2video.dgl b/docs/dgl/wan/wan2-1-vace-14b-txt2video.dgl
new file mode 100644
index 0000000..3bc41ad
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-14b-txt2video.dgl
@@ -0,0 +1,53 @@
+# version: 0.0.0
+# docs/models/wan.md — T2V
+
+var {
+ model_path = "../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_context = tensor::concat([vace_video, vace_mask], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-1-vace-14b-video2video.dgl b/docs/dgl/wan/wan2-1-vace-14b-video2video.dgl
new file mode 100644
index 0000000..372b230
--- /dev/null
+++ b/docs/dgl/wan/wan2-1-vace-14b-video2video.dgl
@@ -0,0 +1,84 @@
+# version: 0.0.0
+# docs/models/wan.md — V2V
+
+var {
+ model_path = "../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/dance_girl.jpg"
+ control_video_path = "./post+depth"
+ output_path = "output.avi"
+
+ prompt = "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere."
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn control_frame_pixels(index, args) {
+ pixels = image::to_f32(fit_canvas(args.frames[:, :, :, index], args.width, args.height, mode = "nearest"), channels = 3)
+ return tensor::unsqueeze(pixels, axis = 2)
+}
+
+fn main() {
+ control_video_media = video::load(control_video_path, max_frames = frames, fps_num = fps, audio = false)
+ control_video = control_video_media.video.frames
+ control_count = tensor::shape(control_video)[3]
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ control_indices = tensor::to_list(tensor::arange(0, control_count))
+ control_parts = map(control_indices, process = control_frame_pixels, args = {frames = control_video, width = width, height = height})
+ control_short = tensor::concat(control_parts, axis = 2)
+ control_pixels = tensor::pad(control_short, [0, 0, 0, 0, 0, frames - tensor::shape(control_short)[2]], value = 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_pixels, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-first-last-frame-a14b.dgl b/docs/dgl/wan/wan2-2-first-last-frame-a14b.dgl
new file mode 100644
index 0000000..3c41d8d
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-first-last-frame-a14b.dgl
@@ -0,0 +1,80 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 FLF2V 14B
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/start_image.png"
+ last_frame_path = "../inputs/end_image.png"
+ output_path = "output.avi"
+
+ prompt = "glass flower blossom"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 18
+ high_noise_steps = 8
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+ last_frame = image::load(last_frame_path)
+
+ model = load_dit(model_path, family = "wan2_2_i2v", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2_2_i2v", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2_2_i2v", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ condition_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ condition_first = tensor::replace_slice(condition_blank, first_frame_pixels, axis = 2, start = 0)
+ condition_pixels = tensor::replace_slice(condition_first, last_frame_pixels, axis = 2, start = frames - 1)
+ condition_latent = vae_to_dit_latents(vae, vae_encode(vae, condition_pixels, rng = rng))
+ condition_shape = tensor::shape(condition_latent)
+ mask_blank = tensor::zeros([condition_shape[0], condition_shape[1], condition_shape[2], 4, 1])
+ first_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+ last_mask = tensor::index_fill(first_mask[:, :, :, 3:4], axis = 2, indices = [condition_shape[2] - 1], value = 1)
+ image_mask = tensor::replace_slice(first_mask, last_mask, axis = 3, start = 3)
+ video_concat = tensor::concat([image_mask, condition_latent], axis = 3)
+ high_base = load_dit(high_model_path, family = "wan2_2_i2v", params_backend = "cpu")
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, concat = video_concat)
+ neg = predictor(model, negative, concat = video_concat)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive, concat = video_concat)
+ high_neg = predictor(high_base, negative, concat = video_concat)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-i2v-a14b.dgl b/docs/dgl/wan/wan2-2-i2v-a14b.dgl
new file mode 100644
index 0000000..fb85a76
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-i2v-a14b.dgl
@@ -0,0 +1,74 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 I2V A14B
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 18
+ high_noise_steps = 8
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2_2_i2v", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2_2_i2v", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2_2_i2v", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ condition_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ condition_pixels = tensor::replace_slice(condition_blank, first_frame_pixels, axis = 2, start = 0)
+ condition_latent = vae_to_dit_latents(vae, vae_encode(vae, condition_pixels, rng = rng))
+ condition_shape = tensor::shape(condition_latent)
+ mask_blank = tensor::zeros([condition_shape[0], condition_shape[1], condition_shape[2], 4, 1])
+ image_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+ video_concat = tensor::concat([image_mask, condition_latent], axis = 3)
+ high_base = load_dit(high_model_path, family = "wan2_2_i2v", params_backend = "cpu")
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, concat = video_concat)
+ neg = predictor(model, negative, concat = video_concat)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive, concat = video_concat)
+ high_neg = predictor(high_base, negative, concat = video_concat)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-s2v-14b.dgl b/docs/dgl/wan/wan2-2-s2v-14b.dgl
new file mode 100644
index 0000000..9a1c004
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-s2v-14b.dgl
@@ -0,0 +1,70 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 S2V 14B
+
+var {
+ model_path = "../models/diffusion_models/wan2.2_s2v-14B-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+ audio_encoder_path = "../models/text_encoders/audio_encoders/wav2vec2_large_english_fp16.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ driving_audio_path = "./input/speech.wav"
+ output_path = "output.avi"
+
+ prompt = "a person is talking"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 81
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ driving_audio_media = audio::load(driving_audio_path)
+ sample_rate = driving_audio_media.sample_rate
+ driving_audio = driving_audio_media.samples
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2_2_s2v", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2_2_s2v", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2_2_s2v", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ reference_video = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng, tiling = tiling()))
+ reference_shape = tensor::shape(reference_video)
+ reference_latent = tensor::reshape(reference_video, reference_shape[:4])
+ audio_encoder = load_audio_encoder(audio_encoder_path, family = "wan2_2_s2v", params_backend = "cpu")
+ speech = encode_audio(audio_encoder, driving_audio, sample_rate = sample_rate)
+ audio_embedding = wan::s2v_audio_window(speech, latent_frames = tensor::shape(base)[2])
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, ref_latents = [reference_latent], audio_embedding = audio_embedding)
+ neg = predictor(model, negative, ref_latents = [reference_latent], audio_embedding = audio_embedding)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent), tiling = tiling())
+ audio = {samples = driving_audio[:, :math::floor(sample_rate * frames / fps)], sample_rate = sample_rate}
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = audio)
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = audio}
+}
diff --git a/docs/dgl/wan/wan2-2-t2v-a14b-image.dgl b/docs/dgl/wan/wan2-2-t2v-a14b-image.dgl
new file mode 100644
index 0000000..ad1dc2a
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-t2v-a14b-image.dgl
@@ -0,0 +1,54 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 T2V A14B T2I
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 9
+ fps = 16
+
+ steps = 18
+ high_noise_steps = 8
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ high_base = load_dit(high_model_path, family = "wan2", params_backend = "cpu")
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive)
+ high_neg = predictor(high_base, negative)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-t2v-a14b-lora.dgl b/docs/dgl/wan/wan2-2-t2v-a14b-lora.dgl
new file mode 100644
index 0000000..53555ac
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-t2v-a14b-lora.dgl
@@ -0,0 +1,60 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 T2V 14B with Lora
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+ lora_0_path = "../models/loras/wan2.2_t2v_lightx2v_4steps_lora_v1.1_low_noise"
+ lora_1_path = "../models/loras/wan2.2_t2v_lightx2v_4steps_lora_v1.1_high_noise"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 8
+ high_noise_steps = 4
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ base_model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ lora_0 = load_lora(lora_0_path)
+ model = apply_lora(base_model, lora_0)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ high_unpatched = load_dit(high_model_path, family = "wan2", params_backend = "cpu")
+ lora_1 = load_lora(lora_1_path)
+ high_base = apply_lora(high_unpatched, lora_1)
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive)
+ high_neg = predictor(high_base, negative)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-t2v-a14b.dgl b/docs/dgl/wan/wan2-2-t2v-a14b.dgl
new file mode 100644
index 0000000..2c8e6b2
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-t2v-a14b.dgl
@@ -0,0 +1,54 @@
+# version: 0.0.0
+# docs/models/wan.md — Wan2.2 T2V A14B
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 18
+ high_noise_steps = 8
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ high_base = load_dit(high_model_path, family = "wan2", params_backend = "cpu")
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive)
+ high_neg = predictor(high_base, negative)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-ti2v-5b-img2video.dgl b/docs/dgl/wan/wan2-2-ti2v-5b-img2video.dgl
new file mode 100644
index 0000000..19b6a93
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-ti2v-5b-img2video.dgl
@@ -0,0 +1,63 @@
+# version: 0.0.0
+# docs/models/wan.md — I2V
+
+var {
+ model_path = "../models/diffusion_models/wan2.2_ti2v_5B_fp16.safetensors"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan2.2_vae.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2_2_ti2v", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2_2_ti2v", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2_2_ti2v", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ video_shape = tensor::shape(video_base)
+ mask_blank = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ base = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ video_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 0)
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng, base = base, denoise_mask = video_mask, mask_input = true)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-ti2v-5b-txt2video.dgl b/docs/dgl/wan/wan2-2-ti2v-5b-txt2video.dgl
new file mode 100644
index 0000000..d4c69cf
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-ti2v-5b-txt2video.dgl
@@ -0,0 +1,45 @@
+# version: 0.0.0
+# docs/models/wan.md — T2V
+
+var {
+ model_path = "../models/diffusion_models/wan2.2_ti2v_5B_fp16.safetensors"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan2.2_vae.safetensors"
+
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 480
+ height = 832
+ frames = 33
+ fps = 16
+
+ steps = 20
+ cfg_scale = 6.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "wan2_2_ti2v", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2_2_ti2v", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2_2_ti2v", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler, flow_shift = 3.0)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/wan/wan2-2-vace-fun-a14b-ref2video.dgl b/docs/dgl/wan/wan2-2-vace-fun-a14b-ref2video.dgl
new file mode 100644
index 0000000..46120ca
--- /dev/null
+++ b/docs/dgl/wan/wan2-2-vace-fun-a14b-ref2video.dgl
@@ -0,0 +1,80 @@
+# version: 0.0.0
+# docs/models/wan.md — R2V
+
+var {
+ model_path = "../models/diffusion_models/Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf"
+ high_model_path = "../models/diffusion_models/Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf"
+ text_path = "../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf"
+ vae_path = "../models/vae/wan_2.1_vae.safetensors"
+
+ first_frame_path = "../inputs/cat_with_sd_cpp_42.png"
+ output_path = "output.avi"
+
+ prompt = "a lovely cat"
+ negative_prompt = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 832
+ height = 480
+ frames = 33
+ fps = 16
+
+ steps = 18
+ high_noise_steps = 8
+ cfg_scale = 3.5
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn main() {
+ first_frame = image::load(first_frame_path)
+
+ model = load_dit(model_path, family = "wan2", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "wan2", params_backend = "cpu", t5_path = text_path)
+ vae = load_vae(vae_path, family = "wan2", params_backend = "cpu")
+
+ rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = rng))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = rng))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+ high_base = load_dit(high_model_path, family = "wan2", params_backend = "cpu")
+
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive, vace_context = vace_context)
+ neg = predictor(model, negative, vace_context = vace_context)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ high_pos = predictor(high_base, positive, vace_context = vace_context)
+ high_neg = predictor(high_base, negative, vace_context = vace_context)
+ high_guided = cfg(high_pos, high_neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = rng)
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = rng)
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]))
+ video::save(output_path, {frames = output_frames, fps_num = fps, fps_den = 1}, audio = {})
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {}}
+}
diff --git a/docs/dgl/z_image/z-image-base-txt2img.dgl b/docs/dgl/z_image/z-image-base-txt2img.dgl
new file mode 100644
index 0000000..37f5fc3
--- /dev/null
+++ b/docs/dgl/z_image/z-image-base-txt2img.dgl
@@ -0,0 +1,43 @@
+# version: 0.0.0
+# docs/models/z_image.md — Z-Image-Base
+
+var {
+ model_path = "../models/diffusion_models/z_image_bf16.safetensors"
+ text_path = "../models/text_encoders/qwen_3_4b.safetensors"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ negative_prompt = ""
+ prompt = "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 1024
+
+ steps = 20
+ cfg_scale = 5.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "z_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "z_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "z_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/z_image/z-image-turbo-controlnet.dgl b/docs/dgl/z_image/z-image-turbo-controlnet.dgl
new file mode 100644
index 0000000..890df2b
--- /dev/null
+++ b/docs/dgl/z_image/z-image-turbo-controlnet.dgl
@@ -0,0 +1,46 @@
+# version: 0.0.0
+
+var {
+ dit_path = "z_image_turbo_bf16.safetensors"
+ encoder_path = "qwen_3_4b.safetensors"
+ vae_path = "ae.sft"
+ controlnet_path = "z-image-control.safetensors"
+
+ input_path = "pose.png"
+ output_path = "z-image-controlnet.png"
+
+ prompt = "A dancer in a bright studio"
+
+ seed = 42
+ width = 512
+ height = 512
+
+ steps = 8
+}
+
+fn z_image_control_hint(vae, image, rng) {
+ encoded = vae_encode(vae, image, rng = rng, posterior = "mode")
+ latent = vae_to_dit_latents(vae, encoded)
+ return tensor::pad(latent, [0, 0, 0, 0, 0, 17])
+}
+
+fn main() {
+ dit = load_dit(dit_path, family = "z_image")
+ text = load_text_encoder(encoder_path, family = "z_image")
+ vae = load_vae(vae_path, family = "z_image")
+ control = controlnet::load(controlnet_path, family = "z_image")
+ pose = tensor::interpolate(image::load(input_path), size = [width, height], axes = [1, 2])
+ rng = make_rng("cuda", seed = seed)
+ hint = z_image_control_hint(vae, pose, rng)
+ condition = controlnet::apply(encode_prompt(text, prompt, width = width, height = height),
+ control, hint, strength = 0.8)
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ noise = tensor::randn_like(base, rng = rng)
+ sampling_predictor = predictor(dit, condition)
+ sigmas = make_sigmas(sampling_predictor, latent = base, scheduler = "discrete", steps = steps)
+ start = add_noise(sampling_predictor, base, noise, sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = sampling_predictor, method = "euler", rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return { image = image, latent = latent }
+}
diff --git a/docs/dgl/z_image/z-image-turbo-txt2img.dgl b/docs/dgl/z_image/z-image-turbo-txt2img.dgl
new file mode 100644
index 0000000..1f7600f
--- /dev/null
+++ b/docs/dgl/z_image/z-image-turbo-txt2img.dgl
@@ -0,0 +1,38 @@
+# version: 0.0.0
+# docs/models/z_image.md — Z-Image-Turbo
+
+var {
+ model_path = "../models/diffusion_models/z_image_turbo-Q3_K.gguf"
+ text_path = "../models/text_encoders/Qwen3-4B-Instruct-2507-Q4_K_M.gguf"
+ vae_path = "../models/vae/ae.sft"
+
+ output_path = "output.png"
+
+ prompt = "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic"
+
+ seed = 42
+ rng_type = "cuda"
+ width = 512
+ height = 1024
+
+ steps = 8
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "z_image", params_backend = "cpu")
+ text = load_text_encoder(text_path, family = "z_image", params_backend = "cpu")
+ vae = load_vae(vae_path, family = "z_image", params_backend = "cpu")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ condition = encode_prompt(text, prompt, width = width, height = height)
+ predict = predictor(model, condition)
+ sigmas = make_sigmas(predict, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(predict, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = predict, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/dgl/z_image_l2p/z-image-l2p-txt2img.dgl b/docs/dgl/z_image_l2p/z-image-l2p-txt2img.dgl
new file mode 100644
index 0000000..5521b7f
--- /dev/null
+++ b/docs/dgl/z_image_l2p/z-image-l2p-txt2img.dgl
@@ -0,0 +1,42 @@
+# version: 0.0.0
+# docs/models/z_image_l2p.md — Text-to-image
+
+var {
+ model_path = "../models/diffusion_models/model-1k-merge.safetensors"
+ text_path = "../models/text_encoders/qwen_3_4b.safetensors"
+
+ output_path = "z_image_l2p.png"
+
+ prompt = "an origami pig on fire in the middle of a dark room with a pentagram on the floor"
+ negative_prompt = ""
+
+ seed = 42
+ rng_type = "cuda"
+ width = 1024
+ height = 1024
+
+ steps = 30
+ cfg_scale = 2.0
+ method = "euler"
+ scheduler = "discrete"
+}
+
+fn main() {
+ model = load_dit(model_path, family = "z_image_l2p", wtype = "bf16")
+ text = load_text_encoder(text_path, family = "z_image_l2p", wtype = "bf16")
+ vae = load_vae(model_path, family = "z_image_l2p", wtype = "bf16")
+
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+ positive = encode_prompt(text, prompt, width = width, height = height)
+ negative = encode_prompt(text, negative_prompt, width = width, height = height)
+ pos = predictor(model, positive)
+ neg = predictor(model, negative)
+ guided = cfg(pos, neg, scale = cfg_scale)
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler)
+ rng = make_rng(rng_type, seed = seed)
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = rng)
+ image = vae_decode(vae, dit_to_vae_latents(vae, latent))
+ image::save(output_path, image)
+ return {image = image, latent = latent}
+}
diff --git a/docs/guides/backend.md b/docs/guides/backend.md
index f7edddf..a45a0ca 100644
--- a/docs/guides/backend.md
+++ b/docs/guides/backend.md
@@ -1,5 +1,9 @@
# Backend selection
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
`stable-diffusion.cpp` has two backend assignments:
- `--backend` selects the runtime backend used to execute model graphs.
@@ -16,6 +20,7 @@ A backend assignment can be a single backend name:
sd-cli -m model.safetensors -p "a cat" --backend cpu
```
+
This applies to every module that does not have a more specific assignment.
Assignments can also target individual modules:
@@ -24,24 +29,28 @@ Assignments can also target individual modules:
sd-cli -m model.safetensors -p "a cat" --backend te=cpu,vae=cuda0,diffusion=vulkan0
```
+
The same syntax is used for parameter placement:
```shell
sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend te=cpu,vae=cpu
```
+
`--params-backend` also accepts the special value `disk`:
```shell
sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend disk
```
+
`--max-vram` can target resolved backend/device names:
```shell
sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,vae=vulkan0 --max-vram cuda0=6,vulkan0=2
```
+
The value is a shared per-device budget for managed weights and registered
runner compute/cache buffers. Live free memory can lower the effective limit
for each graph run. Driver contexts and allocations made outside the managed
@@ -56,6 +65,7 @@ Module names are case-insensitive. Hyphens and underscores in module names are i
sd-cli -m model.safetensors -p "a cat" --backend all=cuda0,te=cpu
```
+
## Multiple devices per module (layer split)
A `--backend` module assignment can list several devices separated by `&`:
@@ -64,6 +74,7 @@ A `--backend` module assignment can list several devices separated by `&`:
sd-cli -m model.safetensors -p "a cat" --backend "diffusion=cuda0&cuda1"
```
+
The module's transformer blocks are then distributed across the listed devices
in contiguous ranges sized proportionally to each device's free memory (minus a
compute-buffer headroom of about 2 GiB per device), and the
@@ -102,6 +113,7 @@ sd-cli -m model.safetensors -p "a cat" --backend "diffusion=cuda0&cuda1" --split
sd-cli -m model.safetensors -p "a cat" --backend "diffusion=cuda0&cuda1,te=cuda0&cuda1" --split-mode diffusion=row,te=layer
```
+
In row mode the module keeps executing on its main (first listed) device, but
its transformer-block matmul weights are allocated in the backend's row-split
buffer type, which slices each weight's rows across the listed devices in
@@ -151,6 +163,7 @@ sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,te=cpu,vae=cuda
sd-cli -m model.safetensors -p "a cat" --auto-fit off
```
+
Budgets reuse `--max-vram`: a positive per-device value caps what auto-fit
plans with on that device, a negative value means "free memory minus that many
GiB", and with no budget set each device's free memory minus a 512 MiB margin
@@ -241,6 +254,7 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct auto,vae=off
sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on
```
+
- `on` requests direct 2D and 3D convolution.
- `off` uses the existing im2col-based convolution paths.
- `auto` first checks support for a representative F16-weight/F32-input direct
@@ -291,6 +305,7 @@ sd-cli -m model.safetensors -p "a cat" --flash-attn auto,te=off,vae=on
sd-cli -m model.safetensors -p "a cat" --flash-attn off,diffusion=auto
```
+
Supported modules are `diffusion`, `te`, `clip_vision`, `vae`, `controlnet`, and
`audio_encoder`. Modules without an assignment use the default (`auto` if no
default is supplied). A bare mode or `all=`, `default=`, or `*=` sets the default;
@@ -351,6 +366,7 @@ sd-cli -m model.safetensors -p "a cat" --flash-attn auto
sd-cli -m model.safetensors -p "a cat" --flash-attn off
```
+
## Modules
| Module | Purpose | Accepted names |
@@ -391,6 +407,7 @@ For example:
sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend cpu
```
+
This runs all modules on `cuda0`, but stores parameters in CPU RAM. During execution, parameters are moved to the runtime backend as needed.
For example:
@@ -399,6 +416,7 @@ For example:
sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend disk
```
+
This runs all modules on `cuda0`, reloads parameters from the model file as needed, and releases those parameter buffers after use.
Outside `--auto-fit`, `disk` is never selected implicitly. If `--params-backend` is not set, parameters use the runtime backend.
@@ -409,6 +427,7 @@ Per-module assignments can be mixed:
sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,te=cpu,vae=cpu --params-backend diffusion=cuda0,te=cpu,vae=cpu
```
+
This keeps text encoding and VAE execution on CPU while the diffusion model runs on GPU.
## Backend sharing and lifetime
diff --git a/docs/guides/c_api.md b/docs/guides/c_api.md
index 055056d..7e66c40 100644
--- a/docs/guides/c_api.md
+++ b/docs/guides/c_api.md
@@ -1,5 +1,8 @@
# C API
+Handwritten equivalents of the generated image/video programs are available in
+the [DGL documentation examples](../dgl/README.md).
+
Use purpose-specific opaque options for model loading, generation and
postprocessing, and
opaque `sd_result_t` objects to read generated outputs. The library allocates and
@@ -42,7 +45,7 @@ int generate(const char* model_path) {
for (size_t i = 0; i < sd_result_get_image_count(result); ++i) {
const sd_tensor_t* image = sd_result_get_image(result, i);
const char* info = sd_result_get_generation_info(result, i);
- /* Read pixels with sd_tensor_data(image), and consume the UTF-8 JSON info. */
+ /* Read pixels with sd_tensor_data(image), and consume the UTF-8 DGL source. */
(void) image;
(void) info;
}
@@ -59,6 +62,14 @@ cleanup:
}
```
+`sd_ctx_create()` retains configuration and leaves component loading lazy. To
+validate a model before accepting generation requests, install any file-access
+callback and call `sd_ctx_prepare(ctx)`. It probes the model, builds and validates
+the configured components, and retains them for later generation without
+sampling. Weight residency still follows `eager_load` and the memory budgets.
+Preparation failures return a non-OK status; callers can discard or retry the
+context. Capability queries reuse the detected model type during generation.
+
Use `sd_generate_video()` with options from `sd_vid_gen_opt_create()` to generate video. Its result
exposes frames through the same image accessors, optional audio through
`sd_result_get_audio()`, and effective frame rate through `sd_result_get_fps()`.
@@ -87,10 +98,6 @@ are required and set to `NULL` on failure. All outputs use `sd_result_t`.
| `sd_vid_gen_opt_create()` | `sd_generate_video()` |
| `sd_upscaler_opt_create()` | `sd_upscaler_create()` |
| `sd_upscale_opt_create()` | `sd_upscale()` |
-| `sd_vid_upscaler_opt_create()` | `sd_vid_upscaler_create()` |
-| `sd_vid_upscale_opt_create()` | `sd_upscale_video()` |
-| `sd_adetailer_opt_create()` | `sd_adetailer_create()` |
-| `sd_adetail_opt_create()` | `sd_adetail()` |
Image upscaling loads `model_path` with optional `n_threads`, `backend`,
`params_backend`, `conv_direct` and `tile_size` (default 128). Execution takes
@@ -118,22 +125,20 @@ sd_upscaler_free(upscaler);
sd_opt_free(load);
```
-Video-upscaler loading requires `diffusion_model_path`, `vae_path` and
-`upscale_factor`; backend/thread/attention settings are optional. The scale must
-match both checkpoints (RelayVSR uses 2 or 4). Execution takes RGB `frames` with
-equal dimensions. Defaults are FPS 24, seed 42, RelayVSR `window_size` 16, and
-SeedVR2 `temporal_chunk_size` 5. Free the model with `sd_vid_upscaler_free()`.
-
-ADetailer loading takes the detector `model_path` and optional backend/thread
-settings. Call `sd_adetail(adetailer, diffusion_ctx, opt, &result)` with an
-`sd_adetail_opt_t`. Set `image` for the source image, and reuse the applicable
-image-generation keys for inpainting: `prompt`, `negative_prompt`, dimensions,
-strength, seed, sampling, LoRAs, cache, VAE tiling and circular axes. Inpainting
-defaults to 512×512 and strength 0.4. `adetail.prompt` and
-`adetail.negative_prompt` supply region prompt overrides; `adetail.args` accepts
-the existing detector/mask/inpaint argument string documented by the CLI
-`--extra-ad-args` option. Reference/control images, hires and batching are not
-ADetailer options. Free the detector with `sd_adetailer_free()`.
+ADetailer, independent video upscaling, AnimateDiff, PhotoMaker, PuLID,
+IP-Adapter, custom preprocessing, SLG, timestep shifting and
+automatic post-generation upscaling are explicit DGL workflows. Their former
+C operation handles or parameter keys are unavailable. Ordinary ControlNet
+remains supported with a configured model and a direct control image.
+
+Contexts retain configuration snapshots and a persistent DGL execution context.
+Model metadata is probed lazily and cached with file dependencies. Creation can
+succeed before a missing or invalid model is discovered by a capability query
+or generation call. Component weights are loaded and reused by dgl.
+
+Reference inputs follow model defaults. Custom reference resizing, cropping,
+position indexing and conditioning must be written in DGL; there is no
+`ref_image_args` override string.
## String-valued modes
@@ -261,8 +266,7 @@ needed independently.
Types below correspond to `sd_opt_set_<type>()`. `context`, `image` and `video`
refer to model loading and image/video generation. Other operation labels match
-the corresponding option constructor, such as `vid_upscale` for
-`sd_vid_upscale_opt_create()`.
+the corresponding option constructor, `upscaler` or `upscale`.
Array lengths are supplied to setters; there are no separately writable count keys.
Top-level keys retain their existing parameter meanings. Nested names map as follows:
@@ -270,90 +274,113 @@ Top-level keys retain their existing parameter meanings. Nested names map as fol
| Key prefix | Configuration |
| --- | --- |
| `sample.*` | Base sampling settings; `method`, `steps`, `sigmas`, `args` select the sampler, step count, custom schedule and extra arguments. |
-| `high_noise_sample.*` | High-noise sampling settings for two-stage video models. |
-| `sample.guidance.*` | Text/image CFG, distilled guidance and SLG settings. |
+| `sample.guidance.*` | Text/image CFG and distilled guidance settings. |
| `cache.*` | [Inference cache](caching.md) settings. |
-| `hires.*` | High-resolution second pass and upscaling settings. |
| `vae_tiling.*` | Spatial/temporal VAE tiling settings. |
-| `photomaker.*` | PhotoMaker identity images/embedding and style strength. |
-| `pulid.*` | PuLID identity embedding path and weight. |
-| `image_preprocess.rules` | [Image preprocessing](image_preprocessing.md) rule string. |
-The library supplies inference defaults. CLI/server inputs that are omitted do
-not override them. Model recipes may supply presets; explicit startup options
+The library supplies inference defaults. Omitted server inputs do not override
+them. The CLI retains its historical seed default of 42; `--seed -1` requests
+a random seed. Other omitted CLI inference options use the library defaults.
+Model recipes may supply presets; explicit startup options
and request fields take priority over recipe values. A supplied value equal to
a default is still explicit and undergoes model applicability checks.
Generation defaults include 512×512, a random seed (`-1`), 20 sampling steps,
model-selected sampler/scheduler, text CFG 7, distilled guidance 3.5, strength
-0.75, and disabled caching/hires/tiling. Image batches default to 1; videos default
+0.75, and disabled caching/tiling. Image batches default to 1; videos default
to 6 requested frames at 16 FPS, subject to model alignment. Omitted eta, flow shift
and image CFG retain automatic defaults; do not pass infinity to request defaults.
-Use `unset`. High-noise steps default to `-1` (automatic allocation). The high-noise pass
-shares the base scheduler, sigmas, flow shift and timestep shifting; set those
-through `sample.*`. Separate high-noise keys for these settings are not supported.
+Use `unset`. Wan dual experts share one `sample.*` configuration and one
+sigma sequence. `high_noise_sample.steps` defaults to -1: `sample.steps` is the
+total, split using `moe_boundary` (default 0.875). With a nonnegative high-noise
+count, the total is `sample.steps + high_noise_sample.steps`; the high-noise
+expert runs that prefix and the low-noise expert runs the remaining steps.
+Zero selects only the low-noise expert. Without a high-noise model this setting
+is ignored. Custom sigmas fix the total; a positive high-noise count is adjusted
+to retain the requested low-noise count when possible. Independent high-noise
+CFG and sampler overrides still require dgl.
+
+Image and LTX/MiniMax-H3 video hires passes use the same DGL execution path.
+Set `hires.enabled=true`. The default is latent bilinear upscaling by 2,
+followed by refinement with the original model and prompt conditions.
+`hires.steps=0` inherits the base step count; denoising strength defaults to
+0.7. Without custom sigmas, a schedule with `floor(steps / strength)` intervals
+is generated and its last `steps + 1` entries are used. `hires.sigmas` instead
+supplies the complete refinement schedule and is not sliced by strength.
+
+`hires.target_width` and `hires.target_height` override the scale. Supplying
+only one makes a square target. Output dimensions follow model alignment.
+Image upscalers are `latent`, `latent_nearest`, `latent_nearest_exact`,
+`latent_antialiased`, `latent_bicubic`, `latent_bicubic_antialiased`, `lanczos`,
+`nearest`, `model`, or `none`. `model` requires `hires.model_path`; images use
+an image upscaler and then VAE re-encoding. Videos accept latent interpolation
+or a compatible latent upscaler. LTX learned upscalers determine their output
+size and frame count from the checkpoint, overriding target dimensions;
+audio duration and keyframe conditioning follow that output. H3 learned
+upscalers use the requested target size. Other video families require an
+explicit DGL refinement workflow.
Sampler/scheduler values follow the [sampling guide](sampling.md); `"auto"`
requests automatic selection. RNG values are `std_default`, `cuda`, `cpu`, `auto`.
-Prediction modes are `auto`, `eps`, `v`, `edm_v`, `sd3_flow`, `flux_flow`,
+Prediction modes are `auto`, `eps`, `v`, `edm_v`, `flow`, `flux_flow`,
`sefi_flow`, `minit2i_flow`, `sensenova_u1_flow`, `flux3_flow`, `h3_av_flow`,
`ming_image_flow`. Weight types use the names returned by `sd_type_name()`,
plus `auto`. VAE formats are `auto`, `flux`, `sd3`,
`flux2`, `wan`. LoRA apply modes are `auto`, `immediately`, `at_runtime`.
Cache modes are `disabled`, `easycache`, `ucache`, `dbcache`, `taylorseer`,
`cache_dit`, `spectrum`. Control types are `none`, `openpose`, `depth`, `scribble`, `canny`, `normal`,
-`segment`, `tile`, `repaint`, with the existing aliases such as `hed` and `lineart`. Hires upscalers are `none`, `latent`,
-`latent_nearest`, `latent_nearest_exact`, `latent_antialiased`, `latent_bicubic`,
-`latent_bicubic_antialiased`, `lanczos`, `nearest`, `model`.
+`segment`, `tile`, `repaint`, with the existing aliases such as `hed` and `lineart`.
Integers must fit the underlying field (32 bits except `seed`, which is signed
64-bit). Dimensions, batch size, video frames, FPS and base steps are positive.
Floating-point settings must be finite and fit float32; strength, denoising
strength, percent/range settings and MOE boundary lie in [0, 1]. Custom sigma
arrays are empty or strictly decreasing nonnegative finite values ending in zero,
-with at least two entries. SLG layer indices are nonnegative integers.
+with at least two entries.
+
+For SDXL's bundled VAE (no `vae_path`), generation emits
+`conv_scale = 0.03125` in the generated DGL to avoid FP16 overflow. Set
+`force_sdxl_vae_conv_scale = true` to apply the same scaling to an explicit
+`vae_path`. TAE decoding retains its own scaling.
| Key | Setter type | Operation |
| --- | --- | --- |
-| `adetail.args` | `string` | adetail |
-| `adetail.negative_prompt` | `string` | adetail |
-| `adetail.prompt` | `string` | adetail |
| `attn_scale` | `float` | context |
| `audio_encoder_path` | `string` | context |
| `audio_vae_path` | `string` | context |
| `auto_fit` | `bool` | context |
-| `backend` | `string` | context, upscaler, vid_upscaler, adetailer |
+| `backend` | `string` | context, upscaler |
| `batch_count` | `int` | image |
| `batched_cfg` | `bool` | context |
-| `cache.bn_compute_blocks` | `int` | image, video, adetail |
-| `cache.end_percent` | `float` | image, video, adetail |
-| `cache.error_decay_rate` | `float` | image, video, adetail |
-| `cache.fn_compute_blocks` | `int` | image, video, adetail |
-| `cache.max_cached_steps` | `int` | image, video, adetail |
-| `cache.max_continuous_cached_steps` | `int` | image, video, adetail |
-| `cache.max_warmup_steps` | `int` | image, video, adetail |
-| `cache.mode` | `string` | image, video, adetail |
-| `cache.reset_error_on_compute` | `bool` | image, video, adetail |
-| `cache.residual_diff_threshold` | `float` | image, video, adetail |
-| `cache.reuse_threshold` | `float` | image, video, adetail |
-| `cache.scm_mask` | `string` | image, video, adetail |
-| `cache.scm_policy_dynamic` | `bool` | image, video, adetail |
-| `cache.spectrum_flex_window` | `float` | image, video, adetail |
-| `cache.spectrum_lam` | `float` | image, video, adetail |
-| `cache.spectrum_m` | `int` | image, video, adetail |
-| `cache.spectrum_stop_percent` | `float` | image, video, adetail |
-| `cache.spectrum_w` | `float` | image, video, adetail |
-| `cache.spectrum_warmup_steps` | `int` | image, video, adetail |
-| `cache.spectrum_window_size` | `int` | image, video, adetail |
-| `cache.start_percent` | `float` | image, video, adetail |
-| `cache.taylorseer_n_derivatives` | `int` | image, video, adetail |
-| `cache.taylorseer_skip_interval` | `int` | image, video, adetail |
-| `cache.use_relative_threshold` | `bool` | image, video, adetail |
-| `circular_x` | `bool` | image, video, adetail |
-| `circular_y` | `bool` | image, video, adetail |
+| `cache.bn_compute_blocks` | `int` | image, video |
+| `cache.end_percent` | `float` | image, video |
+| `cache.error_decay_rate` | `float` | image, video |
+| `cache.fn_compute_blocks` | `int` | image, video |
+| `cache.max_cached_steps` | `int` | image, video. Negative values mean no limit. |
+| `cache.max_continuous_cached_steps` | `int` | image, video. Negative values mean no limit. |
+| `cache.max_warmup_steps` | `int` | image, video |
+| `cache.mode` | `string` | image, video |
+| `cache.reset_error_on_compute` | `bool` | image, video |
+| `cache.residual_diff_threshold` | `float` | image, video |
+| `cache.reuse_threshold` | `float` | image, video |
+| `cache.scm_mask` | `string` | image, video |
+| `cache.scm_policy_dynamic` | `bool` | image, video |
+| `cache.spectrum_flex_window` | `float` | image, video |
+| `cache.spectrum_lam` | `float` | image, video |
+| `cache.spectrum_m` | `int` | image, video |
+| `cache.spectrum_stop_percent` | `float` | image, video |
+| `cache.spectrum_w` | `float` | image, video |
+| `cache.spectrum_warmup_steps` | `int` | image, video |
+| `cache.spectrum_window_size` | `int` | image, video |
+| `cache.start_percent` | `float` | image, video |
+| `cache.taylorseer_n_derivatives` | `int` | image, video |
+| `cache.taylorseer_skip_interval` | `int` | image, video |
+| `cache.use_relative_threshold` | `bool` | image, video |
+| `circular_x` | `bool` | image, video |
+| `circular_y` | `bool` | image, video |
| `clip_g_path` | `string` | context |
| `clip_l_path` | `string` | context |
-| `clip_skip` | `int` | image, video, adetail |
+| `clip_skip` | `int` | image, video. Nonpositive values use the model default. |
| `clip_vision_path` | `string` | context |
| `conditioning_cache_size` | `int` | context |
| `control_frames` | `tensor` | video |
@@ -362,10 +389,10 @@ with at least two entries. SLG layer indices are nonnegative integers.
| `control_strength` | `float` | image |
| `control_type` | `string` | image |
| `conv_direct` | `bool` | upscaler |
-| `conv_direct_mode` | `string` | context, vid_upscaler |
+| `conv_direct_mode` | `string` | context |
| `diffusion_conv_direct` | `bool` | context |
| `diffusion_flash_attn` | `bool` | context |
-| `diffusion_model_path` | `string` | context, vid_upscaler |
+| `diffusion_model_path` | `string` | context |
| `disable_prefetch` | `bool` | context |
| `disable_segmented_compute` | `bool` | context |
| `eager_load` | `bool` | context |
@@ -374,113 +401,80 @@ with at least two entries. SLG layer indices are nonnegative integers.
| `enable_mmap` | `bool` | context |
| `end_image` | `tensor` | video |
| `flash_attn` | `bool` | context |
-| `flash_attn_mode` | `string` | context, vid_upscaler |
+| `flash_attn_mode` | `string` | context |
| `force_sdxl_vae_conv_scale` | `bool` | context |
-| `fps` | `int` | video, vid_upscale |
-| `frames` | `tensor` | vid_upscale |
-| `height` | `int` | image, video, adetail |
+| `fps` | `int` | video |
+| `height` | `int` | image, video |
| `high_noise_diffusion_model_path` | `string` | context |
-| `high_noise_sample.args` | `string` | video |
-| `high_noise_sample.eta` | `float` | video |
-| `high_noise_sample.guidance.distilled_guidance` | `float` | video |
-| `high_noise_sample.guidance.img_cfg` | `float` | video |
-| `high_noise_sample.guidance.slg.layer_end` | `float` | video |
-| `high_noise_sample.guidance.slg.layer_start` | `float` | video |
-| `high_noise_sample.guidance.slg.layers` | `ints` | video |
-| `high_noise_sample.guidance.slg.scale` | `float` | video |
-| `high_noise_sample.guidance.txt_cfg` | `float` | video |
-| `high_noise_sample.method` | `string` | video |
-| `high_noise_sample.steps` | `int` | video |
-| `hires.denoising_strength` | `float` | image, video |
-| `hires.enabled` | `bool` | image, video |
-| `hires.model_path` | `string` | image, video |
-| `hires.scale` | `float` | image, video |
-| `hires.sigmas` | `floats` | image, video |
-| `hires.steps` | `int` | image, video |
-| `hires.target_height` | `int` | image, video |
-| `hires.target_width` | `int` | image, video |
-| `hires.upscale_tile_size` | `int` | image, video |
-| `hires.upscaler` | `string` | image, video |
-| `image` | `tensor` | upscale, adetail |
-| `image_preprocess.rules` | `string` | image, video |
+| `image` | `tensor` | upscale |
| `init_image` | `tensor` | image, video |
-| `ip_adapter_image` | `tensor` | image |
-| `ip_adapter_path` | `string` | context |
-| `ip_adapter_strength` | `float` | image |
| `linear_scale` | `float` | context |
| `llm_path` | `string` | context |
| `llm_vision_path` | `string` | context |
| `lora_apply_mode` | `string` | context |
-| `loras` | `loras` | image, video, adetail |
+| `loras` | `loras` | image, video |
| `mask_image` | `tensor` | image |
+| `max_ram` | `string` | context |
| `max_vram` | `string` | context |
| `model_args` | `string` | context |
-| `model_path` | `string` | context, upscaler, adetailer |
+| `model_path` | `string` | context, upscaler |
| `moe_boundary` | `float` | video |
-| `motion_module_path` | `string` | context |
-| `n_threads` | `int` | context, upscaler, vid_upscaler, adetailer |
-| `negative_prompt` | `string` | image, video, adetail |
-| `params_backend` | `string` | context, upscaler, vid_upscaler, adetailer |
-| `photo_maker_path` | `string` | context |
-| `photomaker.id_embed_path` | `string` | image |
-| `photomaker.id_images` | `tensors` | image |
-| `photomaker.style_strength` | `float` | image |
+| `high_noise_sample.steps` | `int` | video; -1 or nonnegative |
+| `hires.enabled` | `bool` | image, video |
+| `hires.upscaler`, `hires.model_path` | `string` | image, video |
+| `hires.scale` | `float` | image, video; positive, default 2 |
+| `hires.target_width`, `hires.target_height` | `int` | image, video; 0 selects scale |
+| `hires.steps` | `int` | image, video; 0 inherits base steps |
+| `hires.denoising_strength` | `float` | image, video; 0.0001..1 |
+| `hires.upscale_tile_size` | `int` | image model upscaling; default 128 |
+| `hires.sigmas` | `floats` | image, video; optional refinement schedule |
+| `n_threads` | `int` | context, upscaler |
+| `negative_prompt` | `string` | image, video |
+| `params_backend` | `string` | context, upscaler |
| `prediction` | `string` | context |
-| `prompt` | `string` | image, video, adetail |
-| `pulid.id_embedding_path` | `string` | image |
-| `pulid.id_weight` | `float` | image |
-| `pulid_weights_path` | `string` | context |
+| `prompt` | `string` | image, video |
| `qwen_image_layers` | `int` | image |
| `ref_audios` | `audios` | video |
-| `ref_image_args` | `string` | image, video |
| `ref_images` | `tensors` | image, video |
| `ref_videos` | `videos` | video |
| `rng_type` | `string` | context |
| `rpc_servers` | `string` | context |
| `sage_attn` | `bool` | context |
-| `sample.args` | `string` | image, video, adetail |
-| `sample.eta` | `float` | image, video, adetail |
-| `sample.flow_shift` | `float` | image, video, adetail |
-| `sample.guidance.distilled_guidance` | `float` | image, video, adetail |
-| `sample.guidance.img_cfg` | `float` | image, video, adetail |
-| `sample.guidance.slg.layer_end` | `float` | image, video, adetail |
-| `sample.guidance.slg.layer_start` | `float` | image, video, adetail |
-| `sample.guidance.slg.layers` | `ints` | image, video, adetail |
-| `sample.guidance.slg.scale` | `float` | image, video, adetail |
-| `sample.guidance.txt_cfg` | `float` | image, video, adetail |
-| `sample.method` | `string` | image, video, adetail |
-| `sample.scheduler` | `string` | image, video, adetail |
-| `sample.shifted_timestep` | `int` | image, video, adetail |
-| `sample.sigmas` | `floats` | image, video, adetail |
-| `sample.steps` | `int` | image, video, adetail |
+| `sample.args` | `string` | image, video |
+| `sample.eta` | `float` | image, video |
+| `sample.flow_shift` | `float` | image, video |
+| `sample.guidance.distilled_guidance` | `float` | image, video |
+| `sample.guidance.img_cfg` | `float` | image, video |
+| `sample.guidance.txt_cfg` | `float` | image, video |
+| `sample.method` | `string` | image, video |
+| `sample.scheduler` | `string` | image, video |
+| `sample.sigmas` | `floats` | image, video |
+| `sample.steps` | `int` | image, video |
| `sampler_rng_type` | `string` | context |
-| `seed` | `int` | image, video, vid_upscale, adetail |
+| `seed` | `int` | image, video |
| `split_mode` | `string` | context |
-| `strength` | `float` | image, video, adetail |
+| `strength` | `float` | image, video |
| `t5xxl_path` | `string` | context |
| `tae_preview_only` | `bool` | context |
| `taesd_path` | `string` | context |
-| `temporal_chunk_size` | `int` | vid_upscale |
| `tensor_type_rules` | `string` | context |
| `tile_size` | `int` | upscaler |
| `tokenizer` | `string` | context |
| `uncond_diffusion_model_path` | `string` | context |
-| `upscale_factor` | `int` | vid_upscaler |
| `vace_strength` | `float` | video |
| `vae_conv_direct` | `bool` | context |
| `vae_format` | `string` | context |
-| `vae_path` | `string` | context, vid_upscaler |
-| `vae_tiling.enabled` | `bool` | image, video, adetail |
-| `vae_tiling.extra_tiling_args` | `string` | image, video, adetail |
-| `vae_tiling.rel_size_h` | `float` | image, video, adetail |
-| `vae_tiling.rel_size_w` | `float` | image, video, adetail |
-| `vae_tiling.target_overlap` | `float` | image, video, adetail |
-| `vae_tiling.temporal_tiling` | `bool` | image, video, adetail |
-| `vae_tiling.tile_size_h` | `int` | image, video, adetail |
-| `vae_tiling.tile_size_w` | `int` | image, video, adetail |
+| `vae_path` | `string` | context |
+| `vae_tiling.enabled` | `bool` | image, video |
+| `vae_tiling.extra_tiling_args` | `string` | image, video |
+| `vae_tiling.rel_size_h` | `float` | image, video |
+| `vae_tiling.rel_size_w` | `float` | image, video |
+| `vae_tiling.target_overlap` | `float` | image, video |
+| `vae_tiling.temporal_tiling` | `bool` | image, video |
+| `vae_tiling.tile_size_h` | `int` | image, video |
+| `vae_tiling.tile_size_w` | `int` | image, video |
| `video_frames` | `int` | video |
-| `width` | `int` | image, video, adetail |
-| `window_size` | `int` | vid_upscale |
+| `width` | `int` | image, video |
| `wtype` | `string` | context |
## Migration
@@ -495,13 +489,39 @@ Create context/image/video options with the corresponding constructor, populate
them through the shared typed setters, and use `sd_ctx_create()`,
`sd_generate_image()` and `sd_generate_video()`. Read or retain outputs through
`sd_result_get_*()` and `sd_tensor_retain()` and release the result with
-`sd_result_free()`. ADetailer and image/video upscalers use purpose-specific opt constructors and
-return the same opaque result type. Their former public parameter structures,
-long positional signatures and bool-returning entry points have been removed.
-Device selection and cancellation controls retain their interfaces. Preview
-registration now uses a mode string and returns a status.
-
-Native generation JSON retains its existing schema and names, including
-`sample_params.sample_method`; translate these to the corresponding option keys
-(`sample.method`, etc.) when replaying metadata. See
-[generation results and metadata](generation_metadata.md).
+`sd_result_free()`. Standalone image upscaling retains its purpose-specific
+options and actual-scale queries. ADetailer and standalone video-upscale C
+handles have been removed; use explicit DGL workflows.
+
+Generation results now contain the exact executed DGL source. They no longer
+contain generation JSON. See [generation results and metadata](generation_metadata.md).
+Context creation is lazy, and model errors can first surface in a capability
+query or generation call. Context options are copied, while media handles are
+retained. Cancellation, progress and preview retain their public contracts.
+
+### DGL region results
+
+DGL regions are ordinary dictionaries with integer `width` and `height` fields
+and an I64 `boxes` tensor shaped `[4,N]`. The dedicated region value kind has
+been removed. `SD_DGL_VALUE_OPAQUE` remains `11` and `SD_DGL_VALUE_VOID` remains
+`12`.
+
+Returning a region dict directly now gives a `SD_DGL_VALUE_DICT` root and flat
+outputs `boxes`, `width` and `height`. Returning `{regions=r}` gives the flat
+names `regions.boxes`, `regions.width` and `regions.height`. Replace root-region
+exports such as `--tensor result boxes.npy` with `--tensor boxes boxes.npy`.
+Coordinate tensors now use I64; cast old I32 files explicitly with
+`tensor::cast(boxes, "i64")` before passing them to `regions::from_boxes`.
+
+Detection results are also ordinary dicts: F32 `xyxy` `[4,N]`, F32 `scores`
+`[N]`, and I64 `class_ids` `[N]`, with at most 4,096 items. Empty results preserve
+all three fields and ranks. `detections::detect`, `detections::filter`,
+`detections::box_masks` and `detections::to_regions` replace the previous global
+detection helpers. Use `detections::detect` followed by `detections::to_regions`
+to replace the old direct-to-regions `detect` shortcut; detection executes once.
+
+Use `regions::from_boxes`, `regions::crop`, `regions::paste` and
+`regions::plan_mask_crops` for rectangle and mask tasks. The previous global
+region function names are no longer registered. See
+[DGL rectangular regions](../dgl.md#rectangular-regions) for validation and
+composition rules.
diff --git a/docs/guides/caching.md b/docs/guides/caching.md
index 66dfc27..3ddf5e9 100644
--- a/docs/guides/caching.md
+++ b/docs/guides/caching.md
@@ -1,5 +1,9 @@
## Caching
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
Caching methods accelerate diffusion inference by reusing intermediate computations when changes between steps are small.
### Conditioning Cache
@@ -31,6 +35,7 @@ UCache caches the residual difference (output - input) and reuses it when input
sd-cli -m model.safetensors -p "a cat" --cache-mode ucache --cache-option "threshold=1.5"
```
+
#### Parameters
| Parameter | Description | Default |
@@ -67,11 +72,12 @@ Condition-level caching for DiT models. Caches and reuses outputs when input cha
### Cache-DIT (DiT Models)
-For DiT models like FLUX and QWEN, use block-level caching modes.
+For DiT models like FLUX and QWEN, these modes cache each model condition's
+raw output. This path does not perform block-level caching.
#### DBCache
-Caches blocks based on L1 residual difference threshold:
+Reuses whole-model input/output residuals when input changes pass the L1 difference threshold:
```bash
--cache-mode dbcache --cache-option "threshold=0.25,warmup=4"
@@ -79,7 +85,7 @@ Caches blocks based on L1 residual difference threshold:
#### TaylorSeer
-Uses Taylor series approximation to predict block outputs:
+Uses Taylor series approximation to predict each condition's raw model output:
```bash
--cache-mode taylorseer
@@ -97,11 +103,15 @@ Combines DBCache and TaylorSeer:
| Parameter | Description | Default |
|-----------|-------------|---------|
-| `Fn` | Front blocks to always compute | 8 |
-| `Bn` | Back blocks to always compute | 0 |
+| `Fn`, `Bn` | Legacy options: ignored with a warning; block counts are not supported | No effect |
| `threshold` | L1 residual difference threshold | 0.08 |
| `warmup` | Steps before caching starts | 8 |
+Explicit `Fn`/`Bn` settings, including their former defaults, produce a warning.
+The C API keys `cache.fn_compute_blocks` and `cache.bn_compute_blocks` remain
+accepted for compatibility and also warn when set. They have no effect on cache
+execution. Omitted options do not warn.
+
#### SCM Options
Steps Computation Mask controls which steps can be cached:
@@ -112,10 +122,10 @@ Steps Computation Mask controls which steps can be cached:
Mask values: `1` = compute, `0` = can cache.
-| Policy | Description |
-|--------|-------------|
-| `dynamic` | Check threshold before caching |
-| `static` | Always cache on cacheable steps |
+The native `--scm-policy` option is retained for compatibility. Its `dynamic`
+and `static` values do not change this condition-level implementation: a mask
+value of `1` forces computation; `0` permits reuse subject to the cache's gates
+and available history.
```bash
--scm-policy dynamic
@@ -129,6 +139,7 @@ Spectrum uses Chebyshev polynomial fitting blended with Taylor extrapolation to
sd-cli -m model.safetensors -p "a cat" --cache-mode spectrum
```
+
#### Parameters
| Parameter | Description | Default |
diff --git a/docs/guides/edit.md b/docs/guides/edit.md
index 942c395..2426cf0 100644
--- a/docs/guides/edit.md
+++ b/docs/guides/edit.md
@@ -1,114 +1,56 @@
-# Image Editing
-
-Image editing in `stable-diffusion.cpp` allows you to use reference images to guide the generation process, enabling tasks like identity preservation, style transfer, or layout modification.
-
-
-## Supported Models
-
-Depending on the architecture, different models handle reference images differently.
-
-| Model | Default Preset |
-| :--- | :--- |
-| [**FLUX.1-Kontext-dev**](../models/kontext.md) | `flux_kontext` |
-| [**LongCat Image Edit**](../models/longcat_image.md) | `longcat` |
-| [**Qwen Image Edit**](../models/qwen_image_edit.md) | `qwen` |
-| **Qwen Image LAYERED** | `qwen_layered` |
-| [**Flux.2 [Dev] / Flux.2 [Klein]**](../models/flux2.md) | `flux2` |
-| [**FLUX.3 Image Klein / Klein Base**](../models/flux3.md) | `flux2` |
-| [**Boogu Image Edit**](../models/boogu_image.md) | `z_image_omni` |
-| **Krea2 (Community Edit LoRAs)** | `krea2_ostris_edit` |
-| [**Mage-Flow-Edit**](../models/mage_flow.md#image-editing) | `mage_flow` |
-| [**LLaDA-Image**](../models/llada_image.md#image-editing) | `llada_image` |
-| [**Fibo 2.0 Dev / Edit Turbo**](../models/fibo2.md#fibo-20-edit-turbo) | `fibo2` |
-| **Anima (Community Edit LoRAs)** | `cosmos_reference` |
-
-Stable-diffusion.spp also supports basic Unet-based editing models like instruct-pix2pix or CosXL-Edit. This document is not about those.
-
----
-
-## Configuring Reference Modes (`--ref-image-args`)
-
-For a one-time input transform before reference presets and model processing,
-including cropping, padding, and resizing algorithms, see [Image preprocessing](./image_preprocessing.md).
-
-Different DiT-based editing models require different configurations to process reference images correctly (e.g., whether to use a Vision Language Model (VLM) encoder or pass VAE-encoded images directly to the DiT).
-
-To simplify this, we provide **Presets**. By default, the system automatically selects the best preset based on the model architecture. However, you can override this using the `--ref-image-args` argument.
-
-### Usage
-The `--ref-image-args` argument accepts a comma-separated list of key-value pairs:
-
-**Using a preset:**
-`--ref-image-args "preset=qwen_layered"`
-
-**Using a preset with a specific override:**
-`--ref-image-args "preset=krea2_edit,force_ref_timestep_zero=true"`
-
-### Available Presets
-
-| Preset | Primary Use Case |
-| :--- | :--- |
-| `flux_kontext` | FLUX.1 Kontext |
-| `longcat` | LongCat Image Edit |
-| `flux2` | FLUX.2 models |
-| `fibo2` | Fibo 2.0 Dev / Edit Turbo |
-| `qwen` | Qwen Image Edit |
-| `qwen_layered` | Qwen Image Layered |
-| `z_image_omni` | Boogu, Z-Image Omni |
-| `krea2_ostris_edit` | Most Krea2 Community edit LoRAs (trained with Ostris script) |
-| `mage_flow` | Mage-Flow-Edit |
-| `krea2_edit` | Specifically for [lbouaraba/krea2edit](https://huggingface.co/conradlocke/krea2-identity-edit). (or similar) |
-| `cosmos_reference` | For Anima |
-| `default` | Uses the automatic detection based on model architecture. |
-
----
-
-## Advanced Parameter Reference
-
-If presets are insufficient, you can manually configure the following parameters via `--ref-image-args`:
-
-| Key | Type | Description | Allowed Values |
-| :--- | :--- | :--- | :--- |
-| `preset` | string | Overrides the automatic preset. | (See the Presets table above) |
-| `pass_to_vlm` | bool | Whether reference images are passed to the VLM encoder. | `true`, `false` |
-| `pass_to_dit` | bool | Whether VAE-encoded references are passed directly to the DiT. | `true`, `false` |
-| `ref_index_mode` | string | Behavior of the RoPE index. | `fixed`, `increase`, `decrease` |
-| `force_ref_timestep_zero` | bool | Forces timestep=0 for reference tokens. | `true`, `false` (Krea2 only) |
-| `resize_before_vae` | bool | Whether reference images are resized before VAE encoding. | `true`, `false` |
-| `resize_vae_to_target` | bool | Use output dimensions for every reference, overriding area-based sizing. | `true`, `false` |
-| `resize_single_ref_to_target` | bool | Use output dimensions when exactly one reference is supplied. | `true`, `false` |
-| `vae_size_buckets` | bool | Choose aligned dimensions within ±10% of the requested area, minimizing aspect-ratio cropping. Ignored when using output dimensions. | `true`, `false` |
-| `vae_center_crop` | bool | Scale to cover the selected dimensions with Lanczos, then center-crop. Otherwise resize directly. | `true`, `false` |
-| `vae_input_max_pixels` | int | VAE area budget, also capped by output area in ordinary resizing. With size buckets, this is the target area and output area does not cap it. | Integer |
-| `vlm_use_vae_input` | bool | Reuse the resized/cropped VAE input for the VLM, before the VLM's own resize. Has no effect when `resize_before_vae=false`. | `true`, `false` |
-| `vlm_resize_mode` | string | How to resize VLM reference inputs. | `longest_side`, `area`, `none` |
-| `vlm_max_size` | int | Maximum VLM input size; interpreted according to `vlm_resize_mode`. | Integer |
-| `vlm_min_size` | int | Minimum VLM input size; interpreted according to `vlm_resize_mode`. | Integer |
-| `vlm_size` | int | Shortcut to set both VLM min and max size to the same value. | Integer |
-| `ref_image_size` | string | MiniMax-H3 reference image sizing before VAE encoding. `match` scales each reference down to the generation's pixel area; `max` uses a 2048 px short edge. Only references above the limit are downscaled; each axis is then snapped to the nearest 32 px multiple (an axis of at least 16 px grows by at most 16 px, and references are never narrower than 32 px). | `match`, `max` |
-
-`ref_image_size` applies to `vid_gen` with MiniMax-H3 Ref2VA. Reference latents are attended to at every sampling step, so `max` can be several times slower.
-
-### Preset Default Values
-
-For a technical overview of how each preset is configured, see the table below.
-
-| Preset | VLM | RoPE Index | Cond Resize | Special Notes |
-| :--- | :---: | :---: | :---: | :--- |
-| `flux_kontext` | No | `fixed` | `none` | |
-| `longcat` | Yes | `fixed` | `area` | |
-| `flux2` | No | `increase` | `none` | |
-| `fibo2` | Yes | `increase` | `longest` | Shared reference crops; clean reference timesteps; see [Fibo 2.0](../models/fibo2.md) |
-| `qwen` | Yes | `increase` | `area` | |
-| `qwen_layered` | Yes | `decrease` | `area` | |
-| `mage_flow` | Yes | `increase` | `longest` | `vlm_max_size = 384`, VAE input resized to target |
-| `z_image_omni` | Yes | `fixed` | `area` | |
-| `krea2_ostris_edit`| Yes | `increase` | `area` | `force_ref_timestep_zero = true` |
-| `krea2_edit` | Yes | `increase` | `longest` | `vlm_size = 768` |
-| `cosmos_reference` | No | `fixed` | `none` | `resize_before_vae = false` |
-
-**Additional Default Notes:**
-- **VLM Input Sizes:** For most presets, `vlm_max_size` and `vlm_min_size` are set to `-1`, meaning the values are model-dependent and handled automatically. In `area` mode they represent pixel area; in `longest_side` mode they represent a side length in pixels.
-- **VAE Input Size:** `vae_input_max_pixels` defaults to $1024 \times 1024$ pixels (`1048576`).
-- **VAE Geometry:** `resize_vae_to_target` defaults to `true` for Mage-Flow and LLaDA-Image. Fibo enables `resize_single_ref_to_target`, `vae_size_buckets`, and `vae_center_crop`. These options are otherwise `false`. Bucket dimensions use the model's alignment (normally 16, Qwen Image 32), with a minimum side of four alignment units; if no bucket fits, output dimensions are used.
-- **Shared Encoder Input:** `vlm_use_vae_input` defaults to `true` for Fibo and Qwen Image 2.1, otherwise `false`. All these options can be overridden individually after selecting a preset.
+# Image editing
+
+Reference images can guide identity, appearance, layout and instruction editing.
+Pass them with `-r IMAGE` in the CLI or `ref_images` in the C API. The generated
+DGL follows the selected model's default reference policy.
+
+`--img-cfg-scale` (`sample.guidance.img_cfg` in the C API) controls image guidance
+for inpainting, UNet editing and supported reference-latent models such as
+FLUX, Qwen Image Edit, LongCat and Mage Flow. A value other than 1 uses separate
+positive, negative-with-image and negative-without-image predictions. The
+generated DGL preserves this split during hires refinement. Models or inputs
+that do not support image guidance report an error.
+
+## Model defaults
+
+| Model | Reference handling |
+| --- | --- |
+| FLUX.1 Kontext | VAE reference latents with fixed position indexing |
+| FLUX.2 / FLUX.3 Klein | VAE reference latents with increasing position indexing |
+| Qwen Image Edit | Vision inputs and VAE reference latents |
+| Qwen Image 2.1 | Vision inputs derived from the VAE reference pixels |
+| Qwen Image Layered | References use decreasing position indexing |
+| Fibo 2.0 | Shared center crops for VAE and vision encoding; multiple references use aspect-ratio buckets |
+| [Anima Edit](../models/anima.md) | Matching community editing LoRA; VAE reference latents |
+| [Krea2 Edit](../models/krea2.md) | Matching community editing LoRA; vision inputs and VAE reference latents |
+| LLaDA-Image | Reference pixels go to the text encoder's native image components and reference latents to the diffusion model |
+| MiniMax-H3 | Aspect-preserving references aligned to the model grid |
+
+See the [model guides](../models/README.md) for required components and supported
+editing checkpoints. Select community editing LoRAs separately; reference-image
+inputs and editing recipes do not load them automatically.
+Instruct-Pix2Pix and CosXL-Edit also use `-r` for their source
+image; they construct concatenated image conditioning rather than DiT reference
+token sequences.
+
+## Custom reference processing
+
+Reference-policy overrides are not native generation parameters. Write a DGL
+program to choose resizing, cropping, model inputs and reference position rules.
+`--ref-image-args` and the C API `ref_image_args` key have been removed.
+
+For example, a script can prepare its VAE input explicitly:
+
+```dgl
+source = image::load("reference.png")
+pixels = image::to_f32(tensor::interpolate(source, size = [1024, 768], axes = [1, 2]))
+reference = vae_to_dit_latents(vae, vae_encode(vae, pixels, rng = rng))
+condition = encode_prompt(text, prompt, media = [pixels])
+pos = predictor(dit, condition, ref_latents = [reference], ref_index_mode = "increase")
+```
+
+These lines belong inside a complete DGL program with explicit model and RNG
+loading. Start from the [Qwen Image 2.1 edit example](../dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl).
+Vision and VAE inputs can be prepared separately when a workflow needs different
+reference images on those paths. Required model alignment and encoder-native
+preprocessing still apply.
diff --git a/docs/guides/generation_metadata.md b/docs/guides/generation_metadata.md
index f24bda2..9cdaab6 100644
--- a/docs/guides/generation_metadata.md
+++ b/docs/guides/generation_metadata.md
@@ -1,166 +1,98 @@
# Generation results and metadata
-`sd_generate_image()`, `sd_generate_video()`, `sd_adetail()`, `sd_upscale()` and
-`sd_upscale_video()` return opaque `sd_result_t` objects containing generated media
-and generation metadata. See the [C API guide](c_api.md) for option creation and
-migration from the removed parameter/result structures.
-
-```c
-sd_img_gen_opt_t* opt = sd_img_gen_opt_create();
-sd_result_t* result = NULL;
-if (opt && sd_opt_set_string(opt, "prompt", "a small house") == SD_STATUS_OK &&
- sd_generate_image(ctx, opt, &result) == SD_STATUS_OK) {
- for (size_t i = 0; i < sd_result_get_image_count(result); ++i) {
- const sd_tensor_t* image = sd_result_get_image(result, i);
- const char* info = sd_result_get_generation_info(result, i);
- /* Consume image and info together. */
- }
-}
-sd_result_free(result);
-sd_opt_free(opt);
-```
-
-`result_out` is required and set to `NULL` on failure. Successful results own
-their pixels, audio and UTF-8 JSON independently of the generation context.
-Video results expose effective FPS and frame count through accessors; each valid
-frame index returns the shared video generation record. Result views are borrowed
-until `sd_result_free()`. Use `sd_tensor_retain()` to keep media independently,
-and release the retained handle with `sd_tensor_free()`. Free functions accept `NULL`.
-
-## Reproduction parameters
-
-The `sdcpp.generation/v1` JSON is a snapshot of generation parameters for the
-same loaded context and the same input images, masks, reference media and other
-assets. It is created after `GenerationRequest` resolves the request, before
-preprocessing or sampling. A failed call returns no result or metadata.
-
-The JSON schema retains its established generation field names: `prompt`, `negative_prompt`,
-`width`, `height`, `seed`, `sample_params`, `hires`, `cache`,
-`vae_tiling_params`, `image_preprocess`, and the image/video-specific options.
-`kind` distinguishes image and video parameters. There is no inferred `task`.
-Enums use their API string names. Seeds are decimal strings to preserve their
-full integer range. Omitted optional fields retain the parameter initializer's
-defaults, including automatic values represented by infinity in the C API.
-
-Disabled optional features are omitted entirely: `cache`, `hires`, VAE tiling,
-zero/nonpositive `shifted_timestep`, inactive SLG, false circular axes, and empty
-preprocessing/reference arguments. SLG layers remain present when `slg_uncond`
-is enabled even with zero SLG scale. Enabled caches store their mode's parameters;
-TaylorSeer retains the shared DBCache thresholds and limits used by the runtime.
-Image CFG at its neutral value of 1 is omitted and resolves back to 1 by default.
-
-Image strength, ControlNet/IP-Adapter strength and PhotoMaker settings are stored
-when their corresponding input images are supplied. PuLID requires an embedding
-path. Layer count is stored for layered generation; video VACE and high-noise
-settings require those models. High-noise settings omit scheduler, custom sigmas,
-flow shift and timestep shifting because that pass shares the base configuration.
-Explicit nonempty argument strings are retained without rewriting their contents.
-Core sampling values such as seed, sampler, scheduler, steps and guidance remain
-explicit even when equal to their defaults. ADetailer always retains its inpaint
-strength because its input crops are created later.
-Model-required VAE tiling is resolved before serialization, so forced tiling
-(such as MiniMax-H3) remains present even if the caller did not enable it.
-
-The request fixes the seed, dimensions, clip skip, guidance, sampler, scheduler
-and eta before taking the snapshot. `custom_sigmas` is stored only when supplied
-as an input. Automatically computed schedules, preprocessing transforms,
-conditioning/cache decisions, detector regions and output statistics are not
-stored: they are recomputed from the same parameters and inputs.
-
-Each image records the seed used to sample its latent. `batch_count` is omitted;
-replay image parameters with `batch_count = 1`. Layers or frames decoded from
-the same latent share its seed, so layered generation reproduces that group of
-layers. Video frames share the video request. Partial batch cancellation returns
-metadata only for completed outputs. Full cancellation returns no result.
-
-The current image pipeline prepares input latents once using the starting seed
-and shares them across a batch. With a stochastic VAE encoder, replaying a later
-image alone using its sampling seed can change those input latents. Per-image
-seeds therefore do not guarantee independent reproduction for these paths.
-
-To replay a record, create image/video options, translate JSON fields to the documented C API
-keys, restore the same input buffers, and call generation with the same context.
-Setters copy strings and numeric arrays and retain media tensors; keep options alive
-for the call. Paths supplied as generation parameters (LoRAs, hires models and identity
-embeddings) are retained. `models` identifies successfully loaded context model
-files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using
-filenames with extensions and omitting directories. Optional components disabled
-for the request and preview-only TAE are omitted. WebUI text includes `Model`
-(the diffusion model when supplied separately, otherwise the checkpoint, without
-the extension) and `VAE` for a separate active VAE. These names are identifiers,
-not hashes or sufficient configuration to rebuild a context; replay still requires
-the same model files and context settings. The JSON does not embed media buffers
-or model weights.
-
-When `--lora-model-dir` is configured, CLI/server output records express LoRA
-paths relative to that directory, preserving subdirectories and file extensions
-and using `/` separators. This applies to file metadata, API responses and
-ADetailer records. Resolve these paths against the same LoRA directory before
-passing them back to the C API. Core results retain the supplied loading paths.
-Files outside the directory retain `..` components; paths on another volume,
-or records without a configured LoRA directory, retain their original paths.
-
-## Postprocessing
-
-Upscale records `upscale.upscale_factor`. ADetailer records `adetailer` and
-`inpaint_params`, with a concrete starting seed chosen before detection. These
-operations require the same upscaler/detector context and source image on replay.
-They do not store detection results or per-region generation traces.
-
-The CLI/server append operation parameter snapshots to `postprocess` in call
-order. The source generation parameters remain unchanged. Frame postprocessing
-uses the same operation parameters for each video frame. Standalone
-postprocessing does not invent source diffusion parameters for an input image.
+`sd_generate_image()`, `sd_generate_video()` and `sd_upscale()` all execute dgl.
+`sd_result_get_generation_info(result, index)` returns the exact UTF-8 program
+submitted to the DGL compiler. It is not a JSON parameter snapshot.
+
+Debug logging also prints the actual compiled program immediately before
+execution. Use `--log-level debug` in the CLI. Each image in a batch
+logs its own resolved program and seed. Source logging is independent of file
+metadata embedding; tensor bindings remain separate inputs and are not dumped.
+
+## Results and rerunning
+
+Each generated image records its actual seed and executed source. Image batches
+execute individual programs in the same context, reusing loaded models. Multiple
+layers decoded from one latent share the same source. A video records one
+program for the entire sequence, returned for every valid frame index.
+Partial batch cancellation returns records only for completed images; full
+cancellation returns no result.
+
+Rerun a self-contained program through `sd-cli -M dgl --script FILE` or the
+DGL C API. Programs with `fn main(input_image, ...)` require the original host
+tensor bindings through the DGL C API. Generated programs retain component paths,
+resolved options and ordinary LoRA application order. They do not embed model
+weights or input media. Keep the same files, inputs and backends when comparing
+numerical results.
+
+Standalone image upscaling records the complete DGL operation, including every
+requested repeat. No source diffusion settings are invented for an imported
+image. There is no postprocessing JSON merge or separate replay mode.
+
+## WebUI infotext
+
+CLI and server derive WebUI infotext from the DGL source on a best-effort basis.
+Known image flows can expose prompts, seed, dimensions, sampler, scheduler,
+steps, CFG, strength, components and ordinary LoRAs. Ambiguous multi-stage fields
+are omitted. Conversion never changes the executed source or causes inference
+to fail. Standalone upscaling can have empty infotext.
+
+`sd_get_generation_metadata(source)` returns UTF-8 JSON containing these
+parameters and an `infotext` field in one call. It does not execute the program
+or read model files. Unrecognized image flows return `{}`. The returned string
+is borrowed until the next call on the same thread; copy or parse it before then.
+A null result indicates an API error available through `sd_get_last_error()`.
+
+Native server `generation_info` values are DGL strings. WebUI-compatible routes
+also fill `infotexts` separately. Clients must not parse the DGL string as JSON.
## File metadata
-Image encoders accept the native record and derive WebUI infotext from it.
-Images use the existing metadata locations and combined text layout:
-`<WebUI infotext>, SDCPP: <native JSON>`. Standalone postprocessing without
-source diffusion parameters stores only native JSON. Video metadata stores
-only native JSON.
-
-| Container | Metadata location |
-| --- | --- |
-| PNG | `tEXt`, keyword `parameters` |
-| JPEG | COM segment, `parameters` label followed by a NUL byte |
-| WebP image / animated WebP | XMP property `sdcpp:parameters` |
-| AVI | RIFF INFO `ICMT` |
-| WebM | `COMMENT` tag |
-
-Inspect images or videos with `sd-cli -M metadata --image <file>`; the same
-command supports all containers above, with `--metadata-format json` for a
-structured report. Video metadata inspection does not require a model or video
-decoder.
-Matroska containers are also supported, including `.webm` outputs whose audio
-codec causes the muxer to select the Matroska document type.
-
-The XMP namespace is `https://github.com/leejet/stable-diffusion.cpp/ns/1.0/`.
-PNG and JPEG retain the existing UTF-8 text payload convention. Memory encoding
-and file saving use the same implementation. The core result and API
-`generation_info` fields remain pure JSON; the combined layout applies only
-to embedded image metadata.
-
-WebUI conversion is lossy: known samplers and schedules have explicit name
-mappings; unsupported ones use `SDCPP Sampler`/`SDCPP Schedule` fields. Hires
-keeps the base request size and separate configured hires size/steps. The native
-record retains the full generation parameters. Standalone postprocessing has no WebUI
-infotext because its original diffusion settings are unknown.
-
-WebUI prompt text appends ordinary LoRAs as `<lora:filename_without_extension:weight>`.
-Directory components are omitted to match WebUI's filename lookup; the native
-`loras` array retains relative paths and weights. High-noise-only LoRAs and names
-that cannot be represented by this tag syntax remain in the native record only.
-The native prompt is not modified, and postprocessing LoRAs are not added to the
-source generation prompt.
-
-JPEG requires the COM payload, including the label and terminating NUL bytes,
-to fit within 65,533 bytes. Larger records fail encoding with an error rather
-than being truncated. Use PNG or WebP for larger records.
-
-Disabling file metadata omits both image representations, but does not disable
-the core generation record or its API response. Native async image results and
-OpenAI image response items contain `generation_info`; video results contain a
-sequence-level `generation_info`. The SD API `info` object contains per-output
-`generation_info` alongside derived `infotexts` and each image's seed. Its outer
-`parameters` field continues to echo the request.
+The CLI and server image writers store native DGL and WebUI text separately:
+
+| Container | DGL source | WebUI text |
+| --- | --- | --- |
+| PNG | UTF-8 `iTXt`, keyword `dgl` | UTF-8 `iTXt`, keyword `parameters` |
+| JPEG | COM segment labeled `dgl` | COM segment labeled `parameters` |
+| WebP / animated WebP | XMP `sdcpp:dgl` | XMP `sdcpp:parameters` |
+| AVI | RIFF INFO `ICMT` | Not stored separately |
+| WebM / Matroska | `COMMENT` tag | Not stored separately |
+
+Video comments contain the DGL program. The XMP namespace is
+`https://github.com/leejet/stable-diffusion.cpp/ns/1.0/`.
+JPEG labels and values are separated by NUL bytes. A JPEG COM payload is limited
+to 65,533 bytes including labels and terminators; an oversized record makes
+encoding fail rather than silently truncating the program. Other container
+capacity checks follow the same rule.
+
+Memory encoding and file saving use the same metadata writer. Disabling metadata
+still disables embedding. Previously generated files with combined WebUI text
+and `SDCPP:` JSON remain readable.
+
+DGL `image::save` (PNG) and `video::save` (AVI) also embed the exact program
+source by default, using `dgl` and `ICMT` respectively. `image::encode` and
+`video::encode` embed it during encoding; `media::save` writes those bytes
+unchanged. WebM uses the `COMMENT` tag. These script functions store the program
+itself without deriving WebUI infotext. Set the DGL context
+option `embed_metadata` to `off` to omit it; `sd-cli -M dgl` forwards
+`--disable-image-metadata` to that option. The switch also disables embedding
+in the ordinary CLI image and video output paths. It does not affect tensor
+data or the source returned by `sd_result_get_generation_info`.
+
+Inspect metadata with `sd-cli -M metadata --image FILE`, optionally adding
+`--metadata-format json` for a structured report. This command reads container
+metadata without loading a model.
+
+The C API provides the same reader through `sd_read_metadata(path, flags)`.
+Use zero for the default text report, or combine `SD_METADATA_READ_JSON`,
+`SD_METADATA_READ_RAW`, `SD_METADATA_READ_BRIEF` and
+`SD_METADATA_READ_STRUCTURAL`. A null return indicates an error; inspect
+`sd_get_last_error()`. The returned UTF-8 report is borrowed until the next
+call to this function on the same thread.
+
+For applications that encode their own files, `sd_get_metadata_chunk()` builds
+PNG iTXt chunks, JPEG COM segments, XMP text or an AVI INFO chunk. It returns
+borrowed bytes and their exact size; copy them before the next call on the same
+thread. PNG chunks are inserted after IHDR, JPEG segments after SOI, XMP is
+stored in the container's XMP field, and AVI INFO is a RIFF chunk. Empty source
+returns an empty chunk. This function does not write a file.
diff --git a/docs/guides/image_preprocessing.md b/docs/guides/image_preprocessing.md
index dc0aca8..4375959 100644
--- a/docs/guides/image_preprocessing.md
+++ b/docs/guides/image_preprocessing.md
@@ -1,180 +1,29 @@
# Image preprocessing
-Use `--image-preprocess` to transform each image input once, before generation:
-
-```sh
-sd-cli ... \
- --image-preprocess "target=init,mode=crop-resize,filter=lanczos,antialias=true" \
- --image-preprocess "target=mask,filter=nearest-exact" \
- --image-preprocess "target=ref,index=0,mode=fit-pad,width=768,height=768,filter=bicubic"
+Custom input transformations are expressed in [DGL](../dgl.md). Native CLI and
+C API generation use the model's default canvas and reference-image rules.
+The former `--image-preprocess` flag, server `image_preprocess` field and C API
+`image_preprocess.rules` option have been removed. Express their transformations
+in DGL instead. `--ref-image-args` and `ref_image_args` have also been removed.
+
+Use tensor operations to make the transformations visible in the program:
+
+```dgl
+image = image::load("input.png")
+cropped = tensor::crop(image, start = [32, 16], size = [512, 384], axes = [1, 2])
+resized = tensor::interpolate(cropped, size = [768, 512], axes = [1, 2], mode = "lanczos")
+pixels = image::to_f32(resized)
```
-CLI and server image loaders decode at the original resolution. The generation
-entry point merges input defaults with user rules and prepares one transformed
-image per input. The original pipeline then consumes those images, including
-its mandatory canvas adaptation, reference resizing, and encoder preprocessing.
-
-```text
-native-resolution image
- -> input defaults + user overrides
- -> one input transform
- -> original generation pipeline and model-specific processing
-```
-
-These rules do not override internal VAE, CLIP/VLM, ControlNet, or pixel-patch preprocessing.
-`--ref-image-args` retains its existing meaning and runs after this input transform.
-
-## Inputs and defaults
-
-| `target` | Input | Default geometry | Indexed? |
-| --- | --- | --- | --- |
-| `init` | img2img image or video first frame | Center crop to the generation aspect ratio, then resize | No |
-| `end` | Video last frame | Center crop, then resize | No |
-| `mask` | Inpainting mask | Inherit init geometry; otherwise center crop, then resize | No |
-| `control` | Control image | Center crop, then resize | No |
-| `ref` | Reference images | Preserve source dimensions | Yes |
-| `ip-adapter` | IP-Adapter image | Preserve source dimensions | No |
-| `id` | PhotoMaker identity images | Preserve source dimensions | Yes |
-| `control-frame` | Control video frames | Center crop, then resize | Yes |
-
-Canvas defaults use the aligned generation dimensions. Reference, IP-Adapter,
-and identity inputs use their original dimensions unless overridden. Default
-resampling is nearest for images and nearest-exact for masks.
-
-These defaults are shared by CLI, server, and C API. Moving geometry out of
-the loaders replaces the previous CLI/server BOX/sRGB resizing, so default
-pixels are not guaranteed to match earlier builds.
-
-Reference video and audio preprocessing are outside these image rules.
-Preprocessing options apply to `img_gen` and `vid_gen`, not standalone upscale
-or ADetailer mode. ADetailer clears the user's rules for its internal crops.
-
-## Rules
-
-Rules are comma-separated `key=value` lists. Repeat the CLI option or separate
-rules with semicolons. Every rule requires a `target` and at least one option.
-Rule syntax and input compatibility are checked when image/video generation
-starts. Unknown keys, invalid values, duplicate keys in a rule, missing images,
-and out-of-range indices cause generation to fail with an error log.
-
-Omit `index` to configure every image of that type; otherwise use a zero-based
-index. CLI directory inputs follow filename order. Indexed rules override
-type-wide rules field by field, regardless of order. At equal specificity,
-the last value for a field wins. `auto` selects the input preset.
-
-| `mode` | Input transform |
-| --- | --- |
-| `auto` | Use the input's default geometry |
-| `none` | Keep source dimensions without resizing, cropping, or padding |
-| `stretch` | Resize to the target dimensions |
-| `crop` | Crop a target-sized rectangle without resizing; fail if the source is too small |
-| `crop-resize` | Crop to the target aspect ratio, then resize |
-| `fit-pad` | Fit the entire image inside the target dimensions, preserving aspect ratio, then pad |
-
-`width` and `height` must be specified together as positive integers. They
-override the input transform's dimensions, not the generation or encoder size.
-For a native-size preset, specifying dimensions without a mode selects stretch.
-`mode=none` with explicit dimensions different from the source is contradictory
-and is rejected.
-
-`anchor=center|top|bottom|left|right` selects crop/padding placement.
-`pad_color=#RRGGBB` or `#RRGGBBAA` selects padding, defaulting to opaque black.
-A grayscale mask uses the first color component.
-
-`filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos` selects resampling.
-`antialias=auto|true|false` enables antialiasing automatically for filtered
-downscaling; explicit true requires bilinear, bicubic, or Lanczos.
-Filtered RGBA resizing uses premultiplied alpha.
-
-`canny=true|false` enables edge detection for any supported image target,
-defaulting to `false`. It runs once after geometry, before the original
-generation pipeline, including with `mode=none`. Grayscale, grayscale-alpha,
-RGB, and RGBA inputs are supported; alpha is preserved.
-
-Each input has its own Canny setting. Indexed rules can enable or disable it
-for individual references, identity images, or video control frames.
-
-```sh
---image-preprocess "target=init,mode=fit-pad,canny=true"
---image-preprocess "target=ref,index=0,mode=none,canny=true"
---image-preprocess "target=control-frame,index=2,canny=true"
-```
-
-If init and mask source dimensions differ, the mask is first center-cropped to
-the init source aspect ratio and resized to the init source dimensions using
-nearest-exact interpolation. The mask then inherits the init crop, resize, and
-padding coordinates, while retaining its own filter, padding value, and Canny
-setting for that transform. Explicit mask geometry is evaluated after this
-alignment; conflicting geometry is rejected. An omitted mask remains absent
-until the original pipeline creates its default mask.
-
-## Downstream behavior
-
-`mode=none` only skips the input geometry transform. For example:
-
-```sh
---image-preprocess "target=init,mode=none" \
---image-preprocess "target=ref,mode=none"
-```
-
-The init image is still adapted to the generation canvas by the original
-pipeline. Reference images still follow `--ref-image-args` and model-specific
-resizing. CLIP retains its fixed input dimensions and normalization. HiDream-O1
-retains its original pixel-reference and visual preprocessing.
-
-Existing sharing between consumers is preserved: for example, Wan img2video
-uses the same adapted first frame for VAE conditioning and CLIP. High-resolution
-passes reuse the prepared images and apply their original size adaptation;
-they do not apply the user's crop a second time.
-
-To disable reference resizing before VAE encoding, use
-`--ref-image-args "resize_before_vae=false"` or the server field
-`"ref_image_args": "resize_before_vae=false"`. This is separate from
-`target=ref,mode=none`, which only skips input geometry. Model constraints
-still apply; MiniMax-H3 requires 32-pixel-aligned references and warns instead
-of honoring the switch.
-
-## Server requests
-
-Native image/video requests and SDAPI accept `image_preprocess` as a string or
-an array of rule strings:
-
-```json
-{
- "image_preprocess": [
- "target=init,mode=fit-pad,filter=bicubic",
- "target=mask,filter=nearest-exact",
- "target=ref,index=0,mode=none"
- ]
-}
-```
-
-OpenAI-compatible requests accept it through
-`<sd_cpp_extra_args>{...}</sd_cpp_extra_args>` in the prompt.
-Request rules replace server-default rules. Generation metadata records the
-user rules for replay with the same input images; image encodings and channel conventions are unchanged.
-
-## C API
-
-Set the `image_preprocess.rules` key on image/video options. Generation returns
-media and actual parameters together; see [generation metadata](generation_metadata.md):
-
-```c
-sd_img_gen_opt_t* opt = sd_img_gen_opt_create();
-sd_result_t* result = NULL;
-/* Set prompt, original-resolution input images, and generation options. */
-sd_status_t status = sd_opt_set_string(opt, "image_preprocess.rules",
- "target=init,mode=crop-resize,filter=lanczos;target=mask,filter=nearest-exact");
-if (status == SD_STATUS_OK)
- status = sd_generate_image(ctx, opt, &result);
-/* Read outputs through sd_result_get_*() on success. */
-sd_result_free(result);
-sd_opt_free(opt);
-```
+For a mask, use `mode = "nearest_exact"` when resizing and explicitly convert to
+the expected mask layout. For an edge hint, use `image::canny(image)` and pass
+the result to the ControlNet workflow. `tensor::pad`, `tensor::crop` and
+`tensor::interpolate` can express fitting, padding and aspect-ratio handling.
-Leaving the key unset selects input presets. Rule strings are copied by setters.
-The library owns temporary transformed pixels; caller images and arrays are
-not modified. Add `canny=true` to the desired target's rule in
-`image_preprocess.rules` to enable Canny.
+These operations do not disable mandatory VAE or vision-encoder constraints.
+Prepare an input that already satisfies those constraints to avoid automatic
+alignment. Different encoders may receive different prepared tensors.
-The parameter structs have grown; applications and bindings must be rebuilt.
+See the [editing guide](edit.md), the
+[ControlNet example](../dgl/sd/sd1-5-controlnet.dgl), and the
+[built-in catalog](../dgl-builtins.md) for complete interfaces.
diff --git a/docs/guides/int8_convrot.md b/docs/guides/int8_convrot.md
index c105f15..54f36b6 100644
--- a/docs/guides/int8_convrot.md
+++ b/docs/guides/int8_convrot.md
@@ -1,5 +1,9 @@
# INT8 Convrot Safetensors
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` metadata directly. The stored INT8 weights are not converted to another weight type at load time.
This uses sd.cpp's INT8 tensorwise/convrot operators through the patched GGML
diff --git a/docs/guides/performance.md b/docs/guides/performance.md
index 76fb3da..40b7bac 100644
--- a/docs/guides/performance.md
+++ b/docs/guides/performance.md
@@ -1,5 +1,9 @@
## Use Flash Attention to save memory and improve speed.
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
Enabling flash attention for the diffusion model reduces memory usage by varying amounts of MB.
eg.:
- flux 768x768 ~600mb
@@ -49,6 +53,11 @@ run qualifies for it.
Use `--batched-cfg off` to force separate conditional and unconditional
forwards.
+In DGL mode the flag sets `batched_cfg` on the execution context. Compatible
+`cfg(positive, negative, ...)` branches use batching automatically; combinations
+that require separate forwards retain their original execution. See
+[DGL context execution settings](../dgl.md#context-execution-settings).
+
## Use VAE tiling to reduce encode and decode memory usage.
`--vae-tiling` enables spatial tiling for both VAE encoding and decoding. The
@@ -187,6 +196,7 @@ sd-cli --diffusion-model flux1-dev.safetensors ... \
--offload-to-cpu --max-vram -1
```
+
- `--offload-to-cpu`: params in RAM, staged as needed.
- `--max-vram -1`: reserve about 1 GiB from the startup free-memory snapshot; live free memory can still lower the effective limit for every graph.
diff --git a/docs/guides/prompt_images.md b/docs/guides/prompt_images.md
index e6c301f..69dce1b 100644
--- a/docs/guides/prompt_images.md
+++ b/docs/guides/prompt_images.md
@@ -75,8 +75,36 @@ within one prompt are loaded once and appear at every specified position.
Resolved image pixels participate in conditioning-cache comparisons.
Register or clear the callback while the context is idle. Different contexts
-have independent callbacks. Generation parameter structures and generation
-entry points are unchanged.
+have independent callbacks.
+
+## DGL
+The complete [MOSS-Image prompt-image example](../dgl/moss_image/moss-image-sft-inline-images.dgl)
+loads all components, binds the prompt image and saves the generated result.
+See the [DGL example collection](../dgl/README.md) for run instructions.
+
+Use the same markers with `encode_prompt`. DGL reads image paths relative to the
+working directory.
+LLM encoders accept `system` to customize the template while retaining image
+inputs. `raw=true` is the complete text-tokenizer-input path and cannot be
+combined with image/video bindings or `system`. See
+[prompt formatting](../dgl.md#independent-generation-components).
+
+```text
+condition = encode_prompt(encoder, "Draw <image:inputs/subject.png> in watercolor.")
+```
+
+To compose images inside the graph, pass a dictionary of tensors with keys
+matching the markers. Explicit bindings take precedence over opening a path.
+Only referenced bindings enter conditioning; additional dictionary entries do
+not add images. A referenced binding must contain a valid image tensor. If a
+name is absent, the normal file reader and permission callback still apply,
+including when the encoder has cached a previous result.
+
+```text
+subject = image::load("inputs/subject.png")
+condition = encode_prompt(encoder, "Draw <image:subject> in watercolor.",
+ image_bindings = {subject = subject})
+```
## Server
diff --git a/docs/guides/rpc.md b/docs/guides/rpc.md
index 6edd61d..bbd78f8 100644
--- a/docs/guides/rpc.md
+++ b/docs/guides/rpc.md
@@ -1,5 +1,9 @@
# Building and Using the RPC Server with `stable-diffusion.cpp`
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
This guide covers how to build a version of [the RPC server from `llama.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/tools/rpc/README.md) that is compatible with your version of `stable-diffusion.cpp` to manage multi-backends setups. RPC allows you to offload specific model components to a remote server.
> **Note on Model Location:** The model files (e.g., `.safetensors` or `.gguf`) remain on the **Client** machine. The client parses the file and transmits the necessary tensor data and computational graphs to the server. The server does not need to store the model files locally.
@@ -165,6 +169,7 @@ Example: Setting the main backend to the RPC0 device for doing all the work on t
./sd-cli -m models/sd1.5.safetensors -p "A cat" --rpc-servers localhost:50052 --backend RPC0
```
+
---
## 5. Scaling: Multiple RPC Servers
@@ -211,6 +216,7 @@ Pass multiple server addresses separated by commas.
./sd-cli --rpc-servers 192.168.1.10:50052,192.168.1.10:50053,192.168.1.11:50052 [...]
```
+
The client will map these servers to sequential device IDs (e.g., RPC0 from the first server, RPC2, RPC3 from the second, and RPC4 from the third). With this setup, you could for example use RPC0 for the main backend, RPC1 and RPC2 for the text encoders, and RPC3 for the VAE.
---
diff --git a/docs/guides/sage_attention.md b/docs/guides/sage_attention.md
index 2f3a0d0..9243d15 100644
--- a/docs/guides/sage_attention.md
+++ b/docs/guides/sage_attention.md
@@ -1,5 +1,9 @@
# SageAttention
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
`--sage-attn` enables native CUDA SageAttention in the diffusion model, including
the high-noise diffusion model when present. Python, PyTorch, and Triton are not
required at build time or runtime.
@@ -42,10 +46,11 @@ Add `--sage-attn` to enable SageAttention. For example,
from the build directory:
```powershell
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,
形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --sage-attn
```
+
SageAttention currently handles unmasked attention with head dimensions from
1 through 128, including grouped-query attention, different query/key lengths,
and multiple batches. Dimensions below 64 are zero-padded to 64; dimensions
diff --git a/docs/guides/sampling.md b/docs/guides/sampling.md
index 7955ce1..d89ecfa 100644
--- a/docs/guides/sampling.md
+++ b/docs/guides/sampling.md
@@ -1,5 +1,9 @@
# Sampling methods and schedulers
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
The sampling method controls how each denoising step updates the image or video
latent. The scheduler chooses the noise levels (sigmas) used across those steps.
Choose both according to the [model guide](../models/README.md) for your checkpoint.
@@ -31,6 +35,7 @@ For example, using the Stable Diffusion 1.5 checkpoint from the
-p "a lovely cat"
```
+
Run the example from the build directory. See the
[shared command conventions](../models/README.md#example-conventions) for build
layouts and platform-specific syntax.
@@ -40,7 +45,7 @@ layouts and platform-specific syntax.
| `--sampling-method NAME` | Select the sampling method. |
| `--scheduler NAME` | Select the sigma scheduler. |
| `--steps N` | Set the number of sampling steps. |
-| `--high-noise-sampling-method NAME` | Select the high-noise sampler for two-stage video models. |
+| `--high-noise-steps N` | Set the high-noise stage length for two-stage video models. |
| `--sigmas VALUES_OR_PRESET` | Supply a custom schedule instead of generating one with the scheduler. |
| `--hires-sigmas VALUES_OR_PRESET` | Supply a custom schedule for the highres fix second pass. |
@@ -51,6 +56,10 @@ For native server requests, use `sample_params.sample_method` and
`sample_params.scheduler`. See the [server API reference](../../examples/server/api.md)
for request fields and capability discovery.
+Native two-stage video generation shares sampling and guidance settings between
+the high- and low-noise stages. Use a DGL workflow to give each stage a different
+sampler or guidance configuration.
+
## Sampling methods
| Method | CLI value |
diff --git a/docs/guides/tokenizers.md b/docs/guides/tokenizers.md
index fb93550..3d4e3f9 100644
--- a/docs/guides/tokenizers.md
+++ b/docs/guides/tokenizers.md
@@ -1,5 +1,9 @@
# JSON tokenizers
+DGL versions of the generation examples are linked below. Edit each script's
+`var` block and follow the [DGL run instructions](../dgl/README.md).
+
+
Use a Hugging Face `tokenizer.json` to supply the tokenizer vocabulary, merges,
added tokens, and processing stages. **PiD (including PiD 1.5) and Lens (including
Lens Turbo) require an external JSON**; their Gemma 2 and GPT-OSS tokenizers are
@@ -11,6 +15,7 @@ sd-cli --diffusion-model model.gguf --llm text_encoder.gguf \
--tokenizer tokenizer_gemma2.json --vae vae.safetensors -p "a cat"
```
+
Choose the JSON belonging to the text encoder checkpoint. Checking that IDs fit
the embedding table does not establish that two vocabularies have the same
meaning. The JSON file is loaded when the text encoder is created; its embedded
@@ -52,6 +57,7 @@ sd-cli --diffusion-model sd3.gguf --clip_l clip_l.safetensors \
-p "a cat"
```
+
The C API accepts the same string through the `tokenizer` context-option key. An unset or
empty value keeps an embedded tokenizer where available; PiD and Lens require
a nonempty main tokenizer path. The CLI passes the string through;
diff --git a/docs/models/README.md b/docs/models/README.md
index 24da05c..06820fa 100644
--- a/docs/models/README.md
+++ b/docs/models/README.md
@@ -10,6 +10,12 @@ for guide structure, examples, and index updates.
## Example conventions
+Standalone [DGL versions of the documentation examples](../dgl/README.md)
+are grouped by model under `docs/dgl/`. Edit each script's
+`var` block and run it with `sd-cli -M dgl --script FILE.dgl`. Features currently
+available only through DGL are marked in their guides; use the server's DGL
+editor for those workflows rather than a native recipe.
+
Run the executable examples from the CMake build directory, for example after
`cd build` from the repository root. Each guide gives both command forms:
@@ -73,7 +79,7 @@ variant; a VAE or TAE selection must match that model.
- [SenseNova U1.5](sensenova_u1.md)
- [Stable Diffusion 1.x, 2.x, and SDXL, including Turbo variants](sd.md)
- [Stable Diffusion 3 / 3.5](sd3.md)
-- [Z-Image](z_image.md)
+- [Z-Image](z_image.md), including [DGL ControlNet](z_image.md#controlnet)
- [Z-Image L2P](z_image_l2p.md)
## Image editing
diff --git a/docs/models/adetailer.md b/docs/models/adetailer.md
index 2072644..54823bc 100644
--- a/docs/models/adetailer.md
+++ b/docs/models/adetailer.md
@@ -1,22 +1,23 @@
# ADetailer
-`sd-cli` can run a YOLOv8 object detector on an existing or newly generated
-image, and `sd-server` can run it after image generation. Both perform a cropped
-inpaint pass for every detected object. The implementation supports YOLOv8
+A DGL script runs a YOLOv8 object detector on an existing or newly generated
+image, then performs a cropped inpaint pass for each selected object. The implementation supports YOLOv8
detection checkpoints. YOLOv8 segmentation and MediaPipe models are not
supported yet.
+**Currently supported only through dgl.**
+
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| Base image model | [Compatible image-generation model](README.md) | `-m` / model component options | Required for the inpainting pass |
-| YOLOv8 detector | Converted Safetensors checkpoint prepared below | `--ad-model` | Required; detection checkpoints only |
+| Base image model | [Compatible image-generation model](README.md) | `load_unet`, `load_text_encoder`, `load_vae` | Required for the inpainting pass |
+| YOLOv8 detector | Converted Safetensors checkpoint prepared below | `load_detector` | Required; detection checkpoints only |
### Prepare the detector
Run the conversion command from the source repository root, then place the
-converted file at the path used by `--ad-model`.
+converted file at the script's `model_path`.
Ultralytics checkpoints must be converted before use. The converter writes the Safetensors detector used by sd.cpp.
@@ -40,176 +41,79 @@ trusted source.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+### DGL
-### CLI
+| Example | Description |
+| --- | --- |
+| [Repair an existing image](../dgl/adetailer/adetailer-existing-image.dgl) | Detect objects and inpaint selected crops. |
+| [Generate and refine](../dgl/adetailer/adetailer-generated-image.dgl) | Generate the starting image, then run the same detail pass. |
-#### Repair an existing image
+#### Run a script
-Use the dedicated `adetailer` mode to detect and repair objects in an existing
-image:
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -M adetailer `
- -m ..\models\checkpoints\model.safetensors `
- --ad-model ..\models\detectors\face_yolov8n.safetensors `
- -i input.png `
- -p "detailed portrait photo" `
- --negative-prompt "deformed face" `
- --steps 24 `
- --cfg-scale 6 `
- --strength 0.4 `
- --sampling-method dpm++2m `
- --scheduler karras `
- --extra-ad-args "confidence=0.3,inpaint_padding=32,mask_blur=4" `
- -o repaired.png
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\adetailer\adetailer-existing-image.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -M adetailer \
- -m ../models/checkpoints/model.safetensors \
- --ad-model ../models/detectors/face_yolov8n.safetensors \
- -i input.png \
- -p "detailed portrait photo" \
- --negative-prompt "deformed face" \
- --steps 24 \
- --cfg-scale 6 \
- --strength 0.4 \
- --sampling-method dpm++2m \
- --scheduler karras \
- --extra-ad-args confidence=0.3,inpaint_padding=32,mask_blur=4 \
- -o repaired.png
+./bin/sd-cli -M dgl --script ../docs/dgl/adetailer/adetailer-existing-image.dgl
```
-This mode reuses the normal image-generation options for the detail pass:
-
-- `--init-img`, `--output`, `--prompt`, and `--negative-prompt`
-- `--steps`, `--cfg-scale`, `--sampling-method`, and `--scheduler`
-- `--strength`, `--seed`, LoRA settings, VAE tiling, and backend assignments
-- `--width` and `--height`, which also resize the input when specified
-
-`--ad-prompt` and `--ad-negative-prompt` optionally override the normal prompts.
-Values provided in `--extra-ad-args`, such as `steps`, `cfg_scale`,
-`denoising_strength`, or `inpaint_width`, take precedence over inherited values.
-
-#### Repair generated images
-
-ADetailer can also run automatically after normal image generation:
-
-**Windows (PowerShell)**
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
-```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\model.safetensors `
- --ad-model ..\models\detectors\face_yolov8n.safetensors `
- -p "portrait photo" `
- --ad-prompt "[PROMPT], detailed face" `
- --ad-negative-prompt "" `
- --extra-ad-args "confidence=0.3,denoising_strength=0.4,inpaint_width=512,inpaint_height=512"
-```
+The [ADetailer script](../dgl/adetailer/adetailer-existing-image.dgl) loads a YOLOv8 detector
+and an inpainting model. Set `confidence` and `nms` on `detections::detect`, and use
+`detections::filter` to select the boxes to repaint. The script converts
+the boxes to masks, plans crops with `regions::plan_mask_crops`, then assigns prompts and
+seeds and blends each inpainted crop into the current image. See
+[DGL detail refinement](../dgl.md#detail-refinement) for the data layout.
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -m ../models/checkpoints/model.safetensors \
- --ad-model ../models/detectors/face_yolov8n.safetensors \
- -p "portrait photo" \
- --ad-prompt "[PROMPT], detailed face" \
- --ad-negative-prompt "" \
- --extra-ad-args confidence=0.3,denoising_strength=0.4,inpaint_width=512,inpaint_height=512
-```
-
-The same options can be supplied when starting `sd-server`. When `--ad-model` is set, its detector runs after OpenAI-compatible and native asynchronous image generation. SDAPI requests enable ADetailer through `alwayson_scripts`, as described below. If no object is detected, the original image is returned unchanged. RGBA output keeps its original alpha channel, including when detected regions are repaired. Detector initialization or detail-pass failures are reported as request failures.
-
-Server image-generation requests limit base, hires, and detail-pass dimensions to 4096 pixels per side. ADetailer step overrides are limited to 1–100; `steps=0` in `extra_ad_args` keeps inheriting the main generation steps. These limits also apply to inherited dimensions and `extra_ad_args` overrides. Requests exceeding the limits return a parameter error before generation. CLI limits are unchanged.
-
-### SDAPI requests
+#### Repair an existing image
-`/sdapi/v1/txt2img` and `/sdapi/v1/img2img` accept the WebUI ADetailer argument format:
+The script loads `input_path`, detects objects and repairs their crops.
-```json
-{
- "prompt": "portrait photo",
- "alwayson_scripts": {
- "ADetailer": {
- "args": [true, false, {
- "ad_model": "face_yolov8n.pt",
- "ad_prompt": "[PROMPT], detailed face",
- "ad_use_steps": true,
- "ad_steps": 20
- }]
- }
- }
-}
-```
+Edit `detail_prompt`, `negative_prompt`, `seed` and the `settings` dict in the
+script. The detector and mask operations are explicit, so filtering and crop
+geometry can be changed independently.
-The detector must be configured at server startup with `--ad-model`. `ad_model` must match that file's basename; the original `.pt` basename is also accepted for a converted detector. Requests cannot load arbitrary detector paths. Omitting the script, setting its first argument to `false`, selecting `"None"`, or setting `ad_tab_enable` to `false` disables detailing. Dictionary-only argument arrays are also accepted. Multiple enabled dictionaries run in order using the configured detector.
+#### Repair generated images
-Supported settings include prompts, confidence, mask filtering by area or confidence, mask count and area ratios, dilation/erosion, offsets, mask merging/inversion, blur, denoising strength, masked or full-image inpainting, padding, and optional overrides for inpaint dimensions, steps, CFG, sampler/scheduler, and CLIP skip. The corresponding `ad_use_*` flags control whether overrides apply. Otherwise dimensions, steps, CFG, sampler, scheduler, and CLIP skip inherit the main request. Denoising strength defaults to `0.4`. Pixel mask settings are limited to the larger output dimension, with a minimum limit of 32 and a maximum of 16384 pixels.
+This script generates the starting image, then runs the same detail pass.
-Model/VAE switching, non-default noise multipliers, face restoration, ControlNet, and skipping the initial img2img pass are not supported; active requests for these features return an error. Unknown settings and invalid active values also return errors. During detailing, `/sdapi/v1/progress` remains active with an `adetailer` job; its percentage and preview stay fixed until detailing finishes.
+If no object is detected, `fold` returns the original image unchanged.
+The crop/paste helpers preserve the original alpha channel. Detector or detail
+failures stop the script. Run the same program through the server's DGL editor;
+the removed native ADetailer mode and SDAPI recipe are not execution paths.
## Options
-An empty ADetailer prompt inherits the main prompt. `[PROMPT]` inserts the main
-prompt, `[SEP]` assigns different prompts to consecutive masks, and `[SKIP]`
-skips the corresponding mask.
-
-All settings other than the detector path and prompts are passed through
-`--extra-ad-args` as a comma-separated `key=value` list:
-
-| Key | Default | Description |
-| --- | ---: | --- |
-| `input_size` | `640` | Square YOLO input size; must be a multiple of 32 |
-| `confidence` | `0.3` | Detection confidence threshold |
-| `nms` | `0.45` | NMS IoU threshold |
-| `max_detections` | `100` | Maximum detections retained after NMS |
-| `mask_k_largest` | `0` | Keep only the largest K masks; zero keeps all |
-| `mask_filter_method` | `area` | Rank retained masks by `area` or `confidence` |
-| `mask_min_ratio` | `0` | Minimum bbox area relative to the image |
-| `mask_max_ratio` | `1` | Maximum bbox area relative to the image |
-| `dilate_erode` | `4` | Positive values dilate; negative values erode |
-| `x_offset`, `y_offset` | `0` | Mask offset in pixels; positive Y moves upward |
-| `mask_mode` | `none` | `none`, `merge`, or `merge_invert` |
-| `merge_masks`, `invert_mask` | `false` | Boolean alternatives to `mask_mode` |
-| `mask_blur` | `4` | Final composite feather radius |
-| `inpaint_padding` | `32` | Padding around the detected region |
-| `inpaint_only_masked` | `true` | Crop to the detected region; `false` uses the full image with the detection mask |
-| `inpaint_width`, `inpaint_height` | mode-specific | `512x512` after generation; input/output size in `adetailer` mode |
-| `denoising_strength` | mode-specific | `0.4` after generation; inherits `--strength` in `adetailer` mode |
-| `steps` | `0` | Detail steps; zero inherits the main generation |
-| `cfg_scale` | `-1` | Detail CFG; a negative value inherits the main generation |
-| `sample_method` | inherited | Detail sampler name |
-| `scheduler` | inherited | Detail scheduler name |
-| `sort_by` | `none` | `none`, `left_to_right`, `center_to_edge`, or `area` |
-
-Multiple masks are processed serially. Each completed inpaint becomes the input
-for the next mask, and the seed is incremented by the mask index. Use
-`mask_mode=merge` to process all detections in one inpaint pass.
-
-The detector uses the `detector` backend module. For example, keep detection on
-the CPU while diffusion runs on CUDA:
-
-**Windows (PowerShell) arguments**
-
-```text
---backend "diffusion=cuda0,detector=cpu"
-```
+The example's `settings` dict controls crop size, steps, denoising strength,
+CFG, sampler, scheduler and mask blur. Each stage has its own parameters:
-**Linux/macOS arguments**
+| Stage | DGL operation | Settings |
+| --- | --- | --- |
+| Detection | `detections::detect` | `input_size`, `confidence`, `nms`, `max_detections` |
+| Selection | `detections::filter` | `top_k`, `top_k_by`, area ratios and ordering |
+| Masks | `detections::box_masks` and tensor operations | Dilation, offsets, merging and blur |
+| Crop planning | `regions::plan_mask_crops` | `aspect`, `padding` |
+| Refinement | `fold(..., process=refine_region)` | Prompts, `seed + region.index`, `settings` |
-```text
---backend diffusion=cuda0,detector=cpu
-```
+Crops are processed in order; each repaired image becomes the next iteration's
+canvas. To run detection on CPU and diffusion on CUDA, set `backend="cpu"` on
+`load_detector` and `backend="cuda0"` on `load_unet`. The script decides prompt
+assignment explicitly; it does not parse legacy `[SEP]` / `[SKIP]` directives or
+`--extra-ad-args`.
## Notes and limitations
-YOLOv8 segmentation and MediaPipe detectors are not supported. See the
-[server API reference](../../examples/server/api.md) for request schemas.
+YOLOv8 segmentation and MediaPipe detectors are not supported.
+Use the [DGL API](../dgl.md#c-api) to run a complete program from a library caller.
diff --git a/docs/models/anima.md b/docs/models/anima.md
index 38f19f6..8a928cc 100644
--- a/docs/models/anima.md
+++ b/docs/models/anima.md
@@ -12,8 +12,6 @@ Anima generates images from text using a Qwen3 text encoder and a compatible Qwe
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -24,12 +22,47 @@ Use the files and compatibility requirements in [Model files](#model-files).
| Recipe | Component slots |
| --- | --- |
| Anima (`anima`) | `diffusion_model`, `llm`, `vae`, `tae` |
+| Anima Edit (`anima-edit`) | `diffusion_model`, `llm`, `vae`, `tae` |
Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+For Anima Edit, select a matching community editing LoRA separately and provide
+reference images. The references are VAE-encoded for diffusion conditioning;
+selecting the recipe does not load the LoRA automatically.
+
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/anima/anima-preview-txt2img.dgl) | Anima Preview. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\anima\anima-preview-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/anima/anima-preview-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/animatediff.md b/docs/models/animatediff.md
index ef110fe..0e1fcc9 100644
--- a/docs/models/animatediff.md
+++ b/docs/models/animatediff.md
@@ -6,6 +6,8 @@ Use the SD 1.5 checkpoint for the desired characters and style, and choose the
motion module and resolution below. See the
[AnimateDiff paper](https://arxiv.org/abs/2307.04725) for the model reference.
+**Currently supported only through dgl.**
+
## Supported variants
Per [animatediff.net/models](https://animatediff.net/models):
@@ -18,17 +20,17 @@ Per [animatediff.net/models](https://animatediff.net/models):
| `v3_sd15_mm.ckpt` | 1.5 | 512x512 | smoothest, highest quality; pairs with a Domain Adapter LoRA |
| `mm_sdxl_v10_beta` | SDXL | 512x512 | experimental, not yet supported here |
-Match your `-H -W` to the module's native resolution for best results. v3 is
+Match `height` and `width` to the module's native resolution for best results. v3 is
trained at 512x512 - going smaller (e.g. 384x384) still works but the motion
character is closer to v2.
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| SD 1.5 checkpoint | [Compatible SD 1.5 checkpoint](sd.md) | `-m` / `--model` | Required; SDXL motion modules are not supported |
-| Motion module | v1/v2/v3 motion weights listed below | `--motion-module` | Required; match the resolution and version |
-| Domain adapter LoRA | v3 adapter listed below | `--lora-model-dir` and prompt | Optional; v3 only |
+| SD 1.5 checkpoint | [Compatible SD 1.5 checkpoint](sd.md) | `load_unet`, `load_text_encoder`, `load_vae` | Required; SDXL motion modules are not supported |
+| Motion module | v1/v2/v3 motion weights listed below | `animatediff::load` | Required; match the resolution and version |
+| Domain adapter LoRA | v3 adapter listed below | `load_lora`, `apply_lora` | Optional; v3 only |
- Motion module (v3, recommended)
- fp16 safetensors: https://huggingface.co/conrevo/AnimateDiff-A1111/resolve/main/motion_module/mm_sd15_v3.safetensors
@@ -39,170 +41,73 @@ character is closer to v2.
- Domain Adapter LoRA (optional, v3 only, sharpens the base UNet's output
toward the motion module's trained distribution)
- ckpt: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_adapter.ckpt
- - place under your `--lora-model-dir` and reference in the prompt as
- `<lora:v3_sd15_adapter:1.0>`.
+ - load it with `load_lora` and apply it to the UNet with `apply_lora`.
The motion module is `~836 MB` and loads alongside the SD 1.5 UNet via
-`--motion-module`.
+`animatediff::load` / `animatediff::apply`.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
-Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
-server, then select the recipe matching your weights. Fill the component slots
-below and click **Load models**.
-Use the files and compatibility requirements in [Model files](#model-files).
+Open the server's **DGL editor**, edit the same script, and submit it.
+Model and input paths must satisfy the server's file-access policy; save outputs
+inside its `output/` directory. See the [Web UI guide](../../examples/server/README.md#web-ui).
-| Recipe | Component slots |
+### DGL
+
+| Example | Description |
| --- | --- |
-| AnimateDiff SD 1.5 (`animatediff-sd15`) | `model`, `motion_module`, `vae`, `tae` |
+| [Text to video with CPU offload](../dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl) | 8 frames at 512 x 512. |
+| [Text to video, 16 frames](../dgl/animatediff/animatediff-realisticvision-v6-txt2video-16-frames.dgl) | Native motion-module context at 384 x 384. |
+| [Text to video with a VRAM budget](../dgl/animatediff/animatediff-realisticvision-v6-txt2video-vram-budget.dgl) | Run with `--max-vram 2.0`. |
+| [Domain Adapter LoRA](../dgl/animatediff/animatediff-realisticvision-v6-domain-lora.dgl) | Add the v3 adapter before sampling. |
+| [Generate the initial image](../dgl/animatediff/realisticvision-v6-txt2img.dgl) | Prepare a still image for the image-to-video example. |
+| [Image to video](../dgl/animatediff/animatediff-realisticvision-v6-img2video.dgl) | Animate an existing image. |
-Slot IDs correspond to CLI component options: `diffusion_model` uses
-`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
-Select optional components when required by your checkpoint or task.
+#### Run a script
-### CLI
-
-#### Text-to-video
-
-Generate an 8-frame animation at 512x512, seed 42, 20 steps. The sampler /
-scheduler / CFG values below match what mm_sd15_v3 was trained with; using
-SD 1.5 defaults (euler_a, low CFG) produces noise-like output.
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_gen `
- --model ..\models\checkpoints\realisticVisionV60B1.safetensors `
- --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors `
- -p "a red apple on a wooden table" `
- --cfg-scale 8.0 `
- --sampling-method euler `
- --scheduler discrete `
- --steps 20 `
- -H 512 `
- -W 512 `
- --video-frames 8 `
- --fps 8 `
- --offload-to-cpu `
- -s 42 `
- -o out.avi
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\animatediff\animatediff-realisticvision-v6-txt2video-cpu-offload.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -M vid_gen \
- --model ../models/checkpoints/realisticVisionV60B1.safetensors \
- --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \
- -p "a red apple on a wooden table" \
- --cfg-scale 8.0 \
- --sampling-method euler \
- --scheduler discrete \
- --steps 20 \
- -H 512 \
- -W 512 \
- --video-frames 8 \
- --fps 8 \
- --offload-to-cpu \
- -s 42 \
- -o out.avi
+./bin/sd-cli -M dgl --script ../docs/dgl/animatediff/animatediff-realisticvision-v6-txt2video-cpu-offload.dgl
```
-Generate at the motion module's native 16-frame context (recommended for
-best temporal quality). Needs more VRAM at 512x512, so drop to 384x384 or
-use layer streaming:
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
-**Windows (PowerShell)**
+Load the SD 1.5 UNet with `load_unet` and the motion weights with
+`animatediff::load`. Use the same compute backend for both models.
+`animatediff::apply(unet, motion)` returns a denoiser that shares the original
+UNet and adds the motion module. Pass it to both CFG predictors and set the
+frame count through the latent shape, for example with
+`latent_shape(vae, width = 512, height = 512, batch = 16)`.
+The motion checkpoint's positional encoding sets the frame limit.
-```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_gen `
- --model ..\models\checkpoints\realisticVisionV60B1.safetensors `
- --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors `
- -p "photo of coastline, rocks, storm weather, wind, waves, lightning" `
- --cfg-scale 8.0 `
- --sampling-method euler `
- --scheduler discrete `
- --steps 20 `
- -H 384 `
- -W 384 `
- --video-frames 16 `
- --fps 8 `
- --offload-to-cpu `
- -s 42 `
- -o out.avi
-```
+Use `load_lora` and `apply_lora` for an optional v3 domain adapter.
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -M vid_gen \
- --model ../models/checkpoints/realisticVisionV60B1.safetensors \
- --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \
- -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
- --cfg-scale 8.0 \
- --sampling-method euler \
- --scheduler discrete \
- --steps 20 \
- -H 384 \
- -W 384 \
- --video-frames 16 \
- --fps 8 \
- --offload-to-cpu \
- -s 42 \
- -o out.avi
-```
-
-Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060):
-
-**Windows (PowerShell)**
+#### Text-to-video
-```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_gen `
- --model ..\models\checkpoints\realisticVisionV60B1.safetensors `
- --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors `
- -p "photo of coastline, rocks, storm weather, wind, waves, lightning" `
- --cfg-scale 8.0 `
- --sampling-method euler `
- --scheduler discrete `
- --steps 20 `
- -H 384 `
- -W 384 `
- --video-frames 8 `
- --fps 8 `
- --max-vram 2.0 `
- -s 42 `
- -o out.avi
-```
+Generate an 8-frame animation at 512x512, seed 42, 20 steps. The sampler /
+scheduler / CFG values in the scripts match what mm_sd15_v3 was trained with; using
+SD 1.5 defaults (euler_a, low CFG) produces noise-like output.
-**Linux/macOS**
+Generate at the motion module's native 16-frame context (recommended for
+best temporal quality). Needs more VRAM at 512x512, so drop to 384x384 or
+use the VRAM-budget example.
-```sh
-./bin/sd-cli \
- -M vid_gen \
- --model ../models/checkpoints/realisticVisionV60B1.safetensors \
- --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \
- -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
- --cfg-scale 8.0 \
- --sampling-method euler \
- --scheduler discrete \
- --steps 20 \
- -H 384 \
- -W 384 \
- --video-frames 8 \
- --fps 8 \
- --max-vram 2.0 \
- -s 42 \
- -o out.avi
-```
+The streaming example uses `--max-vram 2.0` (verified on RTX 3060).
#### Reference-quality reproduction
@@ -245,97 +150,29 @@ prompt, seed, config as above:
Individual fur strands, glowing inner-ear, and richer forest detail become
visible compared to the no-LoRA rendering.
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_gen `
- --model ..\models\checkpoints\realisticVisionV60B1.safetensors `
- --motion-module ..\models\motion_modules\mm_sd15_v3.safetensors `
- --lora-model-dir .\loras `
- -p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
-```
-
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -M vid_gen \
- --model ../models/checkpoints/realisticVisionV60B1.safetensors \
- --motion-module ../models/motion_modules/mm_sd15_v3.safetensors \
- --lora-model-dir ./loras \
- -p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
-```
-
#### img2video
-Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top.
+Set `input_path` to the pre-rendered image. All output frames start from its encoded latent, then receive noise according to `strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top.
-Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output.
+Left: generated starting image. Right: 8-frame AnimateDiff output.
<img src="../../assets/animatediff/img2video_demo.gif" width="512"/>
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -M img_gen ... `
- -o init.png "#" any high-quality still
-.\bin\Release\sd-cli.exe `
- -M vid_gen `
- --motion-module ..\models\motion_modules\mm_sd15_v3.safetensors `
- -i init.png `
- -p ... `
- --strength 0.75 `
- --cfg-scale 7.0 `
- --sampling-method euler `
- --scheduler karras `
- --steps 25 `
- -H 512 `
- -W 512 `
- --video-frames 8 `
- -s 42 `
- -o out.avi
-```
-
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -M img_gen ... \
- -o init.png "#" any high-quality still
-./bin/sd-cli \
- -M vid_gen \
- --motion-module ../models/motion_modules/mm_sd15_v3.safetensors \
- -i init.png \
- -p ... \
- --strength 0.75 \
- --cfg-scale 7.0 \
- --sampling-method euler \
- --scheduler karras \
- --steps 25 \
- -H 512 \
- -W 512 \
- --video-frames 8 \
- -s 42 \
- -o out.avi
-```
-
-`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static).
+`strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static).
## Notes and limitations
- The motion module was trained at `video_length=16`. Running with
- `--video-frames 16` gives the best coherence; F=8 works but shows a shorter
+ `frames = 16` gives the best coherence; F=8 works but shows a shorter
motion arc. Frame counts up to 32 are supported by the positional encoding
but exceed the trained regime and produce more static output.
-- At `--video-frames 1` the motion module is skipped entirely and the output
- is bit-identical to `-M img_gen`. This avoids the single-token
+- At `frames = 1` the motion module is skipped entirely and the output
+ is bit-identical to ordinary image sampling. This avoids the single-token
temporal-attention degeneracy that would otherwise emit an untrained-magnitude
residual on a single-frame sample.
- The base UNet is frozen, so character identity and style follow the SD 1.5
- checkpoint you pass to `--model`. LoRAs and prompt weighting attach to the
+ checkpoint selected by `weights`. LoRAs and prompt weighting attach to the
base model in the usual way.
- No mid_block motion module in v3. `mm_sdxl_v10_beta` (SDXL variant) is not
supported yet.
-- Output is written as MJPEG AVI. Use `--fps` to set playback speed.
+- Output is written as MJPEG AVI. Use `fps` to set playback speed.
diff --git a/docs/models/boogu_image.md b/docs/models/boogu_image.md
index d7d133c..7aeff1b 100644
--- a/docs/models/boogu_image.md
+++ b/docs/models/boogu_image.md
@@ -12,8 +12,6 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -30,7 +28,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/boogu_image/boogu-image-base-txt2img.dgl) | Boogu Image Base. |
+| [Image editing](../dgl/boogu_image/boogu-image-edit.dgl) | Boogu Image Edit. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\boogu_image\boogu-image-base-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/boogu_image/boogu-image-base-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Boogu Image Base
diff --git a/docs/models/chroma.md b/docs/models/chroma.md
index 51c9685..c02ce56 100644
--- a/docs/models/chroma.md
+++ b/docs/models/chroma.md
@@ -45,8 +45,6 @@ You can download the preconverted gguf weights from [silveroxides/Chroma-GGUF](h
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -62,7 +60,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/chroma/chroma-unlocked-v40-txt2img.dgl) | Chroma Unlocked v40. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\chroma\chroma-unlocked-v40-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/chroma/chroma-unlocked-v40-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Example
diff --git a/docs/models/chroma_radiance.md b/docs/models/chroma_radiance.md
index dbb8ad5..0a938dc 100644
--- a/docs/models/chroma_radiance.md
+++ b/docs/models/chroma_radiance.md
@@ -11,8 +11,6 @@ Chroma1-Radiance generates images directly in pixel space using T5-XXL. No VAE i
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -28,7 +26,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl) | Generate pixels directly, without a VAE. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\chroma_radiance\chroma1-radiance-v0-4-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/chroma_radiance/chroma1-radiance-v0-4-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/distilled_sd.md b/docs/models/distilled_sd.md
index 30e75aa..3a3f53a 100644
--- a/docs/models/distilled_sd.md
+++ b/docs/models/distilled_sd.md
@@ -131,8 +131,6 @@ For the use of this model, both options `--cfg-scale 1` and `--steps 1` are agai
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -151,7 +149,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [SSD1B](../dgl/distilled_sd/ssd1b-txt2img.dgl) | Text-to-image generation. |
+| [SDXS-512-DreamShaper](../dgl/distilled_sd/sdxs-512-dreamshaper-txt2img.dgl) | Distilled text-to-image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\distilled_sd\ssd1b-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/distilled_sd/ssd1b-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### SSD1B
@@ -197,6 +226,9 @@ SSD-1B, Vega, and Tiny SD variants reduce the base model size. Details are in th
[SSD-1B paper](https://arxiv.org/abs/2401.02677v1) and
[Tiny SD paper](https://arxiv.org/pdf/2305.15798.pdf). Follow the [SDXS generation settings](#sdxs-512-dreamshaper); CFG 1 and one step are required.
+For SDXS, a configured `taesd_path` is used for final output even when
+`tae_preview_only` is enabled, because these models require the tiny autoencoder.
+
Run the Python conversion commands in [Model files](#model-files) from the source repository root,
with the listed conversion script and Python dependencies installed. Return to
the build directory for the platform-specific CLI commands.
diff --git a/docs/models/ernie_image.md b/docs/models/ernie_image.md
index e4e3069..41e0d3b 100644
--- a/docs/models/ernie_image.md
+++ b/docs/models/ernie_image.md
@@ -16,8 +16,6 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -34,7 +32,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [ERNIE-Image Turbo](../dgl/ernie_image/ernie-image-turbo-txt2img.dgl) | Text-to-image generation. |
+| [ERNIE-Image](../dgl/ernie_image/ernie-image-txt2img.dgl) | Text-to-image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ernie_image\ernie-image-turbo-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/ernie_image/ernie-image-turbo-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### ERNIE-Image-Turbo
diff --git a/docs/models/esrgan.md b/docs/models/esrgan.md
index 3d1a589..42e820e 100644
--- a/docs/models/esrgan.md
+++ b/docs/models/esrgan.md
@@ -1,40 +1,48 @@
# ESRGAN and MoSR
-ESRGAN and compatible MoSR weights upscale generated images. Pass the chosen
-upscaler with `--upscale-model`.
+ESRGAN and compatible MoSR weights upscale images. **Chaining generation and
+upscaling is currently supported only through DGL**, as shown below. Standalone
+CLI upscaling remains available with `-M upscale --upscale-model FILE -i INPUT`.
## Model files
| Component | Weights and downloads | CLI option | Requirement |
| --- | --- | --- | --- |
| Base model | [Stable Diffusion checkpoint](sd.md) | `-m` / `--model` | Required for generation below |
-| ESRGAN upscaler | [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) | `--upscale-model` | Choose a compatible upscaler |
-| MoSR upscaler | [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth) | `--upscale-model` | Alternative; requires the PixelShuffle upsampler |
+| ESRGAN upscaler | [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) | `load_upscaler` (DGL) / `--upscale-model` (standalone CLI) | Choose a compatible upscaler |
+| MoSR upscaler | [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth) | `load_upscaler` (DGL) / `--upscale-model` (standalone CLI) | Alternative; requires the PixelShuffle upsampler |
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+### DGL
-### CLI
+| Example | Description |
+| --- | --- |
+| [Generate and upscale](../dgl/esrgan/sd1-5-txt2img-esrgan.dgl) | SD 1.5 generation followed by RealESRGAN. |
+| [Upscale an image](../dgl/esrgan/realesrgan-x4plus-anime-6b-upscale.dgl) | Run RealESRGAN on an existing image. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors `
- --upscale-model ..\models\upscale_models\RealESRGAN_x4plus_anime_6B.pth `
- -p "a lovely cat"
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\esrgan\sd1-5-txt2img-esrgan.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \
- --upscale-model ../models/upscale_models/RealESRGAN_x4plus_anime_6B.pth \
- -p "a lovely cat"
+./bin/sd-cli -M dgl --script ../docs/dgl/esrgan/sd1-5-txt2img-esrgan.dgl
```
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
## Notes and limitations
RGBA output retains its alpha channel during model upscaling and hires fix.
diff --git a/docs/models/fibo2.md b/docs/models/fibo2.md
index 4919bda..429c42a 100644
--- a/docs/models/fibo2.md
+++ b/docs/models/fibo2.md
@@ -17,8 +17,6 @@ INT8 convrot weights load directly; see [INT8 convrot](../guides/int8_convrot.md
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -35,7 +33,39 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Dev: text to image](../dgl/fibo2/fibo2-dev-txt2img.dgl) | Generate an image from a prompt. |
+| [Dev: image editing](../dgl/fibo2/fibo2-dev-edit.dgl) | Edit a reference image. |
+| [Edit Turbo](../dgl/fibo2/fibo2-edit-turbo.dgl) | Image editing with distilled weights. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\fibo2\fibo2-dev-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/fibo2/fibo2-dev-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Fibo 2.0 Dev
@@ -147,8 +177,9 @@ Edit Turbo is distilled for reference-image editing in 4 steps with CFG 1.
The BF16 files accept the same options. Use dimensions divisible by 16. Repeat `-r` to use multiple reference images; instructions can refer to `<image_1>`, `<image_2>`, etc. See [Image Editing](../guides/edit.md) for reference-image preprocessing options. Separate GGUF text encoders need their vision projector passed with `--llm_vision` for editing.
-The automatic `fibo2` reference preset combines `resize_single_ref_to_target=true`, `vae_size_buckets=true`, `vae_center_crop=true`, and `vlm_use_vae_input=true`. A single reference is scaled to cover the output dimensions and center-cropped. Multiple references use dimensions divisible by 16, chosen within ±10% of one megapixel to minimize cropping for each image's aspect ratio; `vae_input_max_pixels` overrides that target area. The VAE and vision encoder receive the same crops, with smaller copies used by the vision encoder. These are generic reference-image options and can be overridden individually: for example, `--ref-image-args vae_center_crop=false` uses direct resizing, while `resize_before_vae=false` skips this automatic crop and resize step entirely.
-
-The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Override these with `--ref-image-args`, for example `vlm_max_size=512`, or use `vlm_resize_mode=none` to skip size-based scaling. Vision inputs are still aligned to the encoder's patch grid.
-
-Plain prompts and structured JSON captions are accepted. This integration uses the caption-only text-rendering path (`render_text_images=False`); it does not generate auxiliary glyph images or expand prompts with Bria VGL. Native Fibo mask preprocessing is not connected to `--mask`.
+The default reference policy scales and center-crops a single reference to
+the output dimensions. Multiple references use aspect-ratio buckets near one
+megapixel, aligned to 16 pixels. VAE and vision encoding share those crops;
+the vision encoder uses a smaller copy with a maximum side of 768 pixels.
+Use a [DGL editing workflow](../guides/edit.md#custom-reference-processing) for
+custom geometry or different VAE and vision inputs.
diff --git a/docs/models/flux.md b/docs/models/flux.md
index dd67bbd..eec26dc 100644
--- a/docs/models/flux.md
+++ b/docs/models/flux.md
@@ -47,8 +47,6 @@ For example:
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -65,7 +63,39 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [FLUX.1 dev](../dgl/flux/flux1-dev-txt2img.dgl) | Text-to-image generation. |
+| [FLUX.1 schnell](../dgl/flux/flux1-schnell-txt2img.dgl) | Distilled text-to-image generation. |
+| [FLUX.1 dev with LoRA](../dgl/flux/flux1-dev-lora.dgl) | Apply a LoRA before generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\flux\flux1-dev-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/flux/flux1-dev-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
- `--cfg-scale` is recommended to be set to 1.
diff --git a/docs/models/flux2.md b/docs/models/flux2.md
index fdca1b2..2e9fdf4 100644
--- a/docs/models/flux2.md
+++ b/docs/models/flux2.md
@@ -35,8 +35,6 @@ the text encoder matching the transformer variant.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -56,7 +54,43 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Dev: image editing](../dgl/flux2/flux2-dev-edit.dgl) | Edit a reference image. |
+| [Klein 4B: text to image](../dgl/flux2/flux2-klein-4b-txt2img.dgl) | Distilled 4B checkpoint. |
+| [Klein 4B: image editing](../dgl/flux2/flux2-klein-4b-edit.dgl) | Distilled 4B checkpoint. |
+| [Klein Base 4B: text to image](../dgl/flux2/flux2-klein-base-4b-txt2img.dgl) | Base 4B checkpoint. |
+| [Klein 9B: text to image](../dgl/flux2/flux2-klein-9b-txt2img.dgl) | Distilled 9B checkpoint. |
+| [Klein 9B: image editing](../dgl/flux2/flux2-klein-9b-edit.dgl) | Distilled 9B checkpoint. |
+| [Klein Base 9B: text to image](../dgl/flux2/flux2-klein-base-9b-txt2img.dgl) | Base 9B checkpoint. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\flux2\flux2-dev-edit.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/flux2/flux2-dev-edit.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Flux.2-dev
diff --git a/docs/models/flux3.md b/docs/models/flux3.md
index 7591d7c..5fde15f 100644
--- a/docs/models/flux3.md
+++ b/docs/models/flux3.md
@@ -15,8 +15,6 @@ Use the root `model.safetensors` for the diffusion model.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -33,7 +31,39 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Klein: text to image](../dgl/flux3/flux3-image-klein-txt2img.dgl) | Distilled checkpoint. |
+| [Klein: image editing](../dgl/flux3/flux3-image-klein-edit.dgl) | Edit a reference image. |
+| [Klein Base: text to image](../dgl/flux3/flux3-image-klein-base-txt2img.dgl) | Base checkpoint. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\flux3\flux3-image-klein-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/flux3/flux3-image-klein-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### FLUX.3 Image Klein
diff --git a/docs/models/hidream_o1_image.md b/docs/models/hidream_o1_image.md
index 09f7ac7..80d15f9 100644
--- a/docs/models/hidream_o1_image.md
+++ b/docs/models/hidream_o1_image.md
@@ -11,8 +11,6 @@ HiDream-O1-Image uses a bundled model checkpoint. The example below uses the Dev
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -28,7 +26,48 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/hidream_o1_image/hidream-o1-image-dev-txt2img.dgl) | HiDream O1 Image Dev. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\hidream_o1_image\hidream-o1-image-dev-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/hidream_o1_image/hidream-o1-image-dev-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+HiDream O1 uses initial Gaussian noise with a standard deviation of 8. The DGL
+example controls this with `noise_scale` before `add_noise`:
+
+```text
+noise = tensor::affine(tensor::randn_like(base, rng = rng), scale = noise_scale)
+start = add_noise(predict, base, noise, sigma = sigmas[0])
+```
+
+Use the same RNG for initial noise and sampling. The noise scale is independent
+of the sampler's `eta` parameter.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### HiDream-O1-Image-Dev
@@ -57,3 +96,8 @@ Select optional components when required by your checkpoint or task.
```
<img width="256" alt="HiDream-O1-Image-Dev example" src="../../assets/hidream-o1/dev_example.png" />
+
+## Options
+
+The initial noise scale defaults to 8. Use `--eta` (`sample.eta` in the C API)
+to override it. The same scale applies to the initial and hires sampling stages.
diff --git a/docs/models/hunyuan_video.md b/docs/models/hunyuan_video.md
index f31f8cf..38d9308 100644
--- a/docs/models/hunyuan_video.md
+++ b/docs/models/hunyuan_video.md
@@ -14,8 +14,6 @@ and ByT5 Small GlyphXL for glyph-aware text conditioning.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -35,7 +33,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to video](../dgl/hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl) | HunyuanVideo 1.5, 720p. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\hunyuan_video\hunyuanvideo1-5-720p-txt2video.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/hunyuan_video/hunyuanvideo1-5-720p-txt2video.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-video example
diff --git a/docs/models/ideogram4.md b/docs/models/ideogram4.md
index 64e8121..3a797de 100644
--- a/docs/models/ideogram4.md
+++ b/docs/models/ideogram4.md
@@ -91,8 +91,6 @@ If you want lower VRAM usage, you can change the quantization from q8_0 to a low
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -108,7 +106,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/ideogram4/ideogram4-txt2img.dgl) | Ideogram4 generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ideogram4\ideogram4-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/ideogram4/ideogram4-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/ip_adapter.md b/docs/models/ip_adapter.md
index 854ed88..c8ca32a 100644
--- a/docs/models/ip_adapter.md
+++ b/docs/models/ip_adapter.md
@@ -5,219 +5,105 @@ to guide SD 1.5 or SDXL generation alongside
the text prompt. Classic and Plus adapters are supported, including combination
with Control Net.
+**Currently supported only through dgl.**
+
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| Base checkpoint | [SD 1.5 or SDXL](sd.md) | `-m` / `--model` | Required; match adapter family |
-| CLIP Vision H | `clip_vision_h.safetensors` from [IP-Adapter](https://huggingface.co/h94/IP-Adapter) | `--clip_vision` | Required; ViT-H/14 |
-| SD 1.5 adapter | [`models/ip-adapter_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | For SD 1.5 |
-| SDXL adapter | [`sdxl_models/ip-adapter_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | For SDXL |
-| SD 1.5 Plus | [`models/ip-adapter-plus_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | Alternative for SD 1.5 |
-| SDXL Plus | [`sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | Alternative for SDXL |
+| Base checkpoint | [SD 1.5 or SDXL](sd.md) | `load_unet`, `load_text_encoder`, `load_vae` | Required; match adapter family |
+| CLIP Vision H | `clip_vision_h.safetensors` from [IP-Adapter](https://huggingface.co/h94/IP-Adapter) | `load_vision_encoder` | Required; ViT-H/14 |
+| SD 1.5 adapter | [`models/ip-adapter_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `ip_adapter::load` | For SD 1.5 |
+| SDXL adapter | [`sdxl_models/ip-adapter_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `ip_adapter::load` | For SDXL |
+| SD 1.5 Plus | [`models/ip-adapter-plus_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `ip_adapter::load` | Alternative for SD 1.5 |
+| SDXL Plus | [`sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `ip_adapter::load` | Alternative for SDXL |
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
-Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
-server, then select the recipe matching your weights. Fill the component slots
-below and click **Load models**.
-Use the files and compatibility requirements in [Model files](#model-files).
+Open the server's **DGL editor**, edit the same script, and submit it.
+Model and input paths must satisfy the server's file-access policy; save outputs
+inside its `output/` directory. See the [Web UI guide](../../examples/server/README.md#web-ui).
-| Recipe | Component slots |
-| --- | --- |
-| SD 1.5 IP-Adapter / Plus (`sd1-ip-adapter`) | `model`, `clip_vision`, `ip_adapter`, `vae`, `tae` |
-| SDXL IP-Adapter / Plus (`sdxl-ip-adapter`) | `model`, `clip_vision`, `ip_adapter`, `vae`, `tae` |
+### DGL
-Slot IDs correspond to CLI component options: `diffusion_model` uses
-`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
-Select optional components when required by your checkpoint or task.
+| Example | Description |
+| --- | --- |
+| [SD 1.5 Classic](../dgl/ip_adapter/ip-adapter-sd1-5.dgl) | Use a reference image with SD 1.5. |
+| [SDXL Classic](../dgl/ip_adapter/ip-adapter-sdxl.dgl) | Use a reference image with SDXL. |
+| [SD 1.5 Plus](../dgl/ip_adapter/ip-adapter-plus-sd1-5.dgl) | Use unpooled CLIP Vision features. |
+| [SDXL with ControlNet](../dgl/ip_adapter/ip-adapter-sdxl-controlnet.dgl) | Combine appearance and pose conditions. |
-### CLI
+#### Run a script
-#### Example (SD 1.5)
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sd_v1.5.safetensors `
- --clip_vision ..\models\clip_vision\clip_vision_h.safetensors `
- --ip-adapter ..\models\ipadapter\ip-adapter_sd15.safetensors `
- --ip-adapter-image ..\inputs\reference.png `
- -p "a woman, best quality" `
- -n "lowres, bad anatomy" `
- --cfg-scale 7 `
- --steps 30 `
- --sampling-method dpm++2m `
- --scheduler karras `
- -W 512 `
- -H 512 `
- --ip-adapter-strength 0.8
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ip_adapter\ip-adapter-sd1-5.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sd_v1.5.safetensors \
- --clip_vision ../models/clip_vision/clip_vision_h.safetensors \
- --ip-adapter ../models/ipadapter/ip-adapter_sd15.safetensors \
- --ip-adapter-image ../inputs/reference.png \
- -p "a woman, best quality" \
- -n "lowres, bad anatomy" \
- --cfg-scale 7 \
- --steps 30 \
- --sampling-method dpm++2m \
- --scheduler karras \
- -W 512 \
- -H 512 \
- --ip-adapter-strength 0.8
+./bin/sd-cli -M dgl --script ../docs/dgl/ip_adapter/ip-adapter-sd1-5.dgl
```
-#### Example (SDXL)
-
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sdxl.safetensors `
- --clip_vision ..\models\clip_vision\clip_vision_h.safetensors `
- --ip-adapter ..\models\ipadapter\ip-adapter_sdxl_vit-h.safetensors `
- --ip-adapter-image ..\inputs\reference.png `
- -p "a woman, best quality" `
- -n "lowres, bad anatomy" `
- --cfg-scale 6 `
- --steps 25 `
- --sampling-method dpm++2m `
- --scheduler karras `
- -W 1024 `
- -H 1024 `
- --ip-adapter-strength 0.8 `
- --vae-tiling
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+The [SD 1.5 Classic example](../dgl/ip_adapter/ip-adapter-sd1-5.dgl) loads the
+image and attention projections from one checkpoint with `ip_adapter::load`.
+`ip_adapter::project` returns the projected image tokens as a tensor.
+`ip_adapter::apply` binds those tokens to one text condition, so the positive
+and negative CFG branches can use different tensors. Apply it again to the
+returned condition to add another adapter. `strength` scales each adapter's
+attention contribution. `start_sigma` and `end_sigma` bound the contribution
+from high to low noise. Use `t_to_sigma` to convert a noise time to a bound.
+
+Classic projects a pooled CLIP Vision embedding for the positive branch and a
+zero embedding for the negative branch. Plus uses penultimate hidden states
+from the reference image and from a zeroed, preprocessed CLIP Vision input.
+Use the Plus checkpoint and these calls in the example for Plus.
+
+```dgl
+reference = image::load(input_path)
+embedding = encode_vision(vision, reference, pooled = false, clip_skip = 2)
+empty_embedding = encode_vision(vision, reference, pooled = false, clip_skip = 2, zero_input = true)
+positive_image = ip_adapter::project(adapter, embedding)
+negative_image = ip_adapter::project(adapter, empty_embedding)
```
-**Linux/macOS**
+For SDXL use an SDXL checkpoint and `family = "sdxl"` on `ip_adapter::load`.
-```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sdxl.safetensors \
- --clip_vision ../models/clip_vision/clip_vision_h.safetensors \
- --ip-adapter ../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors \
- --ip-adapter-image ../inputs/reference.png \
- -p "a woman, best quality" \
- -n "lowres, bad anatomy" \
- --cfg-scale 6 \
- --steps 25 \
- --sampling-method dpm++2m \
- --scheduler karras \
- -W 1024 \
- -H 1024 \
- --ip-adapter-strength 0.8 \
- --vae-tiling
-```
+#### Example (SDXL)
-The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
-`--offload-to-cpu`) on GPUs with limited VRAM.
+The SDXL example passes `tiling()` to `vae_decode` to reduce decode memory.
+Set `params_backend="cpu"` on a loader to offload its weights.
#### Plus variants
-Use the Plus weights matching your base model. The same IP-Adapter options
-apply; no extra flag is needed. Plus can also be combined with Control Net.
-
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sd_v1.5.safetensors `
- --clip_vision ..\models\clip_vision\clip_vision_h.safetensors `
- --ip-adapter ..\models\ipadapter\ip-adapter-plus_sd15.safetensors `
- --ip-adapter-image ..\inputs\reference.png `
- -p "a woman, best quality" `
- -n "lowres, bad anatomy" `
- --cfg-scale 7 `
- --steps 30 `
- --sampling-method dpm++2m `
- --scheduler karras `
- -W 512 `
- -H 512 `
- --ip-adapter-strength 0.8
-```
-
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sd_v1.5.safetensors \
- --clip_vision ../models/clip_vision/clip_vision_h.safetensors \
- --ip-adapter ../models/ipadapter/ip-adapter-plus_sd15.safetensors \
- --ip-adapter-image ../inputs/reference.png \
- -p "a woman, best quality" \
- -n "lowres, bad anatomy" \
- --cfg-scale 7 \
- --steps 30 \
- --sampling-method dpm++2m \
- --scheduler karras \
- -W 512 \
- -H 512 \
- --ip-adapter-strength 0.8
-```
+Use Plus weights matching the base model. The Plus script requests unpooled
+CLIP Vision features and encodes a zero input for its negative image condition.
+Use the SDXL with ControlNet script to combine both conditions.
The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the
classic adapters) confirms a Plus file was loaded.
#### Combining with Control Net
-Add the usual Control Net options to keep the reference appearance while
-controlling the pose:
-
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sdxl.safetensors `
- --clip_vision ..\models\clip_vision\clip_vision_h.safetensors `
- --ip-adapter ..\models\ipadapter\ip-adapter_sdxl_vit-h.safetensors `
- --control-net ..\models\controlnet\OpenPoseXL2.safetensors `
- --ip-adapter-image ..\inputs\character.png `
- --control-image ..\inputs\pose.png `
- -p "a character, side view" `
- --cfg-scale 6 `
- --steps 25 `
- -W 1024 `
- -H 1024 `
- --ip-adapter-strength 0.9 `
- --control-strength 0.8 `
- --vae-tiling
-```
-
-**Linux/macOS**
-
-```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sdxl.safetensors \
- --clip_vision ../models/clip_vision/clip_vision_h.safetensors \
- --ip-adapter ../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors \
- --control-net ../models/controlnet/OpenPoseXL2.safetensors \
- --ip-adapter-image ../inputs/character.png \
- --control-image ../inputs/pose.png \
- -p "a character, side view" \
- --cfg-scale 6 \
- --steps 25 \
- -W 1024 \
- -H 1024 \
- --ip-adapter-strength 0.9 \
- --control-strength 0.8 \
- --vae-tiling
-```
+Apply `controlnet::apply` and `ip_adapter::apply` to the same condition to
+preserve the reference appearance while controlling pose.
## Options
-- `--ip-adapter <path>` path to the IP-Adapter weight file.
-- `--ip-adapter-image <path>` path to the reference image.
-- `--ip-adapter-strength <float>` strength of the IP-Adapter injection
- (default 1.0). Lower values let the text prompt dominate; 0.6 to 0.8 is
- a good starting range.
+Set the reference path and component paths in `var`. `ip_adapter::apply`'s
+`strength` scales the image attention contribution; 0.6–0.8 is a useful starting
+range. Optional `start_sigma` / `end_sigma` restrict its sampling interval.
+Classic and Plus use different CLIP Vision representations as shown above.
+Use `controlnet::apply` independently for pose or edge conditioning.
diff --git a/docs/models/iris.md b/docs/models/iris.md
index 643b0d2..efd86ee 100644
--- a/docs/models/iris.md
+++ b/docs/models/iris.md
@@ -11,8 +11,6 @@ Iris-3B generates images directly in pixel space and uses Qwen3-VL-4B-Instruct a
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -28,7 +26,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/iris/iris-3b-txt2img.dgl) | Iris-3B generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\iris\iris-3b-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/iris/iris-3b-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/kontext.md b/docs/models/kontext.md
index c711f32..b40fbf8 100644
--- a/docs/models/kontext.md
+++ b/docs/models/kontext.md
@@ -46,8 +46,6 @@ You can download the preconverted gguf weights from [FLUX.1-Kontext-dev-GGUF](ht
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -63,7 +61,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Image editing](../dgl/kontext/flux1-kontext-dev-edit.dgl) | Edit a reference image with FLUX.1 Kontext dev. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\kontext\flux1-kontext-dev-edit.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/kontext/flux1-kontext-dev-edit.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
- `--cfg-scale` is recommended to be set to 1.
diff --git a/docs/models/krea2.md b/docs/models/krea2.md
index c636711..3fa24f4 100644
--- a/docs/models/krea2.md
+++ b/docs/models/krea2.md
@@ -13,8 +13,6 @@ Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -26,12 +24,50 @@ Use the files and compatibility requirements in [Model files](#model-files).
| --- | --- |
| Krea 2 Raw (`krea2-raw`) | `diffusion_model`, `llm`, `vae`, `tae` |
| Krea 2 Turbo (`krea2-turbo`) | `diffusion_model`, `llm`, `vae`, `tae` |
+| Krea 2 Raw Edit (`krea2-raw-edit`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` |
+| Krea 2 Turbo Edit (`krea2-turbo-edit`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` |
Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+The Edit recipes require a matching Ostris-compatible community editing LoRA,
+selected separately, and reference images. Each reference goes through both
+the vision encoder and VAE. Select a vision projector in `llm_vision` when it
+is not bundled in the text encoder. The recipe does not load the LoRA
+automatically.
+
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/krea2/krea2-raw-txt2img.dgl) | Krea2 Raw. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\krea2\krea2-raw-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/krea2/krea2-raw-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Krea2
diff --git a/docs/models/lcm.md b/docs/models/lcm.md
index 90330d2..30edcb4 100644
--- a/docs/models/lcm.md
+++ b/docs/models/lcm.md
@@ -12,8 +12,6 @@ The example below applies an LCM-LoRA to SD 1.5.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -33,7 +31,37 @@ Select optional components when required by your checkpoint or task.
Select the matching LCM checkpoint or apply a compatible LCM-LoRA, then set the
sampling method, CFG, and steps for that model.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [LCM-LoRA with SD 1.5](../dgl/lcm/lcm-lora-sd1-5.dgl) | Apply the acceleration LoRA before sampling. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\lcm\lcm-lora-sd1-5.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/lcm/lcm-lora-sd1-5.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### LCM-LoRA with SD 1.5
diff --git a/docs/models/lens.md b/docs/models/lens.md
index 42708eb..b209118 100644
--- a/docs/models/lens.md
+++ b/docs/models/lens.md
@@ -16,8 +16,6 @@ Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the te
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -34,7 +32,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Lens](../dgl/lens/lens-txt2img.dgl) | Text-to-image generation. |
+| [Lens Turbo](../dgl/lens/lens-turbo-txt2img.dgl) | Distilled text-to-image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\lens\lens-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/lens/lens-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Lens
diff --git a/docs/models/lingbot_video.md b/docs/models/lingbot_video.md
index a085c89..d94a19e 100644
--- a/docs/models/lingbot_video.md
+++ b/docs/models/lingbot_video.md
@@ -13,8 +13,6 @@ Lingbot Video uses a Lingbot diffusion transformer, the Wan2.1 VAE, and Qwen3-VL
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -31,7 +29,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to video](../dgl/lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl) | LingBot-Video Dens 1.3B. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\lingbot_video\lingbot-video-dens-1-3b-txt2video.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/lingbot_video/lingbot-video-dens-1-3b-txt2video.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### LingBot-Video T2V
diff --git a/docs/models/llada_image.md b/docs/models/llada_image.md
index d627b7b..0517437 100644
--- a/docs/models/llada_image.md
+++ b/docs/models/llada_image.md
@@ -123,8 +123,6 @@ save_file(merged, "llada_connectors.safetensors")
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -141,7 +139,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/llada_image/llada-image-turbo-txt2img.dgl) | LLaDA-Image Turbo. |
+| [Image editing](../dgl/llada_image/llada-image-turbo-edit.dgl) | Edit a reference image. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\llada_image\llada-image-turbo-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/llada_image/llada-image-turbo-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text to image
@@ -223,9 +252,9 @@ Select optional components when required by your checkpoint or task.
<img width="256" alt="LLaDA-Image edit example" src="../../assets/llada_image/edit_example.png" />
-See [edit.md](../guides/edit.md) for the shared reference-image options. LLaDA-Image uses the
-`llada_image` preset by default, resizing the reference image to the output width and height
-before VAE encoding. SigVQ uses bilinear resizing to half the output resolution and inputs
+See [edit.md](../guides/edit.md) for the shared reference-image options. The CLI and C API
+resize the reference image to the output width and height before VAE encoding.
+SigVQ uses bilinear resizing to half the output resolution and inputs
normalized to `[-1, 1]`. CFG keeps the source latent in both branches and uses SigVQ features
only in the positive branch. Editing requires connectors that include the SigVQ weights.
@@ -247,5 +276,7 @@ only in the positive branch. Editing requires connectors that include the SigVQ
- `--scheduler` defaults to `llada_image`, which reproduces the reference Kumaraswamy sigma
grid. `--extra-sample-args uniform=1` selects the uniform grid instead.
- Prompt templating is handled automatically; pass a plain description.
+- Image editing uses the same target-sized reference pixels for VAE encoding
+ and the text encoder's image components.
- VQ-conditioned generation (`generation_mode="vq"`, where the text encoder decodes image
tokens before diffusion) is not implemented.
diff --git a/docs/models/longcat_image.md b/docs/models/longcat_image.md
index ec2fae1..77cd68e 100644
--- a/docs/models/longcat_image.md
+++ b/docs/models/longcat_image.md
@@ -13,8 +13,6 @@ LongCat-Image uses a LongCat diffusion transformer, the FLUX VAE, and Qwen2.5-VL
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -31,7 +29,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/longcat_image/longcat-image-txt2img.dgl) | LongCat Image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\longcat_image\longcat-image-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/longcat_image/longcat-image-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
LongCat uses quoted text for character-level text rendering. Put target text inside single quotes, double quotes, or Chinese quotes.
diff --git a/docs/models/lora.md b/docs/models/lora.md
index 56cb60b..f5fc1cf 100644
--- a/docs/models/lora.md
+++ b/docs/models/lora.md
@@ -12,14 +12,42 @@ using `<lora:NAME:STRENGTH>` in the prompt.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Load a compatible base model with its recipe, then select the LoRA for generation.
See [Web UI guide](../../examples/server/README.md#web-ui) for the shared workflow.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [SD 1.5 with LoRA](../dgl/lora/sd1-5-lora.dgl) | Apply the same LoRA to the UNet and text encoder. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\lora\sd1-5-lora.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/lora/sd1-5-lora.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/ltx2.md b/docs/models/ltx2.md
index 1785487..1cddeb5 100644
--- a/docs/models/ltx2.md
+++ b/docs/models/ltx2.md
@@ -58,8 +58,6 @@ To run the text encoder quantized, convert it once with sd-cli:
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -77,7 +75,63 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+#### LTX-2.3
+
+| Example | Description |
+| --- | --- |
+| [Text to video](../dgl/ltx2/ltx-2-3-dev-txt2video.dgl) | LTX-2.3 dev with audio. |
+| [Image to video](../dgl/ltx2/ltx-2-3-dev-img2video.dgl) | Condition the initial frame. |
+| [First/last-frame conditioning](../dgl/ltx2/ltx-2-3-dev-first-last-frame.dgl) | Set both endpoint images. |
+| [Spatial latent upscaling](../dgl/ltx2/ltx-2-3-spatial-upscale.dgl) | Refine a low-resolution draft with a learned upscaler. |
+
+#### LTX-2.5
+
+| Example | Description |
+| --- | --- |
+| [Text to video](../dgl/ltx2/ltx-2-5-dev-txt2video.dgl) | LTX-2.5 dev with audio. |
+| [Image to video](../dgl/ltx2/ltx-2-5-dev-img2video.dgl) | Condition the initial frame. |
+| [Spatial latent upscaling](../dgl/ltx2/ltx-2-5-spatial-upscale.dgl) | Refine a low-resolution draft with a learned upscaler. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ltx2\ltx-2-3-dev-txt2video.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/ltx2/ltx-2-3-dev-txt2video.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+#### Latent upscaling
+
+`load_latent_upscaler` loads the spatial or temporal checkpoint as an
+independent resource. Convert the video part of the draft diffusion latent
+with `dit_to_vae_latents`, run `upscale_latent`, then convert back
+with `vae_to_dit_latents` before the refine pass. The checkpoint
+determines the scale and which axis to upscale. Repack the audio in the
+script when the refine pass uses both video and audio latents.
+See the [Video Hires example](../dgl/ltx2/ltx-2-3-spatial-upscale.dgl).
+The CLI/C API render the same stages for `--hires`. LTX-2.3 temporal
+upscaling also adjusts the output frame count and audio latent duration;
+spatial or temporal refinement re-encodes supplied keyframes at the output size.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### LTX-2.3 dev T2V
@@ -463,8 +517,16 @@ pass its name without path or extension to `--hires-upscaler`.
## Notes and limitations
+For DGL, pass the separate LTX-2.3 connector file to `load_text_encoder` with
+`connectors_path`. Set `projection_path` to the diffusion checkpoint when it
+contains additional static conditioning weights, including LTX-2.5 connectors.
+The CLI/C API and DGL share this text conditioner. Connector computation
+runs during prompt encoding. The text encoder caches the final video and audio conditions;
+`load_dit` only loads the diffusion network. See the
+[LTX DGL example](../dgl/ltx2/ltx-2-3-dev-txt2video.dgl).
+
Not implemented:
- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE.
-- The temporal latent upscaler and the duration head (`--auto-duration`); pass
+- The duration head (`--auto-duration`); pass
`--video-frames` explicitly.
diff --git a/docs/models/mage_flow.md b/docs/models/mage_flow.md
index eaccdcf..e1169f9 100644
--- a/docs/models/mage_flow.md
+++ b/docs/models/mage_flow.md
@@ -12,8 +12,6 @@
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -31,7 +29,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/mage_flow/mage-flow-turbo-txt2img.dgl) | Mage-Flow Turbo. |
+| [Image editing](../dgl/mage_flow/mage-flow-edit.dgl) | Mage-Flow Edit. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\mage_flow\mage-flow-turbo-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/mage_flow/mage-flow-turbo-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-image
@@ -69,7 +98,11 @@ Use 30 steps for Base models and 4 steps with `--cfg-scale 1` for Turbo models.
#### Image editing
-Mage-Flow-Edit accepts one or more reference images. The default `mage_flow` reference preset sends each image to both Qwen3-VL and the diffusion transformer, caps the VLM copy's longest edge at 384 pixels, and keeps the VAE copy at the requested output resolution.
+Mage-Flow-Edit accepts one or more reference images. The CLI and C API send each
+image to both Qwen3-VL and the diffusion transformer. The vision copy follows
+the encoder's native preprocessing and patch alignment; the VAE copy is resized
+to the requested output resolution. Use a [DGL script](../guides/edit.md#custom-reference-processing)
+to customize these two inputs.
For the Turbo edit checkpoint, use 4 steps and `--cfg-scale 1`.
diff --git a/docs/models/ming_image.md b/docs/models/ming_image.md
index 53dc7a9..0dfdabb 100644
--- a/docs/models/ming_image.md
+++ b/docs/models/ming_image.md
@@ -15,8 +15,6 @@ The example below uses `ming_image_0.1_design_bf16.safetensors` for the DiT. You
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -32,7 +30,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/ming_image/ming-image-0-1-design-txt2img.dgl) | Ming-Image 0.1 Design. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ming_image\ming-image-0-1-design-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/ming_image/ming-image-0-1-design-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-image
diff --git a/docs/models/minimax_h3.md b/docs/models/minimax_h3.md
index 862b04d..9b71cf1 100644
--- a/docs/models/minimax_h3.md
+++ b/docs/models/minimax_h3.md
@@ -56,6 +56,16 @@ Qwen3-VL vision tower, including the three DeepStack mergers. If stored
separately, pass it with `--llm_vision`. Keep using the MiniMax-H3 video and audio
VAEs with either text encoder.
+In DGL, use `load_text_encoder` with `family = "minimax_h3"`. For a separate
+4B connector or vision projector, pass `connectors_path` or `vision_path` to
+that loader. The native generation API trims decoded audio to the duration of
+the output video. In scripts, save decoded frames and audio with
+`video::save(path, {frames=frames, fps_num=24, fps_den=1}, audio=audio, trim_audio=true)`
+to match the generated clip duration.
+For video presentation, pass frame tensors to `encode_prompt` as
+`presentation=[{video={frames=frames, fps_num=24, fps_den=1}, stride=12}]`. Frame rate and sampling
+stride are explicit; VAE reference latents remain a separate input.
+
Both the original time-embedder DiT and the smaller AdaLN curve-table variant
are detected from their weights.
@@ -71,8 +81,6 @@ are detected from their weights.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -91,7 +99,54 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to audio/video](../dgl/minimax_h3/minimax-h3-text-to-av.dgl) | Generate video and its soundtrack together. |
+| [First/last-frame conditioning](../dgl/minimax_h3/minimax-h3-first-last-frame.dgl) | Guide the video with endpoint images. |
+| [Reference to audio/video](../dgl/minimax_h3/minimax-h3-reference-to-av.dgl) | Use image, video and audio references. |
+| [Interpolated latent hires](../dgl/minimax_h3/minimax-h3-latent-hires.dgl) | Resize the draft latent, then run a second sampling pass. |
+| [Learned latent hires](../dgl/minimax_h3/minimax-h3-learned-hires.dgl) | Use a learned latent upscaler between sampling passes. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\minimax_h3\minimax-h3-text-to-av.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/minimax_h3/minimax-h3-text-to-av.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+#### Latent upscaling
+
+Use `load_latent_upscaler` to load the learned spatial upscaler.
+`upscale_latent` takes the video part of the draft diffusion latent
+and returns a diffusion latent for the refine pass. Pass the target width
+and height in latent units. A resize target uses even dimensions
+(multiples of 32 in pixel units).
+
+```dgl
+upscaler = load_latent_upscaler("minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors")
+bigger = upscale_latent(upscaler, draft_av.video, width = 76, height = 42)
+```
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-audio-video
@@ -104,7 +159,7 @@ Select optional components when required by your checkpoint or task.
--vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors `
--audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors `
--llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf `
- -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten`’s energetic surfing." `
+ -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten`鈥檚 energetic surfing." `
--cfg-scale 1.0 `
-W 864 `
-H 480 `
@@ -124,7 +179,7 @@ Select optional components when required by your checkpoint or task.
--vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \
--audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \
--llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \
- -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." \
+ -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten鈥檚 energetic surfing." \
--cfg-scale 1.0 \
-W 864 \
-H 480 \
@@ -202,7 +257,7 @@ video soundtracks, and standalone audio references:
--vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors `
--audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors `
--llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf `
- -p "Use the cat from <Picture 1> as the main character. Keep the cat`’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat`’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat`’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat`’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat`’s identity unchanged." `
+ -p "Use the cat from <Picture 1> as the main character. Keep the cat`鈥檚 appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat`鈥檚 face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat`鈥檚 head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat`鈥檚 fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat`鈥檚 identity unchanged." `
-r ..\inputs\ernie_image\turbo_example.png `
--cfg-scale 1.0 `
-W 864 `
@@ -223,7 +278,7 @@ video soundtracks, and standalone audio references:
--vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \
--audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \
--llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \
- -p "Use the cat from <Picture 1> as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." \
+ -p "Use the cat from <Picture 1> as the main character. Keep the cat鈥檚 appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat鈥檚 face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat鈥檚 head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat鈥檚 fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat鈥檚 identity unchanged." \
-r ../inputs/ernie_image/turbo_example.png \
--cfg-scale 1.0 \
-W 864 \
@@ -238,11 +293,15 @@ video soundtracks, and standalone audio references:
<video src=../../assets/minimax-h3/r2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
`--ref-image`, `--ref-video`, and `--ref-audio` can each be repeated. A
-reference video is a directory of image frames sorted lexicographically and is
-treated as 24 fps. Repeated `--ref-video-audio` WAV files are paired by index
-with repeated `--ref-video` inputs. WAV PCM (8/16/24/32-bit) and 32/64-bit
-floating-point samples are accepted; audio is converted to stereo 32 kHz by the
-pipeline.
+reference video can be a directory of image frames sorted lexicographically,
+treated as 24 fps, or a file decoded and sampled at 24 fps. File inputs support
+AVI, static/animated WebP and this project's independent-frame VP8 WebM;
+see [DGL media I/O](../dgl.md) for codec and build requirements.
+Reference files retain supported embedded audio by default. Repeated
+`--ref-video-audio` WAV files are paired by index with repeated `--ref-video`
+inputs and replace their embedded audio; the replaced track is not decoded.
+WAV PCM (8/16/24/32-bit) and 32/64-bit floating-point samples are accepted; audio is converted to stereo 32 kHz during
+generation.
Reference inputs are presented to Qwen3-VL in image, video, then audio order.
Videos are sampled at 2 fps for the Qwen presentation while their full 24 fps
@@ -250,20 +309,11 @@ latents condition the diffusion transformer. Paired video and audio references
share the same timeline. Ref2VA cannot be combined with `--init-img` or
`--end-img` in one request.
-Reference images keep their aspect ratio and are only downscaled when they
-exceed the configured limit; neither mode scales a reference up. The limit is
-selected by `ref_image_size`. The default, `match`, uses the pixel area of the
-requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or
-`ref_image_size` in the `ref_image_args` video-option string) to use a 2048 px short
-edge instead, which preserves more identity detail at a higher sampling cost:
-reference latents are attended at every sampling step.
-
-Whichever limit applies, each axis is then snapped to the nearest 32-pixel
-multiple. An axis of at least 16 pixels can therefore grow by up to 16 pixels
-past the scaled size (a 48-pixel axis becomes 64), and references are never
-narrower than 32 pixels.
-`resize_before_vae=false` is not supported here; MiniMax-H3 keeps references on
-the 32-pixel grid and reports a warning.
+Reference images preserve their aspect ratio and are downscaled only when
+their pixel area exceeds the generation canvas. Each axis is then snapped to
+the nearest 32-pixel multiple, with a minimum of 32. An axis may grow by up to
+16 pixels during alignment. Custom reference sizes and preprocessing require a
+[DGL program](../guides/edit.md#custom-reference-processing).
The C API exposes the same inputs through `ref_images`, `ref_videos`, and
the `ref_audios` video-option key. Each `sd_video_t` supplies its own
@@ -456,3 +506,5 @@ They already include video flow shift 12, so keep the default flow shift.
block's working memory must also fit within the VRAM budget.
- The default video flow shift is 12. The audio stream is mapped internally to
its shift of 3, so the regular samplers can operate on the packed AV latent.
+ CLI and C API translation keep this mapping consistent when `--flow-shift`
+ is overridden, including hires refinement.
diff --git a/docs/models/minimax_h3_image.md b/docs/models/minimax_h3_image.md
index 6a06236..abc2264 100644
--- a/docs/models/minimax_h3_image.md
+++ b/docs/models/minimax_h3_image.md
@@ -41,8 +41,6 @@ VAE are incompatible. No audio VAE is needed.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -58,7 +56,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/minimax_h3_image/minimax-h3-image-txt2img.dgl) | MiniMax-H3-Image generation. |
+| [Image editing](../dgl/minimax_h3_image/minimax-h3-image-edit.dgl) | Edit a reference image. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\minimax_h3_image\minimax-h3-image-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/minimax_h3_image/minimax-h3-image-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-image
diff --git a/docs/models/minit2i.md b/docs/models/minit2i.md
index 3e09b75..c658a45 100644
--- a/docs/models/minit2i.md
+++ b/docs/models/minit2i.md
@@ -11,8 +11,6 @@ MiniT2I uses a MiniT2I diffusion transformer and `google/flan-t5-large` as the t
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -28,7 +26,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [CUDA](../dgl/minit2i/minit2i-cuda-txt2img.dgl) | Text-to-image generation on CUDA. |
+| [Metal](../dgl/minit2i/minit2i-metal-txt2img.dgl) | Run with `--threads 8`. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\minit2i\minit2i-cuda-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/minit2i/minit2i-cuda-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### CUDA
diff --git a/docs/models/moss_image.md b/docs/models/moss_image.md
index 398b919..30cdb00 100644
--- a/docs/models/moss_image.md
+++ b/docs/models/moss_image.md
@@ -13,9 +13,12 @@ MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as th
Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer.
-## Usage
+For [inline prompt images](../guides/prompt_images.md), use a matching Qwen3.5
+vision projector. Pass it with `--llm_vision` in the CLI or `vision_path` on
+`load_text_encoder` in dgl. See the
+[DGL example](../dgl/moss_image/moss-image-sft-inline-images.dgl).
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+## Usage
### Web UI
@@ -33,7 +36,39 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Base: text to image](../dgl/moss_image/moss-image-base-txt2img.dgl) | MOSS-Image Base. |
+| [SFT: text to image](../dgl/moss_image/moss-image-sft-txt2img.dgl) | MOSS-Image SFT. |
+| [Inline images](../dgl/moss_image/moss-image-sft-inline-images.dgl) | Insert named images into the prompt. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\moss_image\moss-image-base-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/moss_image/moss-image-base-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### MOSS-Image Base
@@ -105,4 +140,4 @@ Select optional components when required by your checkpoint or task.
<img src="../../assets/moss/sft.png" />
-Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline.
+Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The CLI and C API set `skip_layers = [22]` on the negative predictor when translating requests to DGL, matching the upstream pipeline. Handwritten DGL scripts specify this zero-based layer index explicitly, as shown in the examples.
diff --git a/docs/models/ovis_image.md b/docs/models/ovis_image.md
index ddf6c21..fb33962 100644
--- a/docs/models/ovis_image.md
+++ b/docs/models/ovis_image.md
@@ -12,8 +12,6 @@ Ovis-Image generates images from text using an Ovis 2.5 encoder and the FLUX VAE
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -29,7 +27,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/ovis_image/ovis-image-txt2img.dgl) | Ovis-Image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\ovis_image\ovis-image-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/ovis_image/ovis-image-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/p_h3_image.md b/docs/models/p_h3_image.md
index eed5784..2de8088 100644
--- a/docs/models/p_h3_image.md
+++ b/docs/models/p_h3_image.md
@@ -13,6 +13,9 @@ P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax
Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`.
+In DGL, set `family = "p_h3_image"` on the separately loaded DiT, text encoder,
+and VAE. Pass the connector to `load_text_encoder` with `connectors_path`.
+
The MiniMax-H3 Qwen3-VL-32B encoder does not need a text connector. To use it in
either example below, omit `--embeddings-connectors` and replace `--llm` with:
@@ -35,8 +38,6 @@ Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -52,7 +53,38 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/p_h3_image/p-h3-image-txt2img.dgl) | P-H3-Image generation. |
+| [Image editing](../dgl/p_h3_image/p-h3-image-edit.dgl) | Edit a reference image. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\p_h3_image\p-h3-image-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/p_h3_image/p-h3-image-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-image
diff --git a/docs/models/photo_maker.md b/docs/models/photo_maker.md
index 1bc8ab3..5c22a98 100644
--- a/docs/models/photo_maker.md
+++ b/docs/models/photo_maker.md
@@ -3,16 +3,18 @@
[PhotoMaker](https://github.com/TencentARC/PhotoMaker) personalizes image
generation from identity images. It requires an SDXL base model.
+**Currently supported only through dgl.**
+
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| SDXL checkpoint | [SDXL guide](sd.md) | `-m` / `--model` | Required; PhotoMaker is SDXL-only |
-| SDXL VAE | VAE compatible with the selected SDXL checkpoint | `--vae` | Use an override when required |
-| PhotoMaker v1 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMaker) | `--photo-maker` | For v1; the original .bin release is unsupported |
-| PhotoMaker v2 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMakerV2) | `--photo-maker` | Alternative to v1; requires precomputed face embeddings |
-| Identity images | A directory of portraits of the same person | `--pm-id-images-dir` | Required |
-| PhotoMaker v2 embeddings | `id_embeds.bin` from the preparation below | `--pm-id-embed-path` | Required for v2 |
+| SDXL checkpoint | [SDXL guide](sd.md) | `load_unet`, `load_text_encoder` | Required; PhotoMaker is SDXL-only |
+| SDXL VAE | VAE compatible with the selected SDXL checkpoint | `load_vae` | Use an override when required |
+| PhotoMaker v1 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMaker) | `photomaker::load` | For v1; the original .bin release is unsupported |
+| PhotoMaker v2 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMakerV2) | `photomaker::load` | Alternative to v1; requires precomputed face embeddings |
+| Identity images | A directory of portraits of the same person | `image::load_dir` | Required |
+| PhotoMaker v2 embeddings | `id_embeds.npy` from the preparation below | `tensor::load` | Required for v2 |
### Prepare PhotoMaker v2 embeddings
@@ -32,126 +34,69 @@ python scripts\face_detect.py .\inputs\photomaker_examples\scarletthead_woman
python scripts/face_detect.py ./inputs/photomaker_examples/scarletthead_woman
```
-The script requires its Python dependencies, including PyTorch, NumPy, Diffusers,
-and InsightFace. It writes `id_embeds.bin` inside the input directory. Compute it
-once and reuse it for later generations. The image directory and `id_embeds.bin` stay under the repository's `inputs/`
-directory.
+The script uses NumPy, Diffusers, and InsightFace. It writes
+`id_embeds.npy` with one float32 embedding per image, in filename order.
+Reuse the file with the same reference images, in the same order.
+These DGL examples use `.npy` embeddings.
-## Usage
+For DGL, read the file with `tensor::load` and pass it as
+`photomaker::encode_images(id_embedding=...)` alongside the images in the same order.
+See the [PhotoMaker v2 example](../dgl/photo_maker/photomaker-v2-sdxl.dgl).
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+## Usage
### Web UI
-Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
-server, then select the recipe matching your weights. Fill the component slots
-below and click **Load models**.
-Use the files and compatibility requirements in [Model files](#model-files).
-
-| Recipe | Component slots |
-| --- | --- |
-| SDXL PhotoMaker (`sdxl-photomaker`) | `model`, `photo_maker`, `vae`, `tae` |
+Open the server's **DGL editor**, edit the same script, and submit it.
+Model and input paths must satisfy the server's file-access policy; save outputs
+inside its `output/` directory. See the [Web UI guide](../../examples/server/README.md#web-ui).
-Slot IDs correspond to CLI component options: `diffusion_model` uses
-`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
-Select optional components when required by your checkpoint or task.
+### DGL
-Select the appropriate PhotoMaker weights and supply the identity images.
-PhotoMaker v2 also needs the prepared identity embedding.
+| Example | Description |
+| --- | --- |
+| [PhotoMaker v1](../dgl/photo_maker/photomaker-v1-sdxl.dgl) | Generate from reference identity images with SDXL. |
+| [PhotoMaker v2](../dgl/photo_maker/photomaker-v2-sdxl.dgl) | Use reference images and prepared `.npy` identity embeddings. |
-### CLI
+#### Run a script
-#### PhotoMaker Version 1
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sdxlUnstableDiffusers_v11.safetensors `
- --vae ..\models\vae\sdxl_vae.safetensors `
- --photo-maker ..\models\photomaker\photomaker-v1.safetensors `
- --pm-id-images-dir ..\inputs\photomaker_examples\scarletthead_woman `
- -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" `
- -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" `
- --cfg-scale 5.0 `
- --sampling-method euler `
- --steps 50 `
- -H 1024 `
- -W 1024 `
- --pm-style-strength 10 `
- --vae-on-cpu
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\photo_maker\photomaker-v1-sdxl.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors \
- --vae ../models/vae/sdxl_vae.safetensors \
- --photo-maker ../models/photomaker/photomaker-v1.safetensors \
- --pm-id-images-dir ../inputs/photomaker_examples/scarletthead_woman \
- -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" \
- -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" \
- --cfg-scale 5.0 \
- --sampling-method euler \
- --steps 50 \
- -H 1024 \
- -W 1024 \
- --pm-style-strength 10 \
- --vae-on-cpu
-```
-
-#### PhotoMaker Version 2
-
-From the build directory, use the prepared identity embedding:
-
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-cli.exe `
- -m ..\models\checkpoints\sdxlUnstableDiffusers_v11.safetensors `
- --vae ..\models\vae\sdxl_vae.safetensors `
- --photo-maker ..\models\photomaker\photomaker-v2.safetensors `
- --pm-id-images-dir ..\inputs\photomaker_examples\scarletthead_woman `
- --pm-id-embed-path ..\inputs\photomaker_examples\scarletthead_woman\id_embeds.bin `
- -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" `
- -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" `
- --cfg-scale 5.0 `
- --sampling-method euler `
- --steps 50 `
- -H 1024 `
- -W 1024 `
- --pm-style-strength 10 `
- --vae-on-cpu
+./bin/sd-cli -M dgl --script ../docs/dgl/photo_maker/photomaker-v1-sdxl.dgl
```
-**Linux/macOS**
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
-```sh
-./bin/sd-cli \
- -m ../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors \
- --vae ../models/vae/sdxl_vae.safetensors \
- --photo-maker ../models/photomaker/photomaker-v2.safetensors \
- --pm-id-images-dir ../inputs/photomaker_examples/scarletthead_woman \
- --pm-id-embed-path ../inputs/photomaker_examples/scarletthead_woman/id_embeds.bin \
- -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" \
- -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" \
- --cfg-scale 5.0 \
- --sampling-method euler \
- --steps 50 \
- -H 1024 \
- -W 1024 \
- --pm-style-strength 10 \
- --vae-on-cpu
-```
+Load the identity encoder with `photomaker::load` and pass the reference
+images to `photomaker::encode_images`. Save the resulting tensor with `tensor::save`
+to reuse the same identity across prompts or programs. Place the `img`
+trigger after the class word in each prompt. `photomaker::encode_prompt`
+uses the token count to allocate identity slots and returns their mask.
+`photomaker::apply` merges the tokens into those slots. Use
+`switch_predictor` to select the activation step and `apply_lora` to load
+the checkpoint's LoRA onto the denoiser.
## Options
-The prompt must include a class word followed by the trigger word `img`, for
-example `a girl img`. Class words include `man`, `woman`, `girl`, and `boy`.
-For Asian identity images, the original workflow places `Asian` before the class word.
+The prompt must contain a class word followed by the trigger word `img`, for
+example `a woman img`. The scripts expose `style_strength` in the range 0–100
+and pass `floor(steps * style_strength / 100)` to `switch_predictor`. Before that
+step sampling uses the plain text condition; afterward it uses the merged
+identity tokens. The examples use CFG 5 and 1024 × 1024.
-`--pm-style-strength` accepts 0-100 and defaults to 20. The original examples use
-10-20; lower values follow the source identity more closely. Use CFG 5 and
-1024 x 1024 as in the example. On GPUs with 8 GB or less, `--vae-on-cpu` can help
-with VAE memory usage.
+The VAE is explicitly placed on CPU with `load_vae(..., backend="cpu")`.
+Change that loader to select another compute device; identity encoding,
+LoRA application and prompt encoding remain separate operations.
diff --git a/docs/models/pid.md b/docs/models/pid.md
index 7f89f1b..ff44b3c 100644
--- a/docs/models/pid.md
+++ b/docs/models/pid.md
@@ -20,8 +20,6 @@ PiD and PiD 1.5 require an external Gemma 2 `tokenizer.json` matching the text e
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -44,7 +42,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/pid/pid-flux1-512-to-2048-txt2img.dgl) | Four-step generation from 512 to 2048 pixels. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\pid\pid-flux1-512-to-2048-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/pid/pid-flux1-512-to-2048-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/pixart.md b/docs/models/pixart.md
index 336e1ee..f31d276 100644
--- a/docs/models/pixart.md
+++ b/docs/models/pixart.md
@@ -17,8 +17,6 @@ encoder and a 4-channel VAE: SDXL-style for PixArt-Σ and SD1.x-style for PixArt
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -36,7 +34,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl) | PixArt Sigma XL2 at 1024 pixels. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\pixart\pixart-sigma-xl2-1024-ms-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/pixart/pixart-sigma-xl2-1024-ms-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/pulid.md b/docs/models/pulid.md
index 188fbb8..7e5b6fe 100644
--- a/docs/models/pulid.md
+++ b/docs/models/pulid.md
@@ -4,13 +4,15 @@
identity while generating new images with FLUX.1 dev or schnell. Compute the
identity embedding once with the external Python tool, then reuse it.
+**Currently supported only through dgl.**
+
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| FLUX.1 base components | [FLUX.1 guide](flux.md) | `--diffusion-model`, `--vae`, `--clip_l`, `--t5xxl` | Required; dev or schnell |
-| PuLID v0.9.x weights | [guozinan/PuLID](https://huggingface.co/guozinan/PuLID) | `--pulid-weights` | v0.9.1 is the tested version; v1.1 is unsupported |
-| Identity embedding | `source.pulidembd` from preparation below | `--pulid-id-embedding` | Required for identity preservation |
+| FLUX.1 base components | [FLUX.1 guide](flux.md) | `load_dit`, `load_vae`, `load_text_encoder` | Required; dev or schnell |
+| PuLID v0.9.x weights | [guozinan/PuLID](https://huggingface.co/guozinan/PuLID) | `pulid::load` | v0.9.1 is the tested version; v1.1 is unsupported |
+| Identity embedding | `source.pulidembd` from preparation below | `pulid::load_embedding` | Required for identity preservation |
### Prepare the identity embedding
@@ -48,152 +50,73 @@ The examples below read the same file from the build directory.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
-Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
-server, then select the recipe matching your weights. Fill the component slots
-below and click **Load models**.
-Use the files and compatibility requirements in [Model files](#model-files).
+Open the server's **DGL editor**, edit the same script, and submit it.
+Model and input paths must satisfy the server's file-access policy; save outputs
+inside its `output/` directory. See the [Web UI guide](../../examples/server/README.md#web-ui).
-| Recipe | Component slots |
+### DGL
+
+| Example | Description |
| --- | --- |
-| FLUX.1 dev PuLID (`flux1-pulid`) | `diffusion_model`, `clip_l`, `t5xxl`, `pulid_weights`, `vae`, `tae` |
+| [Identity-guided generation](../dgl/pulid/pulid-flux1-schnell.dgl) | Use a prepared identity embedding with FLUX.1 schnell. |
-Slot IDs correspond to CLI component options: `diffusion_model` uses
-`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
-Select optional components when required by your checkpoint or task.
+#### Run a script
-### CLI
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- --diffusion-model ..\models\diffusion_models\flux1-schnell-Q4_K_S.gguf `
- --vae ..\models\vae\ae.safetensors `
- --clip_l ..\models\text_encoders\clip_l.safetensors `
- --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors `
- --pulid-weights ..\models\pulid\pulid_flux_v0.9.1.safetensors `
- --pulid-id-embedding ..\inputs\source.pulidembd `
- -p "candid photograph of a young woman on a beach at sunset" `
- --cfg-scale 1.0 `
- --sampling-method euler `
- --steps 4 `
- -W 512 `
- -H 512 `
- --pulid-id-weight 1.0 `
- --clip-on-cpu `
- --seed 42 `
- -o out.png
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\pulid\pulid-flux1-schnell.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- --diffusion-model ../models/diffusion_models/flux1-schnell-Q4_K_S.gguf \
- --vae ../models/vae/ae.safetensors \
- --clip_l ../models/text_encoders/clip_l.safetensors \
- --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \
- --pulid-weights ../models/pulid/pulid_flux_v0.9.1.safetensors \
- --pulid-id-embedding ../inputs/source.pulidembd \
- -p "candid photograph of a young woman on a beach at sunset" \
- --cfg-scale 1.0 \
- --sampling-method euler \
- --steps 4 \
- -W 512 \
- -H 512 \
- --pulid-id-weight 1.0 \
- --clip-on-cpu \
- --seed 42 \
- -o out.png
+./bin/sd-cli -M dgl --script ../docs/dgl/pulid/pulid-flux1-schnell.dgl
```
-For Flux Dev (instead of Schnell), add `--guidance 3.5` and `--steps 20`.
-
-## Options
-
-| Flag | Purpose |
-|----------------------------|-------------------------------------------------------------------|
-| `--pulid-weights <path>` | Path to `pulid_flux_v0.9.x.safetensors`. Loaded with the model. |
-| `--pulid-id-embedding <p>` | Path to a `.pulidembd` binary produced by the precompute tool. |
-| `--pulid-id-weight <f>` | Identity-injection strength. Typical 0.7-1.2; default 1.0. |
-
-Supply both the PuLID weights and identity embedding to activate identity
-conditioning. `--pulid-id-weight` defaults to 1.0; set it to 0 to disable identity
-injection. Loading `--pulid-weights` without an embedding leaves identity
-conditioning inactive.
-
-### Backend selection
-
-The standard `--backend` flag works as documented. Common patterns:
-
-**Windows (PowerShell) arguments**
-
-```text
-# AMD Vulkan
---backend "diffusion=vulkan0,vae=cpu"
-
-# NVIDIA Vulkan
---backend "diffusion=vulkan1,vae=cpu"
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
-# CUDA
---backend "diffusion=cuda0,vae=cpu"
-```
+Load PuLID weights separately with `pulid::load` on the same compute backend
+as the FLUX.1 denoiser. Read the identity embedding with
+`pulid::load_embedding` and bind it to a text condition with `pulid::apply`
+before calling `predictor`. Each binding sets its own `strength` and can
+set `start_sigma` and `end_sigma` to bound the contribution in sigma space.
+Omitting both bounds enables the binding throughout sampling.
-**Linux/macOS arguments**
+For FLUX.1 dev, choose its checkpoint and set `steps = 20`. The predictor's
+default guidance is 3.5.
-```text
-# AMD Vulkan
---backend diffusion=vulkan0,vae=cpu
+## Options
-# NVIDIA Vulkan
---backend diffusion=vulkan1,vae=cpu
+Load weights with `pulid::load`, read the prepared `.pulidembd` file with
+`pulid::load_embedding`, and attach it to the positive condition with
+`pulid::apply(..., strength=1.0)`. Typical identity strengths are 0.7–1.2.
+Optional `start_sigma` and `end_sigma` bound the active noise interval.
-# CUDA
---backend diffusion=cuda0,vae=cpu
-```
+### Backend selection
-The PuLID cross-attention layers run on the same backend as the main
-diffusion model. They have not yet been independently profiled on every
-backend; only Vulkan and CPU have been tested by the original contributor.
+DGL selects devices on individual loaders. Use the same compute backend for
+`load_dit` and `pulid::load`; for example `backend="cuda0"`, `"vulkan0"` or
+`"vulkan1"`, according to the available devices. To decode on CPU set
+`load_vae(..., backend="cpu")`. `params_backend="cpu"` offloads weights while
+leaving computation on the selected compute device.
+See [backend selection](../guides/backend.md).
## Notes and limitations
-PuLID v1.1 weights are unsupported. Use PuLID and skip-layer guidance
-(`--skip-layers`) separately. One precomputed identity embedding is accepted per
-request; prepare it using the external tool. Identity conditioning is applied
-to the positive CFG branch.
-
-At 512x512, 4 steps (Schnell), the 20 cross-attention layers add roughly
-10% to denoise time and almost nothing to peak VRAM. Tested on a 12 GB
-consumer card alongside Flux Schnell Q4 GGUF + CPU-offloaded clip_l and
-t5xxl + GPU-resident VAE.
-
-At 1024x1024 with Flux Dev Q4 + 20 steps + PuLID, the VAE decode compute
-buffer doesn't fit on a 12 GB card even with `--vae-on-cpu`. Workaround:
-explicitly route VAE to the CPU backend instead of the offload flag:
-
-**Windows (PowerShell) arguments**
-
-```text
---backend "diffusion=vulkan0,vae=cpu"
-```
-
-**Linux/macOS arguments**
-
-```text
---backend diffusion=vulkan0,vae=cpu
-```
-
-The `--vae-on-cpu` flag offloads VAE weights but leaves the compute graph
-on the default backend; this is existing stable-diffusion.cpp behavior,
-not a PuLID-specific issue. Documented here because anyone running PuLID
-at 1024 will hit it.
+PuLID v1.1 weights are unsupported. One precomputed identity embedding is
+used by this script and attached to the positive condition. Prepare it with
+the external tool before running the program.
-The original contributor tested Vulkan and CPU. Other backends were not
-validated in those tests. See [backend selection](../guides/backend.md) for
-device placement. Internal format details and validation procedures are in the
-[PuLID developer notes](../development/pulid.md).
+Large VAE decode workspaces can exceed GPU memory at 1024 × 1024; explicitly
+select `backend="cpu"` on `load_vae` when needed. The original contributor
+validated PuLID on Vulkan and CPU; those results do not establish support or
+performance on every backend.
diff --git a/docs/models/qwen_image.md b/docs/models/qwen_image.md
index 030f2f7..c1379ac 100644
--- a/docs/models/qwen_image.md
+++ b/docs/models/qwen_image.md
@@ -12,8 +12,6 @@ Qwen Image generates images from text using Qwen2.5-VL and the Qwen Image VAE.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -29,7 +27,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/qwen_image/qwen-image-txt2img.dgl) | Qwen Image generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\qwen_image\qwen-image-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/qwen_image/qwen-image-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/qwen_image_2.1.md b/docs/models/qwen_image_2.1.md
index 7ddfda5..0953f97 100644
--- a/docs/models/qwen_image_2.1.md
+++ b/docs/models/qwen_image_2.1.md
@@ -16,8 +16,6 @@ For faster encoding, decoding, or previews, see [TAEQI2.1](taesd.md#qwen-image-2
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -34,7 +32,47 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl) | Generate an image from a prompt. |
+| [Image editing](../dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl) | Edit a reference image; inputs are aligned automatically. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\qwen_image_2.1\qwen-image-2-1-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/qwen_image_2.1/qwen-image-2-1-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+The [editing script](../dgl/qwen_image_2.1/qwen-image-2-1-edit.dgl) loads the reference
+at its original size. `vae_encode` and `encode_prompt` each align their input
+to multiples of 32, as required by the mapping between vision image slots and
+reference latent tokens. The VAE still compresses by a factor of 16.
+Already aligned inputs skip resizing. `latent_shape` creates a compatible output
+canvas from the reference dimensions, without imposing a fixed resolution.
+Use `tensor::interpolate` or `tensor::crop` first to choose a different size.
+See [DGL media dimensions](../dgl.md#media-dimensions) for rounding and video rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
Use image dimensions divisible by 32. The resolution-dependent flow schedule is selected automatically.
@@ -199,3 +237,12 @@ Disable caching with `--model-args qwen_image_2_1_prefix_cache=false`. Numerical
results can differ slightly because caching changes computation sizes. See the
[prefix cache developer notes](../development/qwen_image_prefix_cache.md) for
storage compatibility and implementation details.
+
+### DGL system instructions
+
+`encode_prompt` uses the model's default system message when `system` is omitted.
+`system="your instruction"` replaces its body; `system=""` removes the whole
+system segment and adjusts the output crop accordingly. User/assistant formatting
+and reference-image handling remain model-defined. `raw=true` accepts the complete
+tokenizer input without adding template segments or cropping a template prefix.
+Do not combine it with `system` or image bindings. See [DGL prompt formatting](../dgl.md#independent-generation-components).
diff --git a/docs/models/qwen_image_edit.md b/docs/models/qwen_image_edit.md
index adec6ff..3dacdae 100644
--- a/docs/models/qwen_image_edit.md
+++ b/docs/models/qwen_image_edit.md
@@ -12,8 +12,6 @@ Qwen Image Edit supports reference-image editing with the original, 2509, and 25
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -31,7 +29,39 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Qwen Image Edit](../dgl/qwen_image_edit/qwen-image-edit.dgl) | Edit a reference image. |
+| [Qwen Image Edit 2509](../dgl/qwen_image_edit/qwen-image-edit-2509.dgl) | Use the 2509 checkpoint. |
+| [Qwen Image Edit 2511](../dgl/qwen_image_edit/qwen-image-edit-2511.dgl) | Use the 2511 checkpoint. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\qwen_image_edit\qwen-image-edit.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/qwen_image_edit/qwen-image-edit.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Qwen Image Edit
diff --git a/docs/models/relay_vsr.md b/docs/models/relay_vsr.md
index 4900fa4..ce6eeb6 100644
--- a/docs/models/relay_vsr.md
+++ b/docs/models/relay_vsr.md
@@ -1,19 +1,21 @@
# RelayVSR
[RelayVSR](https://github.com/kopperx/RelayVSR) upscales images and videos by
-2x or 4x with `-M vid_upscale`.
+2x or 4x through separately loaded diffusion and VAE components.
+
+**Currently supported only through dgl.**
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| RelayVSR diffusion model | [ComfyUI RelayVSR](https://huggingface.co/Comfy-Org/RelayVSR) | `--diffusion-model` | Match the requested 2x or 4x scale |
-| RelayVSR VAE | Matching scale-specific VAE from the same repository | `--vae` | Required; ordinary Wan VAE is incompatible |
+| RelayVSR diffusion model | [ComfyUI RelayVSR](https://huggingface.co/Comfy-Org/RelayVSR) | `load_dit` | Match the requested 2x or 4x scale |
+| RelayVSR VAE | Matching scale-specific VAE from the same repository | `load_vae` | Required; ordinary Wan VAE is incompatible |
[ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/RelayVSR) load
directly. Select the matching pair for the requested scale:
-| `--upscale-factor` | `--diffusion-model` | `--vae` |
+| Scale | `load_dit` | `load_vae` |
| --- | --- | --- |
| 2 | `relayvsr_x2_lightx2v_int8_convrot.safetensors` | `relayvsr_flash_vae_x2_fp32.safetensors` |
| 4 | `relayvsr_x4_int8_convrot.safetensors` | `relayvsr_flash_vae_x4_fp32.safetensors` |
@@ -22,60 +24,54 @@ No separate text encoder or ordinary Wan VAE is needed.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Upscale a video](../dgl/relay_vsr/relayvsr-x2-video-upscale.dgl) | Process overlapping windows with the 2x model. |
+| [Upscale an image](../dgl/relay_vsr/relayvsr-x2-image-upscale.dgl) | Run the same 2x model on one image. |
-### CLI
+#### Run a script
-For a video, extract numbered frames into a directory, then run:
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_upscale `
- --diffusion-model ..\models\diffusion_models\relayvsr_x2_lightx2v_int8_convrot.safetensors `
- --vae ..\models\vae\relayvsr_flash_vae_x2_fp32.safetensors `
- --input-frames input_frames `
- --upscale-factor 2 `
- --fps 24 `
- --vsr-window-size 16 `
- -s 42 `
- -o output.avi
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\relay_vsr\relayvsr-x2-video-upscale.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -M vid_upscale \
- --diffusion-model ../models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors \
- --vae ../models/vae/relayvsr_flash_vae_x2_fp32.safetensors \
- --input-frames input_frames \
- --upscale-factor 2 \
- --fps 24 \
- --vsr-window-size 16 \
- -s 42 \
- -o output.avi
+./bin/sd-cli -M dgl --script ../docs/dgl/relay_vsr/relayvsr-x2-video-upscale.dgl
```
-Frames are read in filename order. Set `--fps` to the source video's frame rate.
-Video outputs support AVI, WebM, and animated WebP. To save individual frames,
-use an output pattern such as `-o output/%04d.png`.
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+Set `input_path` to a numbered-frame directory or a video supported by
+`video::load`.
-For a single image, replace `--input-frames input_frames` with `-i input.png`
-and use `-o output.png` (or JPEG/WebP). No FPS setting is needed for an image.
-`-i` / `--init-img` and `--input-frames` are mutually exclusive.
+Frame directories are read in filename order. Set the script's `fps` to the
+desired output rate. `video::save` writes AVI and receives the original audio
+separately. The [single-image script](../dgl/relay_vsr/relayvsr-x2-image-upscale.dgl)
+uses `image::load` / `image::save` instead.
## Notes and limitations
-- `--upscale-factor` is required. Use 2 or 4 with the corresponding weights.
- Output width and height are exactly the input dimensions times the scale.
-- `--vsr-window-size` sets the reference-frame window, including both endpoints.
- The default 16 selects reference frames 0, 15, 30, and so on, and always
- includes the final frame. Every input frame appears once in the output.
-- `--flash-attn auto` and `--conv-direct auto` are enabled by default. See
- [backend selection](../guides/backend.md) to choose a device or weight placement.
-- Sampler, scheduler, and step settings do not configure RelayVSR.
- `--vsr-temporal-chunk-size` is a SeedVR2 option.
-- Long videos require enough system memory to hold the input and output frames.
- Audio is not copied to the output.
+- Choose matching 2× or 4× diffusion/VAE weights. Their architecture determines
+ the output scale; ordinary Wan VAE weights are incompatible.
+- `window_size = 16` splits the input into windows with one overlapping frame.
+ The script samples endpoint keyframes for each window, decodes against the
+ source frames, then joins windows by dropping duplicate overlap frames.
+- Sampling is an explicit one-step flow with `[1, 0]` sigmas and Euler. There
+ is no separate native video-upscale sampler configuration.
+- Loader attention and convolution settings use their normal auto defaults.
+ See [backend selection](../guides/backend.md) for placement.
+- `video::load` limits decoded input to 256 MiB. Window outputs use the general
+ 1 GiB tensor limit, and joined video may use up to 4 GiB. Long clips or large
+ frames need a more granular input/output program.
diff --git a/docs/models/sd.md b/docs/models/sd.md
index 2d1ab53..ae0fded 100644
--- a/docs/models/sd.md
+++ b/docs/models/sd.md
@@ -15,8 +15,6 @@ generation from single-file checkpoints.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -37,7 +35,43 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [SD 1.4](../dgl/sd/sd1-4-txt2img.dgl) | Text-to-image generation. |
+| [SD 1.5](../dgl/sd/sd1-5-txt2img.dgl) | Text-to-image generation. |
+| [SDXL](../dgl/sd/sdxl-txt2img.dgl) | Text-to-image generation. |
+| [SD 1.4 image to image](../dgl/sd/sd1-4-img2img.dgl) | Start from an existing image. |
+| [SD 1.5 ControlNet](../dgl/sd/sd1-5-controlnet.dgl) | Use a pose control image. |
+| [SD 1.5 inpainting](../dgl/sd/sd1-5-inpainting.dgl) | Use an inpainting checkpoint and mask. |
+| [NitroSD Realism / NitroFusion](../dgl/sd/nitrosd-realism-nitrofusion.dgl) | Explicit noise schedule and custom prediction transform. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sd\sd1-4-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/sd/sd1-4-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### SD 1.4
diff --git a/docs/models/sd3.md b/docs/models/sd3.md
index 12cab00..335cbb2 100644
--- a/docs/models/sd3.md
+++ b/docs/models/sd3.md
@@ -13,8 +13,6 @@ Stable Diffusion 3 and 3.5 generate images from text. The example below uses SD3
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -32,7 +30,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/sd3/sd3-5-large-txt2img.dgl) | Stable Diffusion 3.5 Large. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sd3\sd3-5-large-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/sd3/sd3-5-large-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### SD3.5 Large
diff --git a/docs/models/seedvr2.md b/docs/models/seedvr2.md
index 0a48914..60fb927 100644
--- a/docs/models/seedvr2.md
+++ b/docs/models/seedvr2.md
@@ -1,18 +1,20 @@
# SeedVR2
-SeedVR2 upscales images and videos with `-M vid_upscale`. Support is experimental.
+SeedVR2 upscales images and videos through dgl. Support is experimental.
+
+**Currently supported only through dgl.**
## Model files
-| Component | Weights and downloads | CLI option | Requirement |
+| Component | Weights and downloads | DGL usage | Requirement |
| --- | --- | --- | --- |
-| Diffusion model | [ComfyUI SeedVR2](https://huggingface.co/Comfy-Org/SeedVR2) | `--diffusion-model` | Choose 3B or 7B Sharp below |
-| SeedVR2 VAE | `seedvr2_ema_vae_fp16.safetensors` from the same repository | `--vae` | Required; shared by both variants |
+| Diffusion model | [ComfyUI SeedVR2](https://huggingface.co/Comfy-Org/SeedVR2) | `load_dit` | Choose 3B or 7B Sharp below |
+| SeedVR2 VAE | `seedvr2_ema_vae_fp16.safetensors` from the same repository | `load_vae` | Required; shared by both variants |
These [ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/SeedVR2)
load directly, without conversion or a separate text encoder:
-| Model | `--diffusion-model` | `--vae` |
+| Model | `load_dit` | `load_vae` |
| --- | --- | --- |
| 3B | `seedvr2_3b_int8_convrot.safetensors` | `seedvr2_ema_vae_fp16.safetensors` |
| 7B Sharp | `seedvr2_7b_sharp_int8_convrot.safetensors` | `seedvr2_ema_vae_fp16.safetensors` |
@@ -21,103 +23,74 @@ Use the matching SeedVR2 VAE for either model. The 7B model requires more memory
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
+### DGL
-### CLI
+| Example | Description |
+| --- | --- |
+| [Upscale an image](../dgl/seedvr2/seedvr2-3b-image-upscale.dgl) | Use the 3B diffusion model and matching SeedVR2 VAE. |
+| [Upscale a video](../dgl/seedvr2/seedvr2-3b-video-upscale.dgl) | Process frame windows and retain the source audio. |
-#### Upscale an image
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
**Windows (PowerShell)**
```powershell
-.\bin\Release\sd-cli.exe `
- -M vid_upscale `
- --diffusion-model ..\models\diffusion_models\seedvr2_3b_int8_convrot.safetensors `
- --vae ..\models\vae\seedvr2_ema_vae_fp16.safetensors `
- -i input.png `
- --upscale-factor 2 `
- -s 42 `
- -o restored.png
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\seedvr2\seedvr2-3b-image-upscale.dgl
```
**Linux/macOS**
```sh
-./bin/sd-cli \
- -M vid_upscale \
- --diffusion-model ../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \
- --vae ../models/vae/seedvr2_ema_vae_fp16.safetensors \
- -i input.png \
- --upscale-factor 2 \
- -s 42 \
- -o restored.png
+./bin/sd-cli -M dgl --script ../docs/dgl/seedvr2/seedvr2-3b-image-upscale.dgl
```
-`-i` / `--init-img` accepts a single image. Outputs can be PNG, JPEG, or WebP.
-No FPS or temporal chunk setting is needed for an image.
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+The [SeedVR2 script](../dgl/seedvr2/seedvr2-3b-video-upscale.dgl) loads the DiT and
+causal VAE independently. The script controls output resolution with
+bicubic resizing and pads each window to dimensions divisible by 16 and
+a frame count of `4n+1`. It uses the VAE posterior mode, the checkpoint's
+positive embedding, and a one-step Euler sampler. The condition contains
+the scaled VAE latent and a one-channel task mask. Each decoded window is
+cropped to its original frame count and target dimensions before stitching.
+The intermediate float tensors must fit the DGL tensor size limit of 256 MiB
+per tensor. For large output resolutions, use shorter pixel windows.
-#### Upscale a video
-
-Extract the video into numbered frames, then pass the frame directory:
-
-**Windows (PowerShell)**
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-```powershell
-mkdir frames
-ffmpeg -i input.mp4 -fps_mode passthrough frames\%06d.png
-.\bin\Release\sd-cli.exe `
- -M vid_upscale `
- --diffusion-model ..\models\diffusion_models\seedvr2_3b_int8_convrot.safetensors `
- --vae ..\models\vae\seedvr2_ema_vae_fp16.safetensors `
- --input-frames frames `
- --upscale-factor 2 `
- --fps 24 `
- --vsr-temporal-chunk-size 1 `
- -s 42 `
- -o restored.avi
-```
+#### Upscale an image
-**Linux/macOS**
+The image script reads `input_path` and writes a PNG with `image::save`.
-```sh
-mkdir frames
-ffmpeg -i input.mp4 -fps_mode passthrough frames/%06d.png
-./bin/sd-cli \
- -M vid_upscale \
- --diffusion-model ../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \
- --vae ../models/vae/seedvr2_ema_vae_fp16.safetensors \
- --input-frames frames \
- --upscale-factor 2 \
- --fps 24 \
- --vsr-temporal-chunk-size 1 \
- -s 42 \
- -o restored.avi
-```
+#### Upscale a video
-Set `--fps` to the source video's frame rate; 24 is only the example above.
-Frames are read in filename order. Use either `-i` or `--input-frames`, not both.
+Set `input_path` to numbered frames or a video supported by `video::load`.
-Video outputs support AVI, WebM, and animated WebP. To save individual frames,
-use an output pattern such as `-o restored/%06d.png`.
+Set `fps` to the desired frame rate; the example uses 24. Frame directories
+are read in filename order. Video output is AVI, and the script passes the
+source audio separately to `video::save`.
## Notes and limitations
-- `--upscale-factor` is required and must be a positive integer. Output width
- and height are exactly the input dimensions multiplied by this value.
-- `--vsr-temporal-chunk-size` controls how many video frames are processed
- together. Valid values are 1, 5, 9, 13, and so on; the default is 5. A value of
- 1 processes each frame independently and uses less memory. Larger values use
- neighboring frames but require more memory. Chunk boundaries may be visible.
-- Frame count and order are preserved, including a shorter final chunk. Audio
- is not copied to the output.
-- If you run out of GPU memory, try `--vsr-temporal-chunk-size 1`. A five-frame
- chunk at 1728x960 output resolution exceeded 24 GB of GPU memory in testing.
-- `--flash-attn auto` and `--conv-direct auto` are enabled by default. See
- [backend selection](../guides/backend.md) to choose a device or weight placement.
-- SeedVR2 does not use sampler, scheduler, or step settings.
- `--vsr-window-size` is a RelayVSR option and does not affect SeedVR2.
-
-3B has been tested on CUDA and CPU; 7B has been tested on CUDA with small inputs.
-Other backends have not been tested with this integration.
+- `factor` is a positive integer scale applied to pixel dimensions before
+ VAE encoding. Both 3B and 7B Sharp use the matching SeedVR2 VAE.
+- `window_size` controls processing granularity. The video example uses 1 to
+ process each frame independently and reduce memory; 5, 9, 13, … include
+ temporal context at higher memory cost. Each window is padded to `4n+1`
+ frames and cropped after decoding. Chunk boundaries may be visible.
+- Sampling uses the checkpoint's positive embedding and an explicit one-step
+ Euler flow; no text encoder is needed.
+- Input/output order and frame count are preserved. Input, output and
+ intermediate tensors must fit the 256 MiB per-tensor limit. For larger
+ output frames, reduce window size; long clips need incremental I/O.
+- The existing DGL workflow was tested with 3B and 7B Sharp on Vulkan. The
+ documentation examples in this collection are also statically checked;
+ this does not establish image quality or throughput on every backend.
Model reference: [ByteDance-Seed/SeedVR](https://github.com/ByteDance-Seed/SeedVR).
diff --git a/docs/models/sefi_image.md b/docs/models/sefi_image.md
index 1d6ffbd..deb90ad 100644
--- a/docs/models/sefi_image.md
+++ b/docs/models/sefi_image.md
@@ -34,8 +34,6 @@ python3 scripts/convert_qwen3_vl.py "<hf_repo_dir>/Qwen3-VL-XB-Instruct" "<out_d
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -51,7 +49,55 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [1B Turbo](../dgl/sefi_image/sefi-1b-turbo-txt2img.dgl) | Distilled generation; see variant defaults for 2B. |
+| [1B Base](../dgl/sefi_image/sefi-1b-base-txt2img.dgl) | Base generation; see variant defaults for 2B. |
+| [5B Turbo](../dgl/sefi_image/sefi-5b-turbo-txt2img.dgl) | Run with `--max-vram 8` for the streaming example. |
+| [5B RL](../dgl/sefi_image/sefi-image-5b-rl.dgl) | Use the dual-stream sampler. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sefi_image\sefi-1b-turbo-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/sefi_image/sefi-1b-turbo-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+The [SeFi example](../dgl/sefi_image/sefi-image-5b-rl.dgl) passes the two sigma tensors
+from `sefi::make_sigmas` to `sefi::sample`. Both tensors must be rank-one F32,
+nonnegative, nonincreasing and equal in length. Repeated values describe a
+paused stream while the other stream advances. The sampler uses Euler.
+
+```text
+schedule = sefi::make_sigmas(steps = 20)
+latent = sefi::sample(start, schedule.sigmas, schedule.semantic_sigmas,
+ predict = guided, rng = rng)
+```
+
+The call supports `base`, `denoise_mask`, `mask_input`, `finish`, cache and
+preview options through the shared sampling execution layer. Slice the
+texture channels `[16, 144)` before converting the latent for the Flux2 VAE.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### 1B / 2B turbo
@@ -175,4 +221,9 @@ Select optional components when required by your checkpoint or task.
| RL | 0.3 | 50 | 4.0 |
| turbo | 1.0 | 4 | 1.0 |
-The dispatcher picks `alpha` from the filename (`turbo` substring => 1.0, otherwise 0.3). Override via `--extra-sample-args sefi_alpha=<value>` or `sefi_delta_t=<value>`.
+Set `--extra-sample-args sefi_alpha=0.3` for Base and RL; turbo uses the default
+alpha of 1. Override the second timeline offset with `sefi_delta_t=<value>`.
+CLI and C API translation forward both settings to the initial and hires
+schedules. Image-to-image strength trims both timelines together. Custom sigmas
+replace the texture schedule while the semantic schedule uses the same number
+of steps.
diff --git a/docs/models/sensenova_u1.md b/docs/models/sensenova_u1.md
index e77ab39..fa2cee3 100644
--- a/docs/models/sensenova_u1.md
+++ b/docs/models/sensenova_u1.md
@@ -15,8 +15,6 @@ and the tokenizer files.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the server instructions for
@@ -25,7 +23,51 @@ Pass the complete repository directory with `--model` when starting the server,
using the same model path as the CLI examples below. The browser then uses
**Command-line model (all features)** for that loaded checkpoint.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl) | SenseNova U1.5 on CUDA. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\sensenova_u1\sensenova-u1-cuda-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+#### Prompt formatting
+
+The [DGL example](../dgl/sensenova_u1/sensenova-u1-cuda-txt2img.dgl)
+uses the default system message for its positive prompt and passes the complete
+negative input with `raw=true`. Edit the user text inside that raw string to
+change its negative prompt. An empty negative still includes the user/assistant
+markers and `<img>`, and omits the positive template's `<think>` segment.
+Native CLI/C API generation builds this input automatically.
+
+For custom positive instructions, pass `system="your instruction"` to
+`encode_prompt`; `system=""` removes the system segment but keeps the positive
+assistant template. For a completely custom tokenizer input use `raw=true`
+without `system`. See [DGL prompt formatting](../dgl.md#independent-generation-components).
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### CUDA
diff --git a/docs/models/taesd.md b/docs/models/taesd.md
index 607c0b4..b22b358 100644
--- a/docs/models/taesd.md
+++ b/docs/models/taesd.md
@@ -93,14 +93,43 @@ curl -L -o ../models/vae/taeqi2_1.safetensors https://huggingface.co/madebyollin
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Use a compatible base model recipe that offers the `tae` slot, and select the
tiny autoencoder matching that model. See [Web UI guide](../../examples/server/README.md#web-ui).
-### CLI
+### DGL
+
+The [SD 1.5 example](../dgl/taesd/sd1-5-taesd-preview.dgl) uses TAESD to
+decode the final image and passes the same decoder to `preview(decoder = vae)`.
+Running it with `sd-cli -M dgl` writes the final PNG. To receive intermediate
+images, run it in a host that registers a DGL preview callback in `"tae"` mode.
+See [progress and previews](../dgl.md#progress-preview-and-cancellation).
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\taesd\sd1-5-taesd-preview.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/taesd/sd1-5-taesd-preview.dgl
+```
+
+Add `--check` to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
**Windows (PowerShell)**
diff --git a/docs/models/wan.md b/docs/models/wan.md
index 1b46955..1b4d413 100644
--- a/docs/models/wan.md
+++ b/docs/models/wan.md
@@ -32,8 +32,6 @@ file. For lower VAE memory use, see [tiny autoencoders](taesd.md).
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -56,7 +54,71 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+The Wan text encoder zeros UMT5 padding embeddings before returning the prompt
+condition, matching the model's fixed-length text input. This also applies to
+CLI and C API generation through dgl.
+
+#### Wan 2.1
+
+| Example | Description |
+| --- | --- |
+| [Text to video, 1.3B](../dgl/wan/wan2-1-t2v-1-3b.dgl) | Wan 2.1 T2V. |
+| [Text to video, 14B](../dgl/wan/wan2-1-t2v-14b.dgl) | Wan 2.1 T2V. |
+| [Image to video, 14B](../dgl/wan/wan2-1-i2v-14b.dgl) | Wan 2.1 I2V, 480p. |
+| [First/last-frame conditioning, 14B](../dgl/wan/wan2-1-first-last-frame-14b.dgl) | Wan 2.1 FLF2V, 720p. |
+
+#### Wan 2.2
+
+| Example | Description |
+| --- | --- |
+| [Text to video, A14B](../dgl/wan/wan2-2-t2v-a14b.dgl) | Switch between high- and low-noise experts. |
+| [Image to video, A14B](../dgl/wan/wan2-2-i2v-a14b.dgl) | Use the matching I2V experts. |
+| [Speech to video, 14B](../dgl/wan/wan2-2-s2v-14b.dgl) | Condition a speaker image with audio features. |
+| [Single-image generation, A14B](../dgl/wan/wan2-2-t2v-a14b-image.dgl) | Run T2V experts with one output frame. |
+| [Text to video with LoRA](../dgl/wan/wan2-2-t2v-a14b-lora.dgl) | Patch the diffusion experts before sampling. |
+| [TI2V 5B: text to video](../dgl/wan/wan2-2-ti2v-5b-txt2video.dgl) | Generate without an initial image. |
+| [TI2V 5B: image to video](../dgl/wan/wan2-2-ti2v-5b-img2video.dgl) | Condition the initial frame. |
+| [First/last-frame conditioning, A14B](../dgl/wan/wan2-2-first-last-frame-a14b.dgl) | Use the Wan 2.2 I2V experts. |
+
+#### VACE
+
+| Example | Description |
+| --- | --- |
+| [VACE 1.3B: text to video](../dgl/wan/wan2-1-vace-1-3b-txt2video.dgl) | Generate without control frames. |
+| [VACE 1.3B: reference to video](../dgl/wan/wan2-1-vace-1-3b-ref2video.dgl) | Condition a reference image. |
+| [VACE 1.3B: video to video](../dgl/wan/wan2-1-vace-1-3b-video2video.dgl) | Condition control frames. |
+| [VACE 14B: text to video](../dgl/wan/wan2-1-vace-14b-txt2video.dgl) | Generate without control frames. |
+| [VACE 14B: reference to video](../dgl/wan/wan2-1-vace-14b-ref2video.dgl) | Condition a reference image. |
+| [VACE 14B: video to video](../dgl/wan/wan2-1-vace-14b-video2video.dgl) | Condition control frames. |
+| [VACE Fun A14B: reference to video](../dgl/wan/wan2-2-vace-fun-a14b-ref2video.dgl) | Use Wan 2.2 VACE experts. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\wan\wan2-1-t2v-1-3b.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/wan/wan2-1-t2v-1-3b.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Wan2.1 T2V 1.3B
@@ -208,8 +270,6 @@ Select optional components when required by your checkpoint or task.
--cfg-scale 3.5 `
--sampling-method euler `
--steps 10 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 8 `
--flow-shift 3.0 `
-W 832 `
@@ -233,8 +293,6 @@ Select optional components when required by your checkpoint or task.
--cfg-scale 3.5 \
--sampling-method euler \
--steps 10 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 8 \
--flow-shift 3.0 \
-W 832 \
@@ -263,8 +321,6 @@ Select optional components when required by your checkpoint or task.
--cfg-scale 3.5 `
--sampling-method euler `
--steps 10 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 8 `
--flow-shift 3.0 `
-W 832 `
@@ -290,8 +346,6 @@ Select optional components when required by your checkpoint or task.
--cfg-scale 3.5 \
--sampling-method euler \
--steps 10 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 8 \
--flow-shift 3.0 \
-W 832 \
@@ -420,8 +474,6 @@ Notes:
--cfg-scale 3.5 `
--sampling-method euler `
--steps 10 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 8 `
--flow-shift 3.0 `
-W 832 `
@@ -444,8 +496,6 @@ Notes:
--cfg-scale 3.5 \
--sampling-method euler \
--steps 10 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 8 \
--flow-shift 3.0 \
-W 832 \
@@ -473,8 +523,6 @@ Notes:
--cfg-scale 3.5 `
--sampling-method euler `
--steps 4 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 4 `
--flow-shift 3.0 `
-W 832 `
@@ -499,8 +547,6 @@ Notes:
--cfg-scale 3.5 \
--sampling-method euler \
--steps 4 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 4 \
--flow-shift 3.0 \
-W 832 \
@@ -674,8 +720,6 @@ TI2V 5B can decode multiple latent frames together when temporal tiling is enabl
--cfg-scale 3.5 `
--sampling-method euler `
--steps 10 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 8 `
--flow-shift 3.0 `
-W 480 `
@@ -701,8 +745,6 @@ TI2V 5B can decode multiple latent frames together when temporal tiling is enabl
--cfg-scale 3.5 \
--sampling-method euler \
--steps 10 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 8 \
--flow-shift 3.0 \
-W 480 \
@@ -804,6 +846,12 @@ TI2V 5B can decode multiple latent frames together when temporal tiling is enabl
##### V2V
+`--control-video` accepts a frame directory or a video file (AVI, WebP or
+this project's independent-frame VP8 WebM). Files are decoded at the requested
+`--fps` (default 16), without audio; `--video-frames` limits decoded frames
+after resampling. Missing frames are padded by the existing VACE workflow.
+See [DGL media I/O](../dgl.md) for codec restrictions.
+
**Windows (PowerShell)**
```powershell
@@ -1009,8 +1057,6 @@ for the reference image, same as Wan2.1 VACE.
--cfg-scale 3.5 `
--sampling-method euler `
--steps 10 `
- --high-noise-cfg-scale 3.5 `
- --high-noise-sampling-method euler `
--high-noise-steps 8 `
-W 832 `
-H 480 `
@@ -1034,8 +1080,6 @@ for the reference image, same as Wan2.1 VACE.
--cfg-scale 3.5 \
--sampling-method euler \
--steps 10 \
- --high-noise-cfg-scale 3.5 \
- --high-noise-sampling-method euler \
--high-noise-steps 8 \
-W 832 \
-H 480 \
@@ -1056,6 +1100,12 @@ video, like Wan2.1 VACE t2v).
## Notes and limitations
+Dual experts share CFG, sampler, scheduler and flow shift. By default,
+`--high-noise-steps -1` splits `--steps` at `--moe-boundary`. A nonnegative
+`--high-noise-steps` instead adds that many high-noise steps before the
+`--steps` low-noise steps; `0` runs only the low-noise model.
+Use a DGL script for independently configured experts.
+
Wan models require `-M vid_gen`, including single-frame generation. `--video-frames` alone does not select video mode. Library callers must use `generate_video()` instead of `generate_image()`.
Use the task-specific input and sampling settings in each example. VAE
diff --git a/docs/models/z_image.md b/docs/models/z_image.md
index 9b39e3d..ebd72cb 100644
--- a/docs/models/z_image.md
+++ b/docs/models/z_image.md
@@ -12,11 +12,10 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
| Z-Image | [Safetensors](https://huggingface.co/Comfy-Org/z_image/tree/main/split_files/diffusion_models)<br>[GGUF](https://huggingface.co/unsloth/Z-Image-GGUF/tree/main) | `--diffusion-model` | Choose matching variant |
| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main) | `--vae` | Use matching VAE |
| Qwen3 4b | [Safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders)<br>[GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main) | `--llm` | Use matching text encoder |
+| ControlNet | Z-Image control checkpoint with `control_all_x_embedder.2-1.weight` | `--control-net` or DGL `controlnet::load` | Optional. Match the DiT architecture |
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -33,7 +32,51 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Turbo: text to image](../dgl/z_image/z-image-turbo-txt2img.dgl) | Distilled generation. |
+| [Base: text to image](../dgl/z_image/z-image-base-txt2img.dgl) | Base checkpoint. |
+| [Turbo with ControlNet](../dgl/z_image/z-image-turbo-controlnet.dgl) | Load and apply control weights separately. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\z_image\z-image-turbo-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/z_image/z-image-turbo-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+#### ControlNet
+
+The [Z-Image ControlNet script](../dgl/z_image/z-image-turbo-controlnet.dgl)
+loads the diffusion and control weights as separate models. Set the four
+checkpoint paths and the prepared control image path in the script. The script
+encodes the image with the FLUX VAE in posterior mode, converts it to diffusion
+latents and pads the channel axis to 33 before calling `controlnet::apply`.
+Each control model shares the DiT's compute backend. Repeated
+`controlnet::apply` calls combine the controls before sampling. In the current
+Z-Image model, control conditions and reference latents require separate
+predictions.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Z-Image-Turbo
diff --git a/docs/models/z_image_l2p.md b/docs/models/z_image_l2p.md
index c572af8..c67ccab 100644
--- a/docs/models/z_image_l2p.md
+++ b/docs/models/z_image_l2p.md
@@ -12,8 +12,6 @@ directly in pixel space using Z-Image-Turbo and Qwen3 4B. No VAE is required.
## Usage
-CLI examples use the [shared path and shell conventions](README.md#example-conventions).
-
### Web UI
Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the
@@ -29,7 +27,37 @@ Slot IDs correspond to CLI component options: `diffusion_model` uses
`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`.
Select optional components when required by your checkpoint or task.
-### CLI
+### DGL
+
+| Example | Description |
+| --- | --- |
+| [Text to image](../dgl/z_image_l2p/z-image-l2p-txt2img.dgl) | Z-Image L2P generation. |
+
+#### Run a script
+
+Edit paths, prompts and settings in the script's `var` block, then run from
+the build directory. Paths inside the script resolve from the working directory.
+
+**Windows (PowerShell)**
+
+```powershell
+.\bin\Release\sd-cli.exe -M dgl --script ..\docs\dgl\z_image_l2p\z-image-l2p-txt2img.dgl
+```
+
+**Linux/macOS**
+
+```sh
+./bin/sd-cli -M dgl --script ../docs/dgl/z_image_l2p/z-image-l2p-txt2img.dgl
+```
+
+Choose another script from the table to run a different task. Add `--check`
+to validate syntax and types without loading weights. See the
+[shared run instructions](../dgl/README.md#run-an-example) for build paths
+and the server's file-access rules.
+
+### Legacy CLI
+
+CLI examples use the [shared path and shell conventions](README.md#example-conventions).
#### Text-to-image
diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt
index 36b9243..b93b3e7 100644
--- a/examples/cli/CMakeLists.txt
+++ b/examples/cli/CMakeLists.txt
@@ -2,12 +2,12 @@ set(TARGET sd-cli)
add_executable(${TARGET}
../common/common.cpp
- ../common/generation_metadata.cpp
../common/log.cpp
../common/media_io.cpp
../common/media_metadata.cpp
- image_metadata.cpp
main.cpp
+ dgl.cpp
+ media_input.cpp
)
if(APPLE)
sd_set_macos_rpaths(${TARGET})
@@ -26,3 +26,13 @@ if(SD_WEBM)
target_link_libraries(${TARGET} PRIVATE webm)
endif()
target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)
+
+set(SD_DGL_BUILTIN_CATALOG "${PROJECT_SOURCE_DIR}/docs/dgl-builtins.md")
+add_custom_target(dgl-update-builtins
+ COMMAND $<TARGET_FILE:${TARGET}> -M dgl --write-builtins "${SD_DGL_BUILTIN_CATALOG}"
+ DEPENDS ${TARGET}
+ VERBATIM)
+add_custom_target(dgl-check-builtins
+ COMMAND $<TARGET_FILE:${TARGET}> -M dgl --check-builtins "${SD_DGL_BUILTIN_CATALOG}"
+ DEPENDS ${TARGET}
+ VERBATIM)
diff --git a/examples/cli/README.md b/examples/cli/README.md
index 0316f7b..a43e22c 100644
--- a/examples/cli/README.md
+++ b/examples/cli/README.md
@@ -17,17 +17,40 @@ From the build directory, show the full list of command-line arguments:
See [sampling methods and schedulers](../../docs/guides/sampling.md) for supported
names, defaults, and custom sigma schedules.
-Omitted inference options use the [library defaults](../../docs/guides/c_api.md#keys).
-In particular, an omitted seed requests a random seed; use `--seed 42` for a
-repeatable run. Video generation defaults to 6 requested frames before model
-alignment. SLG layers remain unset unless `--skip-layers` is supplied, and the
-conditioning cache uses the library default. Zero and false are explicit inputs;
-invalid values are reported rather than treated as omitted.
+The CLI defaults to seed 42 for repeatable runs. Pass `--seed -1` to request a
+random seed. Other omitted inference options use the
+[library defaults](../../docs/guides/c_api.md#keys).
+Video generation defaults to 6 requested frames before model
+alignment. The conditioning cache uses the library default. Zero and false are
+explicit inputs; invalid values are reported rather than treated as omitted.
+
+IP-Adapter, PhotoMaker, PuLID, AnimateDiff, SLG, NitroFusion, ADetailer and video
+upscaling use explicit DGL workflows. Their former native CLI options and the
+`adetailer` / `vid_upscale` modes have been removed. See the
+[model and component guides](../../docs/models/README.md) for scripts.
+
+`--ref-video` accepts a frame directory or a video file. Directories retain
+filename order and 24 fps; files are decoded through DGL `video::load` at
+24 fps. Supported embedded audio is retained unless a matching
+`--ref-video-audio` WAV overrides it. `--control-video` accepts the same inputs;
+files are sampled at the generation `--fps` (default 16), without audio, and
+`--video-frames` limits decoding after resampling. Existing VACE frame padding
+is unchanged. File formats are AVI, static/animated WebP, and this project's
+independent-frame VP8 WebM. VP9 and Opus are unsupported; see
+[DGL media I/O](../../docs/dgl.md) for codec and build requirements.
Logging defaults to `info`. Use `--log-level <level>` to select `debug`, `verbose`,
`info`, `warn`, or `error` (from most to least detailed). Each level includes
messages at that level and all less detailed levels. `-v` and `--verbose` are
equivalent to `--log-level verbose`. If repeated, the last logging option wins.
+Native image/video generation and upscaling print the translated DGL program
+before execution at debug level (`--log-level debug`).
+
+Run a [DGL program](../../docs/dgl.md) with `sd-cli -M dgl --script program.dgl`.
+DGL mode accepts the same logging options and sends logs to stderr.
+Weight loading displays a progress bar with transfer speed; sampling displays
+its step progress. Context progress callbacks remain active during loading.
+`--run-once on|off` controls model release after last use and defaults to `on`.
`--flash-attn on|off|auto` defaults to `auto`, selecting flash attention for each
compute backend during context creation. See [Backend selection](../../docs/guides/backend.md#flash-attention---flash-attn-onoffauto)
diff --git a/examples/cli/dgl.cpp b/examples/cli/dgl.cpp
new file mode 100644
index 0000000..21db339
--- /dev/null
+++ b/examples/cli/dgl.cpp
@@ -0,0 +1,145 @@
+#include "dgl.h"
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <memory>
+#include <stdexcept>
+
+#include "common/common.h"
+#include "stable-diffusion.h"
+
+ArgOptions DGLModeParams::get_options() {
+ ArgOptions options;
+ options.string_options = {
+ {"", "--script", "path to the .dgl program", 0, &script_path},
+ {"", "--write-builtins", "write the DGL built-in catalog to a file", 0, &write_builtins_path},
+ {"", "--check-builtins", "verify a file against the DGL built-in catalog", 0, &check_builtins_path},
+ {"", "--max-ram", "limit DGL context weight residency in GiB", 0, &max_ram},
+ };
+ options.bool_options = {
+ {"", "--check", "check the DGL script without loading models", true, &check_only},
+ {"", "--list-builtins", "print the DGL built-in catalog", true, &list_builtins},
+ };
+ options.manual_options = {
+ {"", "--run-once", "on|off (default: on). Release DGL models after their last use when on",
+ [this](int argc, const char** argv, int index) {
+ const std::string_view value = index + 1 < argc ? argv[index + 1] : "";
+ if (value != "on" && value != "off") {
+ LOG_ERROR("--run-once requires on or off");
+ return -1;
+ }
+ run_once = value == "on";
+ return 1;
+ }},
+ {"", "--tensor", "save a named result tensor as .npy (NAME FILE, repeatable)",
+ [this](int argc, const char** argv, int index) {
+ if (index + 2 >= argc)
+ return -1;
+ tensor_paths.emplace_back(argv_to_utf8(index + 1, argv), argv_to_utf8(index + 2, argv));
+ return 2;
+ }},
+ };
+ return options;
+}
+
+bool DGLModeParams::has_options() const {
+ return !script_path.empty() || !write_builtins_path.empty() || !check_builtins_path.empty() ||
+ !tensor_paths.empty() || !max_ram.empty() || check_only || run_once.has_value() || list_builtins;
+}
+
+bool DGLModeParams::validate() const {
+ const int actions = int(!script_path.empty()) + int(!write_builtins_path.empty()) + int(!check_builtins_path.empty()) + int(list_builtins);
+ if (actions != 1) {
+ LOG_ERROR("DGL mode requires exactly one of --script, --list-builtins, --write-builtins and --check-builtins");
+ return false;
+ }
+ if (script_path.empty() && (check_only || run_once.has_value() || !tensor_paths.empty() || !max_ram.empty())) {
+ LOG_ERROR("--check, --run-once, --tensor and --max-ram require --script");
+ return false;
+ }
+ return true;
+}
+
+static void check_dgl(bool success) {
+ if (!success)
+ throw std::runtime_error(sd_dgl_error());
+}
+
+int DGLModeParams::run(const SDContextParams& options, bool embed_metadata) const {
+ try {
+ if (script_path.empty()) {
+ const auto* catalog = sd_dgl_builtin_catalog();
+ check_dgl(catalog != nullptr);
+ if (list_builtins) {
+ std::cout << catalog;
+ } else if (!write_builtins_path.empty()) {
+ std::ofstream file(std::filesystem::u8path(write_builtins_path), std::ios::binary);
+ file << catalog;
+ file.close();
+ if (!file)
+ throw std::runtime_error("cannot write the DGL built-in catalog");
+ } else {
+ std::ifstream file(std::filesystem::u8path(check_builtins_path));
+ const std::string existing{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
+ if (!file || existing != catalog)
+ throw std::runtime_error("DGL built-in catalog is out of date. Run the dgl-update-builtins CMake target.");
+ std::cout << "DGL built-in catalog is current.\n";
+ }
+ return 0;
+ }
+
+ const auto path = std::filesystem::u8path(script_path);
+ if (path.extension() != ".dgl")
+ throw std::runtime_error("expected a .dgl file");
+ std::ifstream file(path, std::ios::binary);
+ if (!file)
+ throw std::runtime_error("cannot open program file");
+ const std::string source{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
+ if (check_only) {
+ check_dgl(sd_dgl_check(source.c_str(), false) == SD_STATUS_OK);
+ std::cout << "DGL syntax and types are valid. Models were not loaded.\n";
+ return 0;
+ }
+ if (!options.register_rpc_devices())
+ return 1;
+ auto* raw_context = run_once.value_or(true) ? sd_dgl_ctx_create_once() : sd_dgl_ctx_create();
+ check_dgl(raw_context != nullptr);
+ std::unique_ptr<sd_dgl_ctx_t, decltype(&sd_dgl_ctx_free)> context(raw_context, sd_dgl_ctx_free);
+ check_dgl(options.configure_dgl(context.get()));
+ check_dgl(sd_dgl_ctx_set(context.get(), "embed_metadata", embed_metadata ? "on" : "off"));
+ if (!max_ram.empty())
+ check_dgl(sd_dgl_ctx_set(context.get(), "max_ram", max_ram.c_str()));
+ sd_dgl_result_t* raw_result = nullptr;
+ check_dgl(sd_dgl_run(context.get(), source.c_str(), nullptr, 0, &raw_result) == SD_STATUS_OK);
+ std::unique_ptr<sd_dgl_result_t, decltype(&sd_dgl_result_free)> result(raw_result, sd_dgl_result_free);
+ for (const auto& [name, output_path] : tensor_paths) {
+ auto tensor = sd_dgl_result_tensor(result.get(), name.c_str());
+ if (!tensor)
+ throw std::runtime_error("unknown output tensor '" + name + "'");
+ check_dgl(sd_tensor_save(tensor, output_path.c_str()));
+ }
+ const std::string type = sd_dgl_result_type(result.get());
+ if (type == "string") {
+ const auto* tensor = sd_dgl_result_tensor(result.get(), "result");
+ const auto* data = static_cast<const char*>(sd_tensor_data(tensor));
+ const auto size = sd_tensor_byte_size(tensor);
+ if (size) {
+ std::cout.write(data, size);
+ if (data[size - 1] != '\n')
+ std::cout << '\n';
+ }
+ return 0;
+ }
+ const size_t count = sd_dgl_result_count(result.get());
+ LOG_VERBOSE("Result: %s, tensors: %zu", type.c_str(), count);
+ for (size_t i = 0; i < count; ++i) {
+ LOG_VERBOSE(" %s", sd_dgl_result_name(result.get(), i));
+ }
+ return 0;
+ } catch (const std::exception& error) {
+ LOG_ERROR("%s", error.what());
+ return 1;
+ }
+}
diff --git a/examples/cli/dgl.h b/examples/cli/dgl.h
new file mode 100644
index 0000000..9bee4d5
--- /dev/null
+++ b/examples/cli/dgl.h
@@ -0,0 +1,28 @@
+#ifndef __EXAMPLES_CLI_DGL_H__
+#define __EXAMPLES_CLI_DGL_H__
+
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+struct ArgOptions;
+struct SDContextParams;
+
+struct DGLModeParams {
+ std::string script_path;
+ std::string write_builtins_path;
+ std::string check_builtins_path;
+ std::string max_ram;
+ std::vector<std::pair<std::string, std::string>> tensor_paths;
+ bool check_only = false;
+ std::optional<bool> run_once;
+ bool list_builtins = false;
+
+ ArgOptions get_options();
+ bool has_options() const;
+ bool validate() const;
+ int run(const SDContextParams& options, bool embed_metadata) const;
+};
+
+#endif // __EXAMPLES_CLI_DGL_H__
diff --git a/examples/cli/image_metadata.h b/examples/cli/image_metadata.h
deleted file mode 100644
index 9538167..0000000
--- a/examples/cli/image_metadata.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#pragma once
-
-#include <iosfwd>
-#include <string>
-
-enum class MetadataOutputFormat {
- TEXT,
- JSON,
-};
-
-struct MetadataReadOptions {
- MetadataOutputFormat output_format = MetadataOutputFormat::TEXT;
- bool include_raw = false;
- bool brief = false;
- bool include_structural = false;
-};
-
-bool print_image_metadata(const std::string& image_path,
- const MetadataReadOptions& options,
- std::ostream& out,
- std::string& error);
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index a29fd70..98913f8 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -20,7 +20,8 @@
#include "common/common.h"
#include "common/media_io.h"
#include "common/resource_owners.hpp"
-#include "image_metadata.h"
+#include "dgl.h"
+#include "media_input.h"
namespace fs = std::filesystem;
@@ -34,16 +35,13 @@ const char* previews_str[] = {
std::regex format_specifier_regex("(?:[^%]|^)(?:%%)*(%\\d{0,3}d)");
struct SDCliParams {
- std::string input_frames;
- int video_upscale_factor = 0;
- std::optional<int> vsr_window_size;
- std::optional<int> vsr_temporal_chunk_size;
SDMode mode = IMG_GEN;
std::string output_path = "output.png";
int output_begin_idx = -1;
int compression_quality = 90;
std::string image_path;
std::string metadata_format = "text";
+ DGLModeParams dgl;
sd_log_level_t log_level = SD_LOG_INFO;
bool convert_name = false;
@@ -67,7 +65,6 @@ struct SDCliParams {
ArgOptions options;
options.string_options = {
- {"", "--input-frames", "directory of ordered RGB frames for vid_upscale (use -i for a single image)", 0, &input_frames},
{"-o",
"--output",
"path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs support .avi, .webm, and animated .webp",
@@ -96,9 +93,6 @@ struct SDCliParams {
};
options.int_options = {
- {"", "--upscale-factor", "required positive video upscale factor (RelayVSR: 2 or 4; SeedVR2: integer)", &video_upscale_factor},
- {"", "--vsr-window-size", "RelayVSR window including both endpoints (default: 16)", &vsr_window_size},
- {"", "--vsr-temporal-chunk-size", "SeedVR2 input frames per independent chunk: 1 or 4n+1 (default: 5)", &vsr_temporal_chunk_size},
{"",
"--preview-interval",
"preview interval: in each sampling pass, positive N updates every Nth denoiser step and -N previews only completed logical step N; 0 previews the final completed step of the first pass (base-resolution or high-noise). Default: 1",
@@ -188,7 +182,7 @@ struct SDCliParams {
options.manual_options = {
{"-M",
"--mode",
- "run mode, one of [img_gen, adetailer, vid_gen, vid_upscale, upscale, convert, metadata], default: img_gen",
+ "run mode, one of [" SD_ALL_MODES_STR "], default: img_gen",
on_mode_arg},
{"",
"--preview",
@@ -218,8 +212,11 @@ struct SDCliParams {
}
bool validate() {
- if (mode == VID_UPSCALE && video_upscale_factor <= 0) {
- LOG_ERROR("vid_upscale requires a positive --upscale-factor");
+ if (mode == DGL) {
+ return dgl.validate();
+ }
+ if (dgl.has_options()) {
+ LOG_ERROR("DGL options require --mode dgl");
return false;
}
if (mode != METADATA) {
@@ -284,25 +281,28 @@ void print_usage(int argc, const char* argv[], const std::vector<ArgOptions>& op
options_list[1].print();
std::cout << "\nGeneration Options:\n";
options_list[2].print();
+ std::cout << "\nDGL Options:\n";
+ options_list[3].print();
}
void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextParams& ctx_params, SDGenerationParams& gen_params) {
- std::vector<ArgOptions> options_vec = {cli_params.get_options(), ctx_params.get_options(), gen_params.get_options()};
+ std::vector<ArgOptions> options_vec = {cli_params.get_options(), ctx_params.get_options(), gen_params.get_options(), cli_params.dgl.get_options()};
if (!parse_options(argc, argv, options_vec)) {
print_usage(argc, argv, options_vec);
exit(cli_params.normal_exit ? 0 : 1);
}
- log_level = cli_params.log_level;
- log_color = cli_params.color;
+ log_level = cli_params.log_level;
+ log_color = cli_params.color;
+ log_stream = cli_params.mode == DGL ? stderr : nullptr;
if (ctx_params.list_devices) {
exit(ctx_params.print_devices() ? 0 : 1);
}
bool valid = cli_params.resolve_and_validate();
- if (valid && cli_params.mode != METADATA) {
+ if (valid && cli_params.mode != METADATA && cli_params.mode != DGL) {
valid = ctx_params.resolve_and_validate(cli_params.mode) &&
gen_params.resolve_and_validate(cli_params.mode,
ctx_params.lora_model_dir,
@@ -317,57 +317,7 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP
void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
SDCliParams* cli_params = (SDCliParams*)data;
- log_print(level, log, cli_params->log_level, cli_params->color);
-}
-
-bool load_images_from_dir(const std::string dir,
- std::vector<SDImageOwner>& images,
- int max_image_num = 0) {
- const fs::path dir_path = fs::u8path(dir);
- if (!fs::exists(dir_path) || !fs::is_directory(dir_path)) {
- LOG_ERROR("'%s' is not a valid directory\n", dir.c_str());
- return false;
- }
-
- std::vector<fs::directory_entry> entries;
- for (const auto& entry : fs::directory_iterator(dir_path)) {
- if (entry.is_regular_file()) {
- entries.push_back(entry);
- }
- }
-
- std::sort(entries.begin(), entries.end(),
- [](const fs::directory_entry& a, const fs::directory_entry& b) {
- return a.path().filename().u8string() < b.path().filename().u8string();
- });
-
- for (const auto& entry : entries) {
- std::string path = entry.path().u8string();
- std::string ext = entry.path().extension().u8string();
- std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
-
- if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
- LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str());
- int width = 0;
- int height = 0;
- int loaded_channel = 0;
- uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, 0, 0);
- if (image_buffer == nullptr) {
- LOG_ERROR("load image from '%s' failed", path.c_str());
- return false;
- }
-
- images.emplace_back(SDImage{(uint32_t)width,
- (uint32_t)height,
- (uint32_t)loaded_channel,
- image_buffer});
-
- if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
- break;
- }
- }
- }
- return true;
+ log_print(level, log, cli_params->log_level, cli_params->color, log_stream);
}
std::string format_frame_idx(std::string pattern, int frame_idx) {
@@ -535,9 +485,7 @@ bool save_results(const SDCliParams& cli_params,
return sucessful_reults != 0;
}
- const bool video_upscale_output = cli_params.mode == VID_UPSCALE &&
- (gen_params.init_image_path.empty() || ext_lower == ".avi" || ext_lower == ".webm");
- if ((cli_params.mode == VID_GEN && num_results > 1) || video_upscale_output) {
+ if (cli_params.mode == VID_GEN && num_results > 1) {
if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
ext = ".avi";
std::string params = gen_params.embed_image_metadata
@@ -583,56 +531,6 @@ bool save_results(const SDCliParams& cli_params,
return sucessful_reults != 0;
}
-static bool apply_adetailer(sd_ctx_t* sd_ctx,
- const SDContextParams& ctx_params,
- SDGenerationParams& gen_params,
- SDMode mode,
- SDImageVec& results,
- int num_results) {
- if (gen_params.ad_model_path.empty()) {
- return true;
- }
-
- ADetailerCtxPtr ad_ctx(create_sd_adetailer(gen_params.ad_model_path.c_str(),
- ctx_params.n_threads,
- ctx_params.effective_backend.c_str(),
- ctx_params.effective_params_backend.c_str()));
- if (ad_ctx == nullptr) {
- LOG_ERROR("sd_adetailer_create failed");
- return false;
- }
-
- for (int i = 0; i < num_results; ++i) {
- if (results[i].data == nullptr) {
- continue;
- }
- SDOptBuilder ad_options(gen_params.to_image_opt(true).release());
- ad_options.set_int("seed", generation_seed(results.generation_info(i), gen_params.seed.value_or(-1)));
- if (mode == IMG_GEN) {
- ad_options.unset("width");
- ad_options.unset("height");
- ad_options.unset("strength");
- }
- sd_result_t* detailed = nullptr;
- if (!adetail_sd_image(ad_ctx.get(),
- sd_ctx,
- results[i],
- ad_options.finish(),
- &detailed) ||
- detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) {
- sd_result_free(detailed);
- LOG_ERROR("ADetailer failed for image %d", i + 1);
- return false;
- }
- results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), ctx_params.lora_model_dir);
- sd_image_free(&results[i]);
- results[i] = take_result_image(detailed, 0);
-
- sd_result_free(detailed);
- }
- return true;
-}
-
int main(int argc, const char* argv[]) {
if (argc > 1 && std::string(argv[1]) == "--version") {
std::cout << version_string() << "\n";
@@ -642,92 +540,36 @@ int main(int argc, const char* argv[]) {
SDCliParams cli_params;
SDContextParams ctx_params;
SDGenerationParams gen_params;
+ gen_params.seed = 42;
sd_set_log_callback(sd_log_cb, (void*)&cli_params);
parse_args(argc, argv, cli_params, ctx_params, gen_params);
- if (cli_params.mode == METADATA) {
- MetadataReadOptions options;
- options.output_format = cli_params.metadata_format == "json"
- ? MetadataOutputFormat::JSON
- : MetadataOutputFormat::TEXT;
- options.include_raw = cli_params.metadata_raw;
- options.brief = cli_params.metadata_brief;
- options.include_structural = cli_params.metadata_all;
-
- std::string error;
- if (!print_image_metadata(cli_params.image_path, options, std::cout, error)) {
- LOG_ERROR("%s", error.c_str());
- return 1;
- }
- return 0;
+ if (cli_params.mode == DGL) {
+ return cli_params.dgl.run(ctx_params, gen_params.embed_image_metadata);
}
- if (cli_params.mode == VID_UPSCALE) {
- if (cli_params.input_frames.empty() == gen_params.init_image_path.empty()) {
- LOG_ERROR("vid_upscale requires exactly one input: -i/--init-img or --input-frames");
- return 1;
- }
- ctx_params.prepare_backend_assignments();
- std::vector<SDImageOwner> owned_frames;
- if (!gen_params.init_image_path.empty()) {
- owned_frames.emplace_back();
- if (!load_sd_image_from_file(owned_frames.back().put(), gen_params.init_image_path.c_str(), 0, 0, 3)) {
- LOG_ERROR("load image from '%s' failed", gen_params.init_image_path.c_str());
- return 1;
- }
- } else if (!load_images_from_dir(cli_params.input_frames, owned_frames) || owned_frames.empty()) {
- LOG_ERROR("vid_upscale input contains no readable frames");
- return 1;
- }
- if (owned_frames.size() > size_t(INT_MAX))
+ if (cli_params.mode == METADATA) {
+ uint32_t flags = 0;
+ if (cli_params.metadata_format == "json")
+ flags |= SD_METADATA_READ_JSON;
+ if (cli_params.metadata_raw)
+ flags |= SD_METADATA_READ_RAW;
+ if (cli_params.metadata_brief)
+ flags |= SD_METADATA_READ_BRIEF;
+ if (cli_params.metadata_all)
+ flags |= SD_METADATA_READ_STRUCTURAL;
+ const auto* output = sd_read_metadata(cli_params.image_path.c_str(), flags);
+ if (!output) {
+ LOG_ERROR("%s", sd_get_last_error());
return 1;
- std::vector<SDImage> frames;
- for (const auto& frame : owned_frames)
- frames.push_back(frame.get());
- SDVideoUpscalerParams params;
-
- params.diffusion_model_path = ctx_params.diffusion_model_path.c_str();
- params.vae_path = ctx_params.vae_path.c_str();
- params.upscale_factor = cli_params.video_upscale_factor;
- params.n_threads = ctx_params.n_threads;
- params.backend = ctx_params.effective_backend.c_str();
- params.params_backend = ctx_params.effective_params_backend.c_str();
- params.flash_attn_mode = ctx_params.flash_attn_mode ? ctx_params.flash_attn_mode->c_str() : nullptr;
- std::string conv_direct_mode = ctx_params.conv_direct_mode.value_or("");
- if (conv_direct_mode.empty()) {
- conv_direct_mode = "auto";
- if (ctx_params.diffusion_conv_direct.value_or(false))
- conv_direct_mode += ",diffusion=on";
- if (ctx_params.vae_conv_direct.value_or(false))
- conv_direct_mode += ",vae=on";
}
- params.conv_direct_mode = conv_direct_mode.c_str();
- std::unique_ptr<sd_vid_upscaler_t, decltype(&sd_vid_upscaler_free)> upscaler(
- create_sd_vid_upscaler(params),
- sd_vid_upscaler_free);
- if (!upscaler)
- return 1;
- sd_result_t* output = nullptr;
- SDVideoUpscaleParams upscale_params;
-
- upscale_params.frames = frames.data();
- upscale_params.frame_count = int(frames.size());
- upscale_params.fps = gen_params.fps;
- upscale_params.window_size = cli_params.vsr_window_size;
- upscale_params.temporal_chunk_size = cli_params.vsr_temporal_chunk_size;
- upscale_params.seed = gen_params.seed;
- if (!upscale_sd_video(upscaler.get(), upscale_params, &output))
- return 1;
- std::string video_info = sd_result_get_generation_info(output, 0);
- gen_params.fps = sd_result_get_fps(output);
- SDImageVec results;
- results.adopt_video(output);
- return save_results(cli_params, ctx_params, gen_params, results, int(results.size()), nullptr, video_info) ? 0 : 1;
+ print_utf8(stdout, output);
+ return 0;
}
- if (!gen_params.ad_model_path.empty() && cli_params.mode != IMG_GEN && cli_params.mode != ADETAILER) {
- LOG_ERROR("--ad-model is only supported in image generation and adetailer modes");
+ if (!ctx_params.esrgan_path.empty() && cli_params.mode != UPSCALE) {
+ LOG_ERROR("generation followed by automatic upscaling requires DGL; use -M dgl --script FILE");
return 1;
}
@@ -841,7 +683,7 @@ int main(int argc, const char* argv[]) {
};
if (gen_params.init_image_path.size() > 0) {
- const bool native_init = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
+ const bool native_init = cli_params.mode == IMG_GEN;
if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image, native_init ? 0 : 3)) {
return 1;
}
@@ -866,24 +708,11 @@ int main(int argc, const char* argv[]) {
if (!gen_params.ref_video_paths.empty()) {
gen_params.ref_videos.clear();
- gen_params.ref_videos.reserve(gen_params.ref_video_paths.size());
- for (const auto& path : gen_params.ref_video_paths) {
- std::vector<SDImageOwner> frames;
- if (!load_images_from_dir(path, frames) || frames.empty()) {
- LOG_ERROR("load reference video frames from '%s' failed", path.c_str());
- return 1;
- }
- gen_params.ref_videos.push_back(std::move(frames));
- }
-
gen_params.ref_video_audios.clear();
- gen_params.ref_video_audios.resize(gen_params.ref_videos.size());
- for (size_t i = 0; i < gen_params.ref_video_audio_paths.size(); ++i) {
- if (!load_audio(gen_params.ref_video_audio_paths[i], gen_params.ref_video_audios[i])) {
- return 1;
- }
- }
}
+ std::vector<VideoInput> reference_video_inputs;
+ if (!load_reference_video_inputs(gen_params.ref_video_paths, gen_params.ref_video_audio_paths, reference_video_inputs))
+ return 1;
if (!gen_params.ref_audio_paths.empty()) {
gen_params.ref_audios.clear();
@@ -916,29 +745,16 @@ int main(int argc, const char* argv[]) {
}
}
- if (gen_params.ip_adapter_image_path.size() > 0) {
- if (!load_sd_image_from_file(gen_params.ip_adapter_image.put(),
- gen_params.ip_adapter_image_path.c_str(),
- 0,
- 0)) {
- LOG_ERROR("load image from '%s' failed", gen_params.ip_adapter_image_path.c_str());
- return 1;
- }
- }
-
+ VideoInput control_video_input;
if (!gen_params.control_video_path.empty()) {
gen_params.control_frames.clear();
- if (!load_images_from_dir(gen_params.control_video_path,
- gen_params.control_frames,
- gen_params.video_frames.value_or(0))) {
- return 1;
- }
- }
-
- if (!gen_params.pm_id_images_dir.empty()) {
- gen_params.pm_id_images.clear();
- if (!load_images_from_dir(gen_params.pm_id_images_dir,
- gen_params.pm_id_images)) {
+ if (fs::is_directory(fs::u8path(gen_params.control_video_path))) {
+ if (!load_images_from_dir(gen_params.control_video_path,
+ gen_params.control_frames,
+ gen_params.video_frames.value_or(0)))
+ return 1;
+ } else if (!load_video_input(gen_params.control_video_path, gen_params.fps.value_or(16),
+ gen_params.video_frames.value_or(0), false, control_video_input)) {
return 1;
}
}
@@ -975,40 +791,28 @@ int main(int argc, const char* argv[]) {
},
&prompt_image);
- const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
if (cli_params.mode == IMG_GEN) {
sd_result_t* generated = nullptr;
generate_sd_image(sd_ctx.get(), gen_params.to_image_opt(), &generated);
- results.adopt(generated, ctx_params.lora_model_dir);
+ results.adopt(generated);
num_results = results.count();
- } else if (cli_params.mode == ADETAILER) {
- num_results = 1;
- results.push_back(gen_params.init_image.release());
} else if (cli_params.mode == VID_GEN) {
SDOptPtr vid_gen_params = gen_params.to_video_opt();
- sd_result_t* generated = nullptr;
+ if (!vid_gen_params || !set_video_inputs(vid_gen_params.get(), reference_video_inputs, control_video_input.frames.get()))
+ return 1;
+ sd_result_t* generated = nullptr;
if (generate_sd_video(sd_ctx.get(), vid_gen_params, &generated)) {
gen_params.fps = cli_params.preview_fps = sd_result_get_fps(generated);
generated_audio = retain_result_audio(generated);
num_results = sd_result_get_image_count(generated);
- results.adopt_video(generated, ctx_params.lora_model_dir);
+ results.adopt_video(generated);
video_info = results.generation_info(0);
}
}
if (!results) {
- LOG_ERROR("generate failed");
- return 1;
- }
-
- if (use_img_gen_params &&
- !apply_adetailer(sd_ctx.get(),
- ctx_params,
- gen_params,
- cli_params.mode,
- results,
- num_results)) {
+ LOG_ERROR("generate failed: %s", sd_get_last_error());
return 1;
}
}
@@ -1029,21 +833,17 @@ int main(int argc, const char* argv[]) {
continue;
}
SDImageOwner current_image(results[i]);
- results[i] = {0, 0, 0, nullptr};
- for (int u = 0; u < gen_params.upscale_repeats; ++u) {
- sd_result_t* upscaled = nullptr;
- bool upscale_ok = upscale_sd_image(upscaler_ctx.get(), current_image.get(), &upscaled);
- if (!upscale_ok || !upscaled || sd_result_get_image_count(upscaled) <= 0 || sd_tensor_data(sd_result_get_image(upscaled, 0)) == nullptr) {
- sd_result_free(upscaled);
- LOG_ERROR("upscale failed");
- break;
- }
- results.append_generation_info(i, sd_result_get_generation_info(upscaled, 0));
- SDImage upscaled_image = take_result_image(upscaled, 0);
-
+ results[i] = {0, 0, 0, nullptr};
+ sd_result_t* upscaled = nullptr;
+ if (!upscale_sd_image(upscaler_ctx.get(), current_image.get(), &upscaled, gen_params.upscale_repeats) || !upscaled) {
+ LOG_ERROR("upscale failed: %s", sd_get_last_error());
sd_result_free(upscaled);
- current_image.reset(upscaled_image);
+ return 1;
}
+ const std::string source = sd_result_get_generation_info(upscaled, 0);
+ current_image.reset(take_result_image(upscaled, 0));
+ results.set_generation_info(i, source);
+ sd_result_free(upscaled);
results[i] = current_image.release(); // Set the final upscaled image as the result
}
}
diff --git a/examples/cli/media_input.cpp b/examples/cli/media_input.cpp
new file mode 100644
index 0000000..ce9d073
--- /dev/null
+++ b/examples/cli/media_input.cpp
@@ -0,0 +1,149 @@
+#include "media_input.h"
+
+#include <algorithm>
+#include <cctype>
+#include <filesystem>
+#include <sstream>
+
+#include "common/media_io.h"
+#include "json.hpp"
+
+bool load_images_from_dir(const std::string& dir,
+ std::vector<SDImageOwner>& images,
+ int max_image_num) {
+ const std::filesystem::path dir_path = std::filesystem::u8path(dir);
+ if (!std::filesystem::exists(dir_path) || !std::filesystem::is_directory(dir_path)) {
+ LOG_ERROR("'%s' is not a valid directory\n", dir.c_str());
+ return false;
+ }
+
+ std::vector<std::filesystem::directory_entry> entries;
+ for (const auto& entry : std::filesystem::directory_iterator(dir_path)) {
+ if (entry.is_regular_file()) {
+ entries.push_back(entry);
+ }
+ }
+
+ std::sort(entries.begin(), entries.end(),
+ [](const std::filesystem::directory_entry& a, const std::filesystem::directory_entry& b) {
+ return a.path().filename().u8string() < b.path().filename().u8string();
+ });
+
+ for (const auto& entry : entries) {
+ std::string path = entry.path().u8string();
+ std::string ext = entry.path().extension().u8string();
+ std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+
+ if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
+ LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str());
+ int width = 0;
+ int height = 0;
+ int loaded_channel = 0;
+ uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, 0, 0);
+ if (image_buffer == nullptr) {
+ LOG_ERROR("load image from '%s' failed", path.c_str());
+ return false;
+ }
+
+ images.emplace_back(SDImage{(uint32_t)width,
+ (uint32_t)height,
+ (uint32_t)loaded_channel,
+ image_buffer});
+
+ if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
+ break;
+ }
+ }
+ }
+ return true;
+}
+
+bool load_video_input(const std::string& path, int fps, size_t max_frames, bool audio, VideoInput& result) {
+ std::ostringstream source;
+ source << "fn main() { return video::load(" << nlohmann::json(path).dump()
+ << ", fps_num = " << fps << ", max_frames = " << max_frames
+ << ", audio = " << (audio ? "true" : "false") << ") }";
+ std::unique_ptr<sd_dgl_ctx_t, decltype(&sd_dgl_ctx_free)> context(sd_dgl_ctx_create_once(), sd_dgl_ctx_free);
+ if (!context) {
+ LOG_ERROR("cannot prepare video input '%s': %s", path.c_str(), sd_dgl_error());
+ return false;
+ }
+ sd_dgl_result_t* raw_result = nullptr;
+ if (sd_dgl_run(context.get(), source.str().c_str(), nullptr, 0, &raw_result) != SD_STATUS_OK) {
+ LOG_ERROR("cannot read video input '%s': %s", path.c_str(), sd_dgl_error());
+ return false;
+ }
+ std::unique_ptr<sd_dgl_result_t, decltype(&sd_dgl_result_free)> loaded(raw_result, sd_dgl_result_free);
+ const auto* root = sd_dgl_result_root(loaded.get());
+ const auto* sound = sd_dgl_value_field(root, "audio");
+ int64_t sample_rate = 0;
+ if (sd_dgl_value_get_int(sd_dgl_value_field(sound, "sample_rate"), &sample_rate) != SD_STATUS_OK) {
+ LOG_ERROR("video input has no audio sample rate: %s", sd_dgl_error());
+ return false;
+ }
+ VideoInput input;
+ input.frames.reset(sd_tensor_retain(sd_dgl_result_tensor(loaded.get(), "video.frames")));
+ input.samples.reset(sd_tensor_retain(sd_dgl_result_tensor(loaded.get(), "audio.samples")));
+ input.sample_rate = uint32_t(sample_rate);
+ input.fps = fps;
+ if (!input.frames || !input.samples) {
+ LOG_ERROR("video input is missing frame or audio tensors: %s", sd_dgl_error());
+ return false;
+ }
+ result = std::move(input);
+ return true;
+}
+
+bool load_reference_video_inputs(const std::vector<std::string>& paths, const std::vector<std::string>& audio_paths, std::vector<VideoInput>& result) {
+ std::vector<VideoInput> inputs;
+ inputs.reserve(paths.size());
+ for (size_t i = 0; i < paths.size(); ++i) {
+ VideoInput input;
+ const bool external_audio = i < audio_paths.size();
+ if (std::filesystem::is_directory(std::filesystem::u8path(paths[i]))) {
+ std::vector<SDImageOwner> frames;
+ if (!load_images_from_dir(paths[i], frames) || frames.empty()) {
+ LOG_ERROR("load reference video frames from '%s' failed", paths[i].c_str());
+ return false;
+ }
+ std::vector<SDImage> views;
+ views.reserve(frames.size());
+ for (const auto& frame : frames)
+ views.push_back(frame.get());
+ input.frames = frames_tensor(views.data(), views.size());
+ if (!input.frames)
+ return false;
+ } else if (!load_video_input(paths[i], 24, 0, !external_audio, input)) {
+ return false;
+ }
+ if (external_audio) {
+ std::vector<float> samples;
+ uint32_t rate = 0, channels = 0;
+ if (!load_wav_from_file(audio_paths[i], samples, rate, channels)) {
+ LOG_ERROR("load WAV audio from '%s' failed", audio_paths[i].c_str());
+ return false;
+ }
+ const int64_t shape[] = {channels, int64_t(samples.size() / channels)};
+ input.samples.reset(sd_tensor_create(SD_TENSOR_F32, 2, shape, samples.data(), samples.size() * sizeof(float)));
+ input.sample_rate = rate;
+ if (!input.samples)
+ return false;
+ }
+ inputs.push_back(std::move(input));
+ }
+ result = std::move(inputs);
+ return true;
+}
+
+bool set_video_inputs(sd_opt_t* options, const std::vector<VideoInput>& references, const sd_tensor_t* control) {
+ std::vector<sd_video_t> views;
+ views.reserve(references.size());
+ for (const auto& video : references)
+ views.push_back(video.view());
+ if ((!views.empty() && sd_opt_set_videos(options, "ref_videos", views.data(), views.size()) != SD_STATUS_OK) ||
+ (control && sd_opt_set_tensor(options, "control_frames", control) != SD_STATUS_OK)) {
+ LOG_ERROR("cannot set video inputs: %s", sd_get_last_error());
+ return false;
+ }
+ return true;
+}
diff --git a/examples/cli/media_input.h b/examples/cli/media_input.h
new file mode 100644
index 0000000..aff5a9d
--- /dev/null
+++ b/examples/cli/media_input.h
@@ -0,0 +1,20 @@
+#ifndef __SD_EXAMPLES_CLI_MEDIA_INPUT_H__
+#define __SD_EXAMPLES_CLI_MEDIA_INPUT_H__
+
+#include "common/resource_owners.hpp"
+
+struct VideoInput {
+ SDTensorPtr frames;
+ SDTensorPtr samples;
+ uint32_t sample_rate = 0;
+ int fps = 24;
+
+ sd_video_t view() const { return {frames.get(), fps, {sample_rate, sample_rate ? samples.get() : nullptr}}; }
+};
+
+bool load_images_from_dir(const std::string& path, std::vector<SDImageOwner>& images, int max_image_num = 0);
+bool load_video_input(const std::string& path, int fps, size_t max_frames, bool audio, VideoInput& result);
+bool load_reference_video_inputs(const std::vector<std::string>& paths, const std::vector<std::string>& audio_paths, std::vector<VideoInput>& result);
+bool set_video_inputs(sd_opt_t* options, const std::vector<VideoInput>& references, const sd_tensor_t* control);
+
+#endif // __SD_EXAMPLES_CLI_MEDIA_INPUT_H__
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 0c155c1..2e2aea0 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -30,12 +30,11 @@ namespace fs = std::filesystem;
const char* const modes_str[] = {
"img_gen",
- "adetailer",
"vid_gen",
"convert",
"upscale",
"metadata",
- "vid_upscale",
+ "dgl",
};
static void prepend_backend_assignment(std::string& spec, const char* assignment) {
@@ -484,16 +483,6 @@ ArgOptions SDContextParams::get_options() {
"path to control net model",
0,
&control_net_path},
- {"",
- "--ip-adapter",
- "path to IP-Adapter model (requires --clip_vision)",
- 0,
- &ip_adapter_path},
- {"",
- "--motion-module",
- "path to AnimateDiff motion module (SD 1.5); enables video generation on --video-frames > 1",
- 0,
- &motion_module_path},
{"",
"--embd-dir",
"embeddings directory",
@@ -521,16 +510,6 @@ ArgOptions SDContextParams::get_options() {
"qwen_image_2_1_prefix_cache_type (auto or a type name from --type)",
(int)',',
&model_args},
- {"",
- "--photo-maker",
- "path to PHOTOMAKER model",
- 0,
- &photo_maker_path},
- {"",
- "--pulid-weights",
- "path to PuLID Flux weights",
- 0,
- &pulid_weights_path},
{"",
"--upscale-model",
"path to esrgan model.",
@@ -779,7 +758,7 @@ ArgOptions SDContextParams::get_options() {
on_sampler_rng_arg},
{"",
"--prediction",
- "prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, sefi_flow, minit2i_flow, sensenova_u1_flow, flux3_flow, h3_av_flow, ming_image_flow]",
+ "prediction type override, one of [eps, v, edm_v, flow, flux_flow, sefi_flow, minit2i_flow, sensenova_u1_flow, flux3_flow, h3_av_flow, ming_image_flow]",
on_prediction_arg},
{"",
"--lora-apply-mode",
@@ -794,8 +773,12 @@ ArgOptions SDContextParams::get_options() {
return options;
}
+bool SDContextParams::register_rpc_devices() const {
+ return !rpc_servers || sd_add_rpc_servers(rpc_servers->c_str());
+}
+
bool SDContextParams::print_devices() const {
- if (!sd_add_rpc_servers(rpc_servers ? rpc_servers->c_str() : nullptr)) {
+ if (!register_rpc_devices()) {
return false;
}
size_t device_list_size = sd_list_devices(nullptr, 0);
@@ -979,6 +962,21 @@ std::string SDContextParams::to_string() const {
return oss.str();
}
+bool SDContextParams::configure_dgl(sd_dgl_ctx_t* ctx) const {
+ auto set = [ctx](const char* key, const auto& option) {
+ return !option || sd_dgl_ctx_set(ctx, key, std::to_string(*option).c_str());
+ };
+ return (!max_vram || sd_dgl_ctx_set(ctx, "max_vram", max_vram->c_str())) &&
+ set("auto_fit", auto_fit) &&
+ set("conditioning_cache_size", conditioning_cache_size) &&
+ set("batched_cfg", batched_cfg) &&
+ set("n_threads", n_threads) &&
+ set("enable_mmap", enable_mmap) &&
+ set("eager_load", eager_load) &&
+ set("disable_prefetch", disable_prefetch) &&
+ set("disable_segmented_compute", disable_segmented_compute);
+}
+
SDOptPtr SDContextParams::to_context_opt() {
prepare_backend_assignments();
embedding_vec.clear();
@@ -1008,10 +1006,14 @@ SDOptPtr SDContextParams::to_context_opt() {
opt.set_string("audio_encoder_path", audio_encoder_path);
opt.set_string("taesd_path", taesd_path);
opt.set_string("control_net_path", control_net_path);
- opt.set_string("ip_adapter_path", ip_adapter_path);
- opt.set_string("motion_module_path", motion_module_path);
- opt.set_string("photo_maker_path", photo_maker_path);
- opt.set_string("pulid_weights_path", pulid_weights_path);
+ if (!ip_adapter_path.empty())
+ opt.set_string("ip_adapter_path", ip_adapter_path);
+ if (!motion_module_path.empty())
+ opt.set_string("motion_module_path", motion_module_path);
+ if (!photo_maker_path.empty())
+ opt.set_string("photo_maker_path", photo_maker_path);
+ if (!pulid_weights_path.empty())
+ opt.set_string("pulid_weights_path", pulid_weights_path);
opt.set_string("tensor_type_rules", tensor_type_rules);
opt.set_int("n_threads", n_threads);
opt.set_int("conditioning_cache_size", conditioning_cache_size);
@@ -1065,29 +1067,9 @@ ArgOptions SDGenerationParams::get_options() {
"the negative prompt (default: \"\")",
0,
&negative_prompt},
- {"",
- "--ad-model",
- "path to a converted YOLOv8 detection model for ADetailer",
- 0,
- &ad_model_path},
- {"",
- "--ad-prompt",
- "ADetailer prompt; empty inherits the main prompt, supports [PROMPT], [SEP], and [SKIP]",
- 0,
- &ad_prompt},
- {"",
- "--ad-negative-prompt",
- "ADetailer negative prompt; empty inherits the main negative prompt, supports [PROMPT] and [SEP]",
- 0,
- &ad_negative_prompt},
- {"",
- "--extra-ad-args",
- "extra ADetailer args, key=value list. Supports input_size, confidence, nms, max_detections, mask_k_largest, mask_min_ratio, mask_max_ratio, dilate_erode, x_offset, y_offset, mask_mode, merge_masks, invert_mask, mask_blur, inpaint_padding, inpaint_width, inpaint_height, denoising_strength, steps, cfg_scale, sample_method, scheduler, sort_by",
- (int)',',
- &extra_ad_args},
{"-i",
"--init-img",
- "path to the init image, or a single input image for vid_upscale",
+ "path to the init image",
0,
&init_image_path},
{"",
@@ -1105,33 +1087,12 @@ ArgOptions SDGenerationParams::get_options() {
"path to control image, control net",
0,
&control_image_path},
- {"",
- "--ip-adapter-image",
- "path to the IP-Adapter reference image",
- 0,
- &ip_adapter_image_path},
{"",
"--control-video",
- "path to control video frames, It must be a directory path. The video frames inside should be stored as images in "
- "lexicographical (character) order. For example, if the control video path is `frames`, the directory contain images "
- "such as 00.png, 01.png, ... etc.",
+ "control video file (AVI, WebP or independent-frame VP8 WebM) or frame directory in filename order; "
+ "files are sampled at the output FPS without audio",
0,
&control_video_path},
- {"",
- "--pm-id-images-dir",
- "path to PHOTOMAKER input id images dir",
- 0,
- &pm_id_images_dir},
- {"",
- "--pm-id-embed-path",
- "path to PHOTOMAKER v2 id embed",
- 0,
- &pm_id_embed_path},
- {"",
- "--pulid-id-embedding",
- "path to PuLID id embedding",
- 0,
- &pulid_id_embedding_path},
{"",
"--hires-upscaler",
"highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent (nearest-exact), "
@@ -1148,7 +1109,7 @@ ArgOptions SDGenerationParams::get_options() {
&control_type},
{"",
"--extra-sample-args",
- "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; llada_image supports uniform; lms supports lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support noise_sampler with value iid (default except for dpm++2m_sde_bt) or brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or sampler_rng",
+ "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; llada_image supports uniform; lms supports lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support noise_sampler with value iid (default except for dpm++2m_sde_bt) or brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or sampler_rng",
(int)',',
&extra_sample_args},
{"",
@@ -1156,14 +1117,6 @@ ArgOptions SDGenerationParams::get_options() {
"extra VAE tiling args, key=value list. Supported video VAEs accept temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap (default: 1); Wan2.2 TI2V 5B accepts temporal_decode_frames (default: 1)",
(int)',',
&extra_tiling_args},
- {"",
- "--ref-image-args",
- "Key-value list to set up the way the reference images are processed (empty = auto-detect from model weigths)",
- (int)',',
- &ref_image_args},
- {"", "--image-preprocess",
- "Image preprocessing rule: target=init|end|mask|control|ref|ip-adapter|id|control-frame,index=N,mode=auto|none|stretch|crop|crop-resize|fit-pad,filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos,antialias=auto|true|false,width=W,height=H,anchor=center|top|bottom|left|right,pad_color=#RRGGBB[AA],canny=true|false. Repeat for multiple rules.",
- (int)';', &image_preprocess},
};
options.int_options = {
@@ -1181,7 +1134,7 @@ ArgOptions SDGenerationParams::get_options() {
&sample_params.sample_steps},
{"",
"--high-noise-steps",
- "(high noise) number of sample steps (default: -1 = auto)",
+ "high-noise steps; >= 0 adds to --steps, -1 splits --steps at --moe-boundary (default: -1)",
&high_noise_sample_params.sample_steps},
{"",
"--clip-skip",
@@ -1204,11 +1157,6 @@ ArgOptions SDGenerationParams::get_options() {
"--fps",
"fps (default: 24)",
&fps},
- {"",
- "--timestep-shift",
- "shift timestep for NitroFusion models (default: 0). "
- "recommended N for NitroSD-Realism around 250 and 500 for NitroSD-Vibrant",
- &sample_params.shifted_timestep},
{"",
"--upscale-repeats",
"Run the ESRGAN upscaler this many times (default: 1)",
@@ -1248,18 +1196,6 @@ ArgOptions SDGenerationParams::get_options() {
"--guidance",
"distilled guidance scale for models with guidance input (default: 3.5)",
&sample_params.guidance.distilled_guidance},
- {"",
- "--slg-scale",
- "skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5 medium",
- &sample_params.guidance.slg.scale},
- {"",
- "--skip-layer-start",
- "SLG enabling point (default: 0.01)",
- &sample_params.guidance.slg.layer_start},
- {"",
- "--skip-layer-end",
- "SLG disabling point (default: 0.2)",
- &sample_params.guidance.slg.layer_end},
{"",
"--eta",
"noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and dpm++2m_sde_bt)",
@@ -1268,54 +1204,14 @@ ArgOptions SDGenerationParams::get_options() {
"--flow-shift",
"shift value for Flow models like SD3.x or WAN (default: auto)",
&sample_params.flow_shift},
- {"",
- "--high-noise-cfg-scale",
- "(high noise) unconditional guidance scale: (default: 7.0)",
- &high_noise_sample_params.guidance.txt_cfg},
- {"",
- "--high-noise-img-cfg-scale",
- "(high noise) image guidance scale for inpaint or image edit models (default: same as --cfg-scale)",
- &high_noise_sample_params.guidance.img_cfg},
- {"",
- "--high-noise-guidance",
- "(high noise) distilled guidance scale for models with guidance input (default: 3.5)",
- &high_noise_sample_params.guidance.distilled_guidance},
- {"",
- "--high-noise-slg-scale",
- "(high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)",
- &high_noise_sample_params.guidance.slg.scale},
- {"",
- "--high-noise-skip-layer-start",
- "(high noise) SLG enabling point (default: 0.01)",
- &high_noise_sample_params.guidance.slg.layer_start},
- {"",
- "--high-noise-skip-layer-end",
- "(high noise) SLG disabling point (default: 0.2)",
- &high_noise_sample_params.guidance.slg.layer_end},
- {"",
- "--high-noise-eta",
- "(high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and dpm++2m_sde_bt)",
- &high_noise_sample_params.eta},
{"",
"--strength",
"strength for noising/unnoising (default: 0.75)",
&strength},
- {"",
- "--pm-style-strength",
- "",
- &pm_style_strength},
- {"",
- "--pulid-id-weight",
- "strength of PuLID identity injection",
- &pulid_id_weight},
{"",
"--control-strength",
"strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image",
&control_strength},
- {"",
- "--ip-adapter-strength",
- "strength to apply IP-Adapter (default: 1.0)",
- &ip_adapter_strength},
{"",
"--moe-boundary",
"timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1",
@@ -1339,11 +1235,6 @@ ArgOptions SDGenerationParams::get_options() {
};
options.bool_options = {
- {"",
- "--increase-ref-index",
- "automatically increase the indices of references images based on the order they are listed (starting with 1).",
- true,
- &increase_ref_index},
{"",
"--circular",
"enable circular padding on both axes for tileable output",
@@ -1358,7 +1249,7 @@ ArgOptions SDGenerationParams::get_options() {
true, &circular_y},
{"",
"--disable-image-metadata",
- "do not embed generation metadata on image files",
+ "do not embed generation metadata in image or video files (including DGL saves)",
false,
&embed_image_metadata},
{"",
@@ -1395,15 +1286,6 @@ ArgOptions SDGenerationParams::get_options() {
return 1;
};
- auto on_high_noise_sample_method_arg = [&](int argc, const char** argv, int index) {
- if (++index >= argc) {
- return -1;
- }
- const char* arg = argv[index];
- high_noise_sample_params.sample_method = arg;
- return 1;
- };
-
auto on_scheduler_arg = [&](int argc, const char** argv, int index) {
if (++index >= argc) {
return -1;
@@ -1413,68 +1295,6 @@ ArgOptions SDGenerationParams::get_options() {
return 1;
};
- auto on_skip_layers_arg = [&](int argc, const char** argv, int index) {
- if (++index >= argc) {
- return -1;
- }
- std::string layers_str = argv[index];
- if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') {
- return -1;
- }
-
- layers_str = layers_str.substr(1, layers_str.size() - 2);
-
- std::regex regex("[, ]+");
- std::sregex_token_iterator iter(layers_str.begin(), layers_str.end(), regex, -1);
- std::sregex_token_iterator end;
- std::vector<std::string> tokens(iter, end);
- std::vector<int> layers;
- for (const auto& token : tokens) {
- try {
- size_t used = 0;
- const int layer = std::stoi(token, &used);
- if (used != token.size())
- return -1;
- layers.push_back(layer);
- } catch (const std::exception&) {
- return -1;
- }
- }
- skip_layers = layers;
- return 1;
- };
-
- auto on_high_noise_skip_layers_arg = [&](int argc, const char** argv, int index) {
- if (++index >= argc) {
- return -1;
- }
- std::string layers_str = argv[index];
- if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') {
- return -1;
- }
-
- layers_str = layers_str.substr(1, layers_str.size() - 2);
-
- std::regex regex("[, ]+");
- std::sregex_token_iterator iter(layers_str.begin(), layers_str.end(), regex, -1);
- std::sregex_token_iterator end;
- std::vector<std::string> tokens(iter, end);
- std::vector<int> layers;
- for (const auto& token : tokens) {
- try {
- size_t used = 0;
- const int layer = std::stoi(token, &used);
- if (used != token.size())
- return -1;
- layers.push_back(layer);
- } catch (const std::exception&) {
- return -1;
- }
- }
- high_noise_skip_layers = layers;
- return 1;
- };
-
auto parse_sigmas_arg = [&](const char* value, std::optional<std::vector<float>>* target, const char* option_name) {
if (target == nullptr || value == nullptr) {
return -1;
@@ -1724,18 +1544,13 @@ ArgOptions SDGenerationParams::get_options() {
options.manual_options = {
{"-s",
"--seed",
- "RNG seed; unset or a negative value requests a random seed",
+ "RNG seed (default: " + (seed ? std::to_string(*seed) : "random") + "); a negative value requests a random seed",
on_seed_arg},
{"",
"--sampling-method",
"sampling method, one of [" + sample_methods + "], "
"default: euler for Flux/SD3/Wan, euler_a otherwise",
on_sample_method_arg},
- {"",
- "--high-noise-sampling-method",
- "(high noise) sampling method, one of [" + sample_methods + "], "
- "default: euler for Flux/SD3/Wan, euler_a otherwise",
- on_high_noise_sample_method_arg},
{"",
"--scheduler",
"denoiser sigma scheduler, one of [" + schedulers + "], "
@@ -1749,25 +1564,17 @@ ArgOptions SDGenerationParams::get_options() {
"--hires-sigmas",
"custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image, minimax-h3-image, hyperflow-h3-flash-4, hyperflow-h3-flash-6, hyperflow-h3-flash-8.",
on_hires_sigmas_arg},
- {"",
- "--skip-layers",
- "layers to skip for SLG steps; unset uses the library default",
- on_skip_layers_arg},
- {"",
- "--high-noise-skip-layers",
- "(high noise) layers to skip for SLG steps; unset uses the library default",
- on_high_noise_skip_layers_arg},
{"-r",
"--ref-image",
"reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple times)",
on_ref_image_arg},
{"",
"--ref-video",
- "MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used multiple times)",
+ "MiniMax-H3 Ref2VA reference video file or frame directory, sampled at 24 fps (can be used multiple times)",
on_ref_video_arg},
{"",
"--ref-video-audio",
- "WAV soundtrack paired by index with --ref-video (can be used multiple times)",
+ "WAV soundtrack overriding embedded audio, paired by index with --ref-video (can be used multiple times)",
on_ref_video_audio_arg},
{"",
"--ref-audio",
@@ -1779,11 +1586,11 @@ ArgOptions SDGenerationParams::get_options() {
on_audio_arg},
{"",
"--cache-mode",
- "caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)",
+ "caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT condition-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)",
on_cache_mode_arg},
{"",
"--cache-option",
- "named cache params (key=value format, comma-separated). easycache/ucache: threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=. Examples: \"threshold=0.25\" or \"threshold=1.5,reset=0\"",
+ "named cache params (key=value format, comma-separated). easycache/ucache: threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: threshold=,warmup= (Fn/Bn are ignored with a warning); spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=. Examples: \"threshold=0.25\" or \"threshold=1.5,reset=0\"",
on_cache_option_arg},
{"",
"--scm-mask",
@@ -2124,29 +1931,6 @@ static bool resolve_model_file_from_dir(const std::string& model_name,
return false;
}
-bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str) {
- const auto value = json::parse(json_str, nullptr, false);
- std::string rules;
- if (value.is_string()) {
- rules = value.get<std::string>();
- } else if (value.is_array()) {
- for (const auto& item : value) {
- if (!item.is_string()) {
- LOG_ERROR("image_preprocess must contain rule strings");
- return false;
- }
- if (!rules.empty())
- rules += ";";
- rules += item.get<std::string>();
- }
- } else {
- LOG_ERROR("image_preprocess must be a string or array of strings");
- return false;
- }
- image_preprocess = std::move(rules);
- return true;
-}
-
bool SDGenerationParams::parse_prompt_images_json(const std::string& json_str) {
auto images = json::parse(json_str, nullptr, false);
if (!images.is_object()) {
@@ -2213,11 +1997,14 @@ bool SDGenerationParams::from_json_str(
return false;
}
- if (j.contains("image_preprocess") && !parse_image_preprocess_json(j["image_preprocess"].dump()))
- return false;
+ if (j.contains("image_preprocess"))
+ throw std::invalid_argument("image_preprocess requires a DGL script");
if (j.contains("prompt_images") && !parse_prompt_images_json(j["prompt_images"].dump()))
return false;
+ if ((j.contains("ref_image_args") && j["ref_image_args"] != "") ||
+ (j.contains("increase_ref_index") && j["increase_ref_index"] != false))
+ throw std::invalid_argument("reference overrides require a DGL script");
auto load_if_exists = [&](const char* key, auto& out) { load_json_input(j, key, out); };
load_if_exists("prompt", prompt);
@@ -2229,7 +2016,6 @@ bool SDGenerationParams::from_json_str(
load_if_exists("cache_mode", cache_mode);
load_if_exists("cache_option", cache_option);
load_if_exists("scm_mask", scm_mask);
- load_if_exists("ref_image_args", ref_image_args);
load_if_exists("clip_skip", clip_skip);
load_if_exists("width", width);
@@ -2248,7 +2034,6 @@ bool SDGenerationParams::from_json_str(
load_if_exists("moe_boundary", moe_boundary);
load_if_exists("vace_strength", vace_strength);
- load_if_exists("increase_ref_index", increase_ref_index);
load_if_exists("embed_image_metadata", embed_image_metadata);
if (j.contains("hires") && j["hires"].is_object()) {
@@ -2479,10 +2264,10 @@ bool SDGenerationParams::initialize_cache_params() {
cache_params.use_relative_threshold = (std::stof(val) != 0.0f);
} else if (key == "reset") {
cache_params.reset_error_on_compute = (std::stof(val) != 0.0f);
- } else if (key == "Fn" || key == "fn") {
- cache_params.Fn_compute_blocks = std::stoi(val);
- } else if (key == "Bn" || key == "bn") {
- cache_params.Bn_compute_blocks = std::stoi(val);
+ } else if (key == "Fn" || key == "fn" || key == "Bn" || key == "bn") {
+ if (std::stoi(val) < 0)
+ throw std::out_of_range("block count must be nonnegative");
+ LOG_WARN("cache option '%s' is ignored: condition-level caching does not support front/back block counts", key.c_str());
} else if (key == "warmup") {
if (cache_mode == "spectrum") {
cache_params.spectrum_warmup_steps = std::stoi(val);
@@ -2592,10 +2377,6 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
/** Validates params for a generation mode; enforces mode-specific input rules. */
bool SDGenerationParams::validate(SDMode mode) {
- if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
- LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
- return false;
- }
if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) {
LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
return false;
@@ -2611,21 +2392,15 @@ bool SDGenerationParams::validate(SDMode mode) {
return false;
}
- if (mode == UPSCALE || mode == ADETAILER) {
+ if (mode == UPSCALE) {
if (init_image_path.length() == 0) {
- LOG_ERROR("error: %s mode needs an init image (--init-img)\n",
- mode == UPSCALE ? "upscale" : "adetailer");
+ LOG_ERROR("error: upscale mode needs an init image (--init-img)");
return false;
}
}
- if (mode == ADETAILER && ad_model_path.empty()) {
- LOG_ERROR("error: adetailer mode needs a detector model (--ad-model)\n");
- return false;
- }
-
- if (mode == IMG_GEN || mode == ADETAILER || mode == VID_GEN) {
- auto opt = mode == VID_GEN ? to_video_opt() : to_image_opt(mode == ADETAILER);
+ if (mode == IMG_GEN || mode == VID_GEN) {
+ auto opt = mode == VID_GEN ? to_video_opt() : to_image_opt();
if (!opt) {
LOG_ERROR("%s", sd_get_last_error());
return false;
@@ -2671,10 +2446,13 @@ static void set_sample_options(SDOptBuilder& opt, const std::string& prefix, con
}
static void set_cache_options(SDOptBuilder& opt, const SDCacheParams& params) {
- opt.set_int("cache.bn_compute_blocks", params.Bn_compute_blocks);
+ const SDCacheParams defaults;
+ if (params.Bn_compute_blocks != defaults.Bn_compute_blocks)
+ opt.set_int("cache.bn_compute_blocks", params.Bn_compute_blocks);
opt.set_float("cache.end_percent", params.end_percent);
opt.set_float("cache.error_decay_rate", params.error_decay_rate);
- opt.set_int("cache.fn_compute_blocks", params.Fn_compute_blocks);
+ if (params.Fn_compute_blocks != defaults.Fn_compute_blocks)
+ opt.set_int("cache.fn_compute_blocks", params.Fn_compute_blocks);
opt.set_int("cache.max_cached_steps", params.max_cached_steps);
opt.set_int("cache.max_continuous_cached_steps", params.max_continuous_cached_steps);
opt.set_int("cache.max_warmup_steps", params.max_warmup_steps);
@@ -2708,7 +2486,7 @@ static void set_tiling_options(SDOptBuilder& opt, const SDTilingParams& params)
opt.set_int("vae_tiling.tile_size_w", params.tile_size_w);
}
-SDOptPtr SDGenerationParams::to_image_opt(bool adetail) {
+SDOptPtr SDGenerationParams::to_image_opt() {
lora_vec.clear();
lora_vec.reserve(lora_map.size() + high_noise_lora_map.size());
for (const auto& kv : lora_map) {
@@ -2724,68 +2502,37 @@ SDOptPtr SDGenerationParams::to_image_opt(bool adetail) {
ref_image_views.push_back(ref_image.get());
}
- pm_id_image_views.clear();
- pm_id_image_views.reserve(pm_id_images.size());
- for (auto& image : pm_id_images) {
- pm_id_image_views.push_back(image.get());
- }
-
vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str();
- std::string effective_ref_image_args = ref_image_args;
- if (!adetail && increase_ref_index) {
- if (!effective_ref_image_args.empty())
- effective_ref_image_args += ",";
- effective_ref_image_args += "ref_index_mode=increase";
- }
-
- SDOptBuilder opt(adetail ? sd_adetail_opt_create() : sd_img_gen_opt_create());
+ SDOptBuilder opt(sd_img_gen_opt_create());
opt.set_string("prompt", prompt);
opt.set_string("negative_prompt", negative_prompt);
opt.set_int("clip_skip", clip_skip);
- if (!adetail)
- opt.set_image("init_image", init_image.get());
- if (!adetail)
- opt.set_string("ref_image_args", effective_ref_image_args);
- if (!adetail)
- opt.set_image("mask_image", mask_image.get());
+ opt.set_image("init_image", init_image.get());
+ opt.set_image("mask_image", mask_image.get());
opt.set_int("width", width);
opt.set_int("height", height);
opt.set_float("strength", strength);
opt.set_int("seed", seed);
- if (!adetail)
- opt.set_int("batch_count", batch_count);
- if (!adetail)
- opt.set_int("qwen_image_layers", qwen_image_layers);
- if (!adetail)
- opt.set_image("control_image", control_image.get());
- if (!adetail)
- opt.set_float("control_strength", control_strength);
- if (!adetail)
- opt.set_string("control_type", control_type);
- if (!adetail)
- opt.set_image("ip_adapter_image", ip_adapter_image.get());
- if (!adetail)
- opt.set_float("ip_adapter_strength", ip_adapter_strength);
- if (!adetail)
+ opt.set_int("batch_count", batch_count);
+ opt.set_int("qwen_image_layers", qwen_image_layers);
+ opt.set_image("control_image", control_image.get());
+ opt.set_float("control_strength", control_strength);
+ opt.set_string("control_type", control_type);
+ opt.set_image("ip_adapter_image", ip_adapter_image.get());
+ opt.set_float("ip_adapter_strength", ip_adapter_strength);
+ if (hires_enabled.value_or(false))
opt.set_bool("hires.enabled", hires_enabled);
- if (!adetail)
- opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional<std::string>(resolved_hires_upscaler));
- if (!adetail)
+ opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional<std::string>(resolved_hires_upscaler));
+ if (!hires_upscaler_model_path.empty())
opt.set_string("hires.model_path", hires_upscaler_model_path);
- if (!adetail)
- opt.set_float("hires.scale", hires_scale);
- if (!adetail)
- opt.set_int("hires.target_width", hires_width);
- if (!adetail)
- opt.set_int("hires.target_height", hires_height);
- if (!adetail)
- opt.set_int("hires.steps", hires_steps);
- if (!adetail)
- opt.set_float("hires.denoising_strength", hires_denoising_strength);
- if (!adetail)
- opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size);
+ opt.set_float("hires.scale", hires_scale);
+ opt.set_int("hires.target_width", hires_width);
+ opt.set_int("hires.target_height", hires_height);
+ opt.set_int("hires.steps", hires_steps);
+ opt.set_float("hires.denoising_strength", hires_denoising_strength);
+ opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size);
if (circular)
opt.set_bool("circular_x", true);
else
@@ -2795,32 +2542,11 @@ SDOptPtr SDGenerationParams::to_image_opt(bool adetail) {
else
opt.set_bool("circular_y", circular_y);
opt.set_loras("loras", lora_vec.data(), lora_vec.size());
- if (!adetail)
- opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size());
- if (!adetail)
- opt.set_images("photomaker.id_images", pm_id_image_views.data(), pm_id_image_views.size());
- if (!adetail)
- opt.set_string("photomaker.id_embed_path", pm_id_embed_path);
- if (!adetail)
- opt.set_float("photomaker.style_strength", pm_style_strength);
- if (!adetail)
- opt.set_string("pulid.id_embedding_path", pulid_id_embedding_path);
- if (!adetail)
- opt.set_float("pulid.id_weight", pulid_id_weight);
- if (!adetail)
- opt.set_floats("hires.sigmas", hires_custom_sigmas);
- if (!adetail)
- opt.set_string("image_preprocess.rules", image_preprocess);
+ opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size());
+ opt.set_floats("hires.sigmas", hires_custom_sigmas);
set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args);
set_cache_options(opt, cache_params);
set_tiling_options(opt, vae_tiling_params);
- if (adetail) {
- if (!ad_prompt.empty())
- opt.set_string("adetail.prompt", ad_prompt);
- if (!ad_negative_prompt.empty())
- opt.set_string("adetail.negative_prompt", ad_negative_prompt);
- opt.set_string("adetail.args", extra_ad_args);
- }
return opt.finish();
}
@@ -2886,9 +2612,11 @@ SDOptPtr SDGenerationParams::to_video_opt() {
opt.set_int("video_frames", video_frames);
opt.set_int("fps", fps);
opt.set_float("vace_strength", vace_strength);
- opt.set_bool("hires.enabled", hires_enabled);
+ if (hires_enabled.value_or(false))
+ opt.set_bool("hires.enabled", hires_enabled);
opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional<std::string>(resolved_hires_upscaler));
- opt.set_string("hires.model_path", hires_upscaler_model_path);
+ if (!hires_upscaler_model_path.empty())
+ opt.set_string("hires.model_path", hires_upscaler_model_path);
opt.set_float("hires.scale", hires_scale);
opt.set_int("hires.target_width", hires_width);
opt.set_int("hires.target_height", hires_height);
@@ -2903,14 +2631,12 @@ SDOptPtr SDGenerationParams::to_video_opt() {
opt.set_bool("circular_y", true);
else
opt.set_bool("circular_y", circular_y);
- opt.set_string("ref_image_args", ref_image_args);
opt.set_loras("loras", lora_vec.data(), lora_vec.size());
opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size());
opt.set_videos("ref_videos", ref_video_views.data(), ref_video_views.size());
opt.set_audios("ref_audios", ref_audio_views.data(), ref_audio_views.size());
opt.set_frames("control_frames", control_frame_views.data(), control_frame_views.size());
opt.set_floats("hires.sigmas", hires_custom_sigmas);
- opt.set_string("image_preprocess.rules", image_preprocess);
opt.set_string("sample.scheduler", high_noise_sample_params.scheduler);
opt.set_float("sample.flow_shift", high_noise_sample_params.flow_shift);
opt.set_int("sample.shifted_timestep", high_noise_sample_params.shifted_timestep);
@@ -2974,12 +2700,6 @@ std::string SDGenerationParams::to_string() const {
<< " ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n"
<< " ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n"
<< " control_video_path: \"" << control_video_path << "\",\n"
- << " image_preprocess: " << image_preprocess << ",\n"
- << " ref_image_args: " << ref_image_args << ",\n"
- << " increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n"
- << " pm_id_images_dir: \"" << pm_id_images_dir << "\",\n"
- << " pm_id_embed_path: \"" << pm_id_embed_path << "\",\n"
- << " pm_style_strength: " << pm_style_strength << ",\n"
<< " skip_layers: " << (skip_layers ? vec_to_string(*skip_layers) : "unset") << ",\n"
<< " sample_params: " << SAFE_STR(sample_params_str.c_str()) << ",\n"
<< " high_noise_skip_layers: " << (high_noise_skip_layers ? vec_to_string(*high_noise_skip_layers) : "unset") << ",\n"
diff --git a/examples/common/common.h b/examples/common/common.h
index eb4a043..f4418ad 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -1,7 +1,6 @@
#ifndef __EXAMPLES_COMMON_COMMON_H__
#define __EXAMPLES_COMMON_COMMON_H__
-#include "generation_metadata.h"
#include "options.h"
#include <cmath>
@@ -23,16 +22,15 @@ std::string argv_to_utf8(int index, const char** argv);
#define BOOL_STR(b) ((b) ? "true" : "false")
extern const char* const modes_str[];
-#define SD_ALL_MODES_STR "img_gen, adetailer, vid_gen, convert, upscale, metadata, vid_upscale"
+#define SD_ALL_MODES_STR "img_gen, vid_gen, convert, upscale, metadata, dgl"
enum SDMode {
IMG_GEN,
- ADETAILER,
VID_GEN,
CONVERT,
UPSCALE,
METADATA,
- VID_UPSCALE,
+ DGL,
MODE_COUNT
};
@@ -202,6 +200,8 @@ struct SDContextParams {
bool resolve_and_validate(SDMode mode);
std::string to_string() const;
SDOptPtr to_context_opt();
+ bool configure_dgl(sd_dgl_ctx_t* ctx) const;
+ bool register_rpc_devices() const;
};
struct SDGenerationParams {
@@ -221,7 +221,6 @@ struct SDGenerationParams {
std::optional<float> strength;
std::optional<float> control_strength;
std::optional<float> ip_adapter_strength;
- bool increase_ref_index = false;
bool embed_image_metadata = true;
std::string init_image_path;
@@ -229,7 +228,6 @@ struct SDGenerationParams {
std::string mask_image_path;
std::string control_image_path;
std::optional<std::string> control_type;
- std::string ip_adapter_image_path;
std::vector<std::string> ref_image_paths;
std::vector<std::string> ref_video_paths;
std::vector<std::string> ref_video_audio_paths;
@@ -258,16 +256,6 @@ struct SDGenerationParams {
SDTilingParams vae_tiling_params;
std::string extra_tiling_args;
- std::string ref_image_args;
- std::string image_preprocess;
-
- std::string pm_id_images_dir;
- std::string pm_id_embed_path;
- std::optional<float> pm_style_strength;
-
- std::string pulid_id_embedding_path;
- std::optional<float> pulid_id_weight;
-
int upscale_repeats = 1;
std::optional<int> upscale_tile_size;
@@ -305,7 +293,6 @@ struct SDGenerationParams {
SDImageOwner mask_image;
SDImageOwner control_image;
SDImageOwner ip_adapter_image;
- std::vector<SDImageOwner> pm_id_images;
std::vector<SDImageOwner> control_frames;
// Media views used while populating the opaque options.
@@ -313,7 +300,6 @@ struct SDGenerationParams {
std::vector<std::vector<SDImage>> ref_video_frame_views;
std::vector<SDReferenceVideo> ref_video_views;
std::vector<SDAudio> ref_audio_views;
- std::vector<SDImage> pm_id_image_views;
std::vector<SDImage> control_frame_views;
SDGenerationParams();
@@ -324,7 +310,6 @@ struct SDGenerationParams {
ArgOptions get_options();
bool from_json_str(const std::string& json_str,
const std::function<std::string(const std::string&)>& lora_path_resolver = {});
- bool parse_image_preprocess_json(const std::string& json_str);
bool parse_prompt_images_json(const std::string& json_str);
bool initialize_cache_params();
bool extract_and_remove_lora(const std::string& lora_model_dir);
@@ -338,7 +323,7 @@ struct SDGenerationParams {
const std::string& lora_model_dir,
const std::string& hires_upscalers_dir,
bool strict = false);
- SDOptPtr to_image_opt(bool adetail = false);
+ SDOptPtr to_image_opt();
SDOptPtr to_video_opt();
std::string to_string() const;
};
diff --git a/examples/common/generation_metadata.cpp b/examples/common/generation_metadata.cpp
deleted file mode 100644
index 05205ff..0000000
--- a/examples/common/generation_metadata.cpp
+++ /dev/null
@@ -1,164 +0,0 @@
-#include "generation_metadata.h"
-
-#include <cctype>
-#include <filesystem>
-#include <map>
-
-#include "json.hpp"
-
-static void relativize_lora_paths(nlohmann::json& info, const std::filesystem::path& root) {
- if (info.contains("loras")) {
- for (auto& lora : info["loras"]) {
- const auto path = lora.value("path", "");
- if (path.empty())
- continue;
- std::error_code error;
- const auto absolute = std::filesystem::absolute(std::filesystem::u8path(path), error);
- if (error)
- continue;
- const auto relative = absolute.lexically_normal().lexically_relative(root);
- if (!relative.empty())
- lora["path"] = relative.generic_u8string();
- }
- }
- if (info.contains("inpaint_params"))
- relativize_lora_paths(info["inpaint_params"], root);
- if (info.contains("postprocess")) {
- for (auto& operation : info["postprocess"])
- relativize_lora_paths(operation, root);
- }
-}
-
-std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir) {
- if (info.empty() || lora_model_dir.empty())
- return info;
- std::error_code error;
- const auto root = std::filesystem::absolute(std::filesystem::u8path(lora_model_dir), error);
- if (error)
- return info;
- auto result = nlohmann::json::parse(info);
- relativize_lora_paths(result, root.lexically_normal());
- return result.dump();
-}
-
-std::string merge_generation_info(const std::string& source, const std::string& operation) {
- if (source.empty())
- return operation;
- auto result = nlohmann::json::parse(source);
- const auto next = nlohmann::json::parse(operation);
- result["postprocess"].push_back(next);
- return result.dump();
-}
-
-int64_t generation_seed(const std::string& info, int64_t fallback) {
- if (info.empty())
- return fallback;
- const auto value = nlohmann::json::parse(info);
- const auto& params = value.contains("inpaint_params") ? value.at("inpaint_params") : value;
- return params.contains("seed") ? std::stoll(params.at("seed").get<std::string>()) : fallback;
-}
-
-std::string generation_info_to_webui(const std::string& text) {
- using nlohmann::json;
- if (text.empty())
- return "";
- const auto info = json::parse(text);
- if (info.value("schema", "") != "sdcpp.generation/v1")
- return "";
- // Standalone postprocessing has no source diffusion settings to import.
- if (!info.contains("sample_params"))
- return "";
- const auto& sampling = info.at("sample_params");
- std::string result = info.value("prompt", "");
- if (info.contains("loras")) {
- bool prompt_trimmed = false;
- for (const auto& lora : info.at("loras")) {
- // WebUI tags cannot restrict a LoRA to the high-noise model.
- if (lora.value("is_high_noise", false))
- continue;
- auto name = lora.at("path").get<std::string>();
- const auto separator = name.find_last_of("/\\");
- if (separator != std::string::npos)
- name.erase(0, separator + 1);
- const auto extension = name.find_last_of('.');
- if (extension != std::string::npos && extension > 0)
- name.erase(extension);
- if (name.empty() || name.find_first_of(":<>\r\n") != std::string::npos)
- continue;
- if (!result.empty()) {
- if (!prompt_trimmed) {
- while (!result.empty() && std::isspace(static_cast<unsigned char>(result.back())))
- result.pop_back();
- prompt_trimmed = true;
- }
- if (!result.empty())
- result += " ";
- }
- result += "<lora:" + name + ":" + lora.at("multiplier").dump() + ">";
- }
- }
- const auto negative = info.value("negative_prompt", "");
- if (!negative.empty())
- result += "\nNegative prompt: " + negative;
- result += "\n";
- bool first = true;
- const auto field = [&](const char* key, const json& value) {
- std::string v = value.is_string() ? value.get<std::string>() : value.dump();
- if (v.find_first_of(",\n\r:\"") != std::string::npos)
- v = json(v).dump(-1, ' ', false);
- if (!first)
- result += ", ";
- first = false;
- result += std::string(key) + ": " + v;
- };
- field("Steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps"));
- static const std::map<std::string, std::string> samplers = {
- {"euler", "Euler"}, {"euler_a", "Euler a"}, {"heun", "Heun"}, {"dpm2", "DPM2"}, {"dpm++2s_a", "DPM++ 2S a"}, {"dpm++2m", "DPM++ 2M"}, {"dpm++2m_sde", "DPM++ 2M SDE"}, {"lcm", "LCM"}};
- const auto method = sampling.at("sample_method").get<std::string>();
- auto mapped = samplers.find(method);
- field(mapped != samplers.end() ? "Sampler" : "SDCPP Sampler", mapped != samplers.end() ? mapped->second : method);
- static const std::map<std::string, std::string> schedules = {
- {"discrete", "Normal"}, {"karras", "Karras"}, {"exponential", "Exponential"}, {"sgm_uniform", "SGM Uniform"}, {"simple", "Simple"}, {"ddim_uniform", "DDIM"}, {"beta", "Beta"}};
- if (sampling.contains("scheduler")) {
- const auto scheduler = sampling.at("scheduler").get<std::string>();
- auto mapped_schedule = schedules.find(scheduler);
- field(mapped_schedule != schedules.end() ? "Schedule type" : "SDCPP Schedule", mapped_schedule != schedules.end() ? mapped_schedule->second : scheduler);
- }
- field("CFG scale", sampling.at("guidance").at("txt_cfg"));
- field("Seed", info.at("seed"));
- field("Size", std::to_string(info.at("width").get<int>()) + "x" + std::to_string(info.at("height").get<int>()));
- if (info.contains("models")) {
- const auto& models = info.at("models");
- auto model = models.value("diffusion_model", models.value("model", ""));
- const auto extension = model.find_last_of('.');
- if (extension != std::string::npos && extension > 0)
- model.erase(extension);
- if (!model.empty())
- field("Model", model);
- if (models.contains("vae"))
- field("VAE", models.at("vae"));
- }
- if (sampling.contains("eta"))
- field("Eta", sampling.at("eta"));
- if (info.value("clip_skip", 0) > 0)
- field("Clip skip", info.at("clip_skip"));
- if (info.contains("strength"))
- field("Denoising strength", info.at("strength"));
- if (info.contains("hires") && info.at("hires").value("enabled", false)) {
- const auto& hires = info.at("hires");
- field("Hires resize", std::to_string(hires.at("target_width").get<int>()) + "x" + std::to_string(hires.at("target_height").get<int>()));
- if (hires.value("steps", 0) > 0)
- field("Hires steps", hires.at("steps"));
- field("Denoising strength", hires.at("denoising_strength"));
- field("Hires upscaler", hires.at("upscaler"));
- }
- field("Version", "stable-diffusion.cpp");
- return result;
-}
-
-std::string generation_info_to_image_metadata(const std::string& info) {
- if (info.empty())
- return "";
- const auto parameters = generation_info_to_webui(info);
- return parameters.empty() ? info : parameters + ", SDCPP: " + info;
-}
diff --git a/examples/common/generation_metadata.h b/examples/common/generation_metadata.h
deleted file mode 100644
index 7c9e9ff..0000000
--- a/examples/common/generation_metadata.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef __EXAMPLES_COMMON_GENERATION_METADATA_H__
-#define __EXAMPLES_COMMON_GENERATION_METADATA_H__
-
-#include <cstdint>
-#include <string>
-
-std::string merge_generation_info(const std::string& source, const std::string& operation);
-std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir);
-int64_t generation_seed(const std::string& info, int64_t fallback = -1);
-std::string generation_info_to_webui(const std::string& text);
-std::string generation_info_to_image_metadata(const std::string& info);
-
-#endif // __EXAMPLES_COMMON_GENERATION_METADATA_H__
diff --git a/examples/common/log.cpp b/examples/common/log.cpp
index 12d0683..5a656d0 100644
--- a/examples/common/log.cpp
+++ b/examples/common/log.cpp
@@ -4,6 +4,7 @@
sd_log_level_t log_level = SD_LOG_INFO;
bool log_color = false;
+FILE* log_stream = nullptr;
std::string sd_basename(const std::string& path) {
size_t pos = path.find_last_of('/');
@@ -79,10 +80,10 @@ bool parse_log_level(const std::string& name, sd_log_level_t& level) {
return false;
}
-void log_print(enum sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color) {
+void log_print(enum sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color, FILE* stream) {
int tag_color;
const char* level_str;
- FILE* out_stream = (level == SD_LOG_ERROR) ? stderr : stdout;
+ FILE* out_stream = stream ? stream : ((level == SD_LOG_ERROR) ? stderr : stdout);
if (!log || level < min_level) {
return;
@@ -141,5 +142,5 @@ void example_log_printf(sd_log_level_t level, const char* file, int line, const
}
message += " --- " + sd_basename(file) + ":" + std::to_string(line) + '\n';
- log_print(level, message.c_str(), log_level, log_color);
+ log_print(level, message.c_str(), log_level, log_color, log_stream);
}
diff --git a/examples/common/log.h b/examples/common/log.h
index f623b0e..fd5bd4c 100644
--- a/examples/common/log.h
+++ b/examples/common/log.h
@@ -18,12 +18,13 @@
extern sd_log_level_t log_level;
extern bool log_color;
+extern FILE* log_stream;
std::string sd_basename(const std::string& path);
void print_utf8(FILE* stream, const char* utf8);
const char* log_level_name(sd_log_level_t level);
bool parse_log_level(const std::string& name, sd_log_level_t& level);
-void log_print(sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color);
+void log_print(sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color, FILE* stream = nullptr);
void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
#define LOG_DEBUG(format, ...) example_log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp
index 4159e46..941c70b 100644
--- a/examples/common/media_io.cpp
+++ b/examples/common/media_io.cpp
@@ -1,5 +1,4 @@
#include "media_io.h"
-#include "generation_metadata.h"
#include "log.h"
#include "media_metadata.h"
#include "resource_owners.hpp"
@@ -377,7 +376,7 @@ bool encode_webp_image_to_vector(const uint8_t* image,
out.assign(encoded.get(), encoded.get() + encoded_size);
- return embed_webp_metadata(out, generation_info_to_image_metadata(parameters));
+ return embed_webp_metadata(out, parameters);
}
#ifdef SD_USE_WEBM
@@ -710,7 +709,6 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
const std::string& parameters,
int quality) {
std::vector<uint8_t> buffer;
- const auto metadata = format == EncodedImageFormat::WEBP ? std::string() : generation_info_to_image_metadata(parameters);
auto write_func = [&buffer](void* context, void* data, int size) {
(void)context;
@@ -733,7 +731,7 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
result = stbi_write_jpg_to_func(c_func, &ctx, width, height, channels, image, quality);
break;
case EncodedImageFormat::PNG:
- result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, metadata.empty() ? nullptr : metadata.c_str());
+ result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, nullptr);
break;
case EncodedImageFormat::WEBP:
#ifdef SD_USE_WEBP
@@ -751,7 +749,8 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
break;
}
- if (result && format == EncodedImageFormat::JPEG && !embed_jpeg_metadata(buffer, metadata)) {
+ if (result && ((format == EncodedImageFormat::JPEG && !embed_jpeg_metadata(buffer, parameters)) ||
+ (format == EncodedImageFormat::PNG && !embed_png_metadata(buffer, parameters)))) {
LOG_ERROR("generation metadata exceeds the container capacity or the encoded image is invalid");
result = 0;
}
@@ -813,27 +812,13 @@ uint8_t* load_image_from_memory(const char* image_bytes,
}
static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {
- if (parameters.empty()) {
- return;
- }
-
- std::vector<uint8_t> info_content;
-
- write_fourcc(info_content, "INFO");
-
- const size_t comment_size = parameters.size() + 1;
- write_fourcc(info_content, "ICMT");
- write_u32_le(info_content, static_cast<uint32_t>(comment_size));
- info_content.insert(info_content.end(), parameters.begin(), parameters.end());
- info_content.push_back(0);
- if (comment_size & 1u) {
- info_content.push_back(0);
- }
-
- write_fourcc(data, "LIST");
- write_u32_le(data, static_cast<uint32_t>(info_content.size()));
- data.insert(data.end(), info_content.begin(), info_content.end());
- size_t start_pos = data.size();
+ const void* info = nullptr;
+ size_t size = 0;
+ if (sd_get_metadata_chunk(parameters.c_str(), SD_METADATA_AVI, &info, &size) != SD_STATUS_OK)
+ throw std::runtime_error(sd_get_last_error());
+ const auto* bytes = static_cast<const uint8_t*>(info);
+ if (size)
+ data.insert(data.end(), bytes, bytes + size);
}
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
diff --git a/examples/common/media_io.h b/examples/common/media_io.h
index d765db7..5951f39 100644
--- a/examples/common/media_io.h
+++ b/examples/common/media_io.h
@@ -16,7 +16,7 @@ enum class EncodedImageFormat {
EncodedImageFormat encoded_image_format_from_path(const std::string& path);
-// parameters is the core sdcpp.generation/v1 JSON; empty disables both metadata formats.
+// parameters is the executed DGL source; empty disables generation metadata.
std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
const uint8_t* image,
int width,
diff --git a/examples/common/media_metadata.cpp b/examples/common/media_metadata.cpp
index d5e9af3..d665c28 100644
--- a/examples/common/media_metadata.cpp
+++ b/examples/common/media_metadata.cpp
@@ -1,47 +1,43 @@
#include "media_metadata.h"
-std::string generation_xmp(const std::string& info) {
- std::string escaped;
- for (char c : info) {
- switch (c) {
- case '&':
- escaped += "&";
- break;
- case '<':
- escaped += "<";
- break;
- case '>':
- escaped += ">";
- break;
- case '"':
- escaped += """;
- break;
- case '\'':
- escaped += "'";
- break;
- default:
- escaped += c;
- break;
- }
- }
- return "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">"
- "<rdf:Description rdf:about=\"\" xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">"
- "<sdcpp:parameters>" +
- escaped + "</sdcpp:parameters></rdf:Description></rdf:RDF></x:xmpmeta>";
+#include <cstring>
+
+#include "stable-diffusion.h"
+
+std::string generation_xmp(const std::string& source) {
+ const void* data = nullptr;
+ size_t size = 0;
+ if (sd_get_metadata_chunk(source.c_str(), SD_METADATA_XMP, &data, &size) != SD_STATUS_OK || !size)
+ return "";
+ return std::string(static_cast<const char*>(data), size);
}
-bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters) {
- if (parameters.empty())
+bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& source) {
+ if (source.empty())
return true;
- const char label[] = "parameters";
- const size_t overhead = sizeof(label) + 1;
- if (image.size() < 2 || parameters.size() > 65533 - overhead)
+ if (image.size() < 2)
+ return false;
+ const void* data = nullptr;
+ size_t size = 0;
+ if (sd_get_metadata_chunk(source.c_str(), SD_METADATA_JPEG, &data, &size) != SD_STATUS_OK)
+ return false;
+ const auto* bytes = static_cast<const uint8_t*>(data);
+ if (size)
+ image.insert(image.begin() + 2, bytes, bytes + size);
+ return true;
+}
+
+bool embed_png_metadata(std::vector<uint8_t>& image, const std::string& source) {
+ if (source.empty())
+ return true;
+ if (image.size() < 33 || std::memcmp(image.data() + 12, "IHDR", 4))
+ return false;
+ const void* data = nullptr;
+ size_t size = 0;
+ if (sd_get_metadata_chunk(source.c_str(), SD_METADATA_PNG, &data, &size) != SD_STATUS_OK)
return false;
- const size_t length = overhead + parameters.size() + 2;
- std::vector<uint8_t> segment = {0xff, 0xfe, static_cast<uint8_t>(length >> 8), static_cast<uint8_t>(length)};
- segment.insert(segment.end(), label, label + sizeof(label));
- segment.insert(segment.end(), parameters.begin(), parameters.end());
- segment.push_back(0);
- image.insert(image.begin() + 2, segment.begin(), segment.end());
+ const auto* bytes = static_cast<const uint8_t*>(data);
+ if (size)
+ image.insert(image.begin() + 33, bytes, bytes + size);
return true;
}
diff --git a/examples/common/media_metadata.h b/examples/common/media_metadata.h
index 1843dee..1ca49dc 100644
--- a/examples/common/media_metadata.h
+++ b/examples/common/media_metadata.h
@@ -6,6 +6,7 @@
#include <vector>
std::string generation_xmp(const std::string& parameters);
+bool embed_png_metadata(std::vector<uint8_t>& image, const std::string& source);
bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters);
#endif // __EXAMPLES_COMMON_MEDIA_METADATA_H__
diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp
index 8ba69e7..d76e7e6 100644
--- a/examples/common/resource_owners.hpp
+++ b/examples/common/resource_owners.hpp
@@ -12,7 +12,6 @@
#include <vector>
#include <optional>
-#include "generation_metadata.h"
#include "log.h"
#include "stable-diffusion.h"
@@ -123,17 +122,6 @@ static inline SDAudio* retain_result_audio(const sd_result_t* result) {
return view;
}
-static inline bool preprocess_canny(SDImage& image, float high, float low, float weak, float strong, bool inverse) {
- auto input = image_tensor(image);
- sd_tensor_t* output = nullptr;
- if (!input || sd_preprocess_canny(input.get(), high, low, weak, strong, inverse, &output) != SD_STATUS_OK)
- return false;
- sd_image_free(&image);
- image = tensor_image_view(output);
- image.storage = output;
- return true;
-}
-
struct FreeDeleter {
void operator()(void* ptr) const {
free(ptr);
@@ -164,21 +152,12 @@ struct UpscalerCtxDeleter {
}
};
-struct ADetailerCtxDeleter {
- void operator()(sd_adetailer_t* ctx) const {
- if (ctx != nullptr) {
- sd_adetailer_free(ctx);
- }
- }
-};
-
template <typename T>
using FreeUniquePtr = std::unique_ptr<T, FreeDeleter>;
-using FilePtr = std::unique_ptr<FILE, FileCloser>;
-using SDCtxPtr = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
-using UpscalerCtxPtr = std::unique_ptr<sd_upscaler_t, UpscalerCtxDeleter>;
-using ADetailerCtxPtr = std::unique_ptr<sd_adetailer_t, ADetailerCtxDeleter>;
+using FilePtr = std::unique_ptr<FILE, FileCloser>;
+using SDCtxPtr = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
+using UpscalerCtxPtr = std::unique_ptr<sd_upscaler_t, UpscalerCtxDeleter>;
struct SDOptDeleter {
void operator()(sd_opt_t* opt) const { sd_opt_free(opt); }
@@ -359,86 +338,13 @@ static inline sd_upscaler_t* create_sd_upscaler(const char* path, std::optional<
return result;
}
-static inline bool upscale_sd_image(sd_upscaler_t* ctx, SDImage image, sd_result_t** result) {
+static inline bool upscale_sd_image(sd_upscaler_t* ctx, SDImage image, sd_result_t** result, int repeats = 1) {
if (result)
*result = nullptr;
SDOptPtr opt(sd_upscale_opt_create());
auto tensor = image_tensor(image);
return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK &&
- sd_upscale(ctx, opt.get(), result) == SD_STATUS_OK;
-}
-
-static inline sd_adetailer_t* create_sd_adetailer(const char* path, std::optional<int> threads, const char* backend, const char* params_backend) {
- SDOptPtr opt(sd_adetailer_opt_create());
- if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK ||
- (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) ||
- (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) ||
- (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK))
- return nullptr;
- sd_adetailer_t* result = nullptr;
- sd_adetailer_create(opt.get(), &result);
- return result;
-}
-
-static inline bool adetail_sd_image(sd_adetailer_t* detector, sd_ctx_t* ctx, SDImage image, SDOptPtr opt, sd_result_t** result) {
- if (result)
- *result = nullptr;
- auto tensor = image_tensor(image);
- return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK &&
- sd_adetail(detector, ctx, opt.get(), result) == SD_STATUS_OK;
-}
-
-struct SDVideoUpscalerParams {
- const char* diffusion_model_path{};
- const char* vae_path{};
- // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints.
- int upscale_factor{};
- std::optional<int> n_threads;
- const char* backend{};
- const char* params_backend{};
- // auto/on/off or diffusion/vae assignments; null selects auto.
- const char* flash_attn_mode{};
- const char* conv_direct_mode{};
-};
-
-struct SDVideoUpscaleParams {
- const SDImage* frames{};
- int frame_count{};
- std::optional<int> fps;
- std::optional<int64_t> seed;
- // RelayVSR reference window, including both endpoints; defaults to 16.
- std::optional<int> window_size;
- // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5.
- std::optional<int> temporal_chunk_size;
-};
-
-static inline sd_vid_upscaler_t* create_sd_vid_upscaler(const SDVideoUpscalerParams& config) {
- SDOptPtr opt(sd_vid_upscaler_opt_create());
- if (!opt || sd_opt_set_string(opt.get(), "diffusion_model_path", config.diffusion_model_path) != SD_STATUS_OK ||
- sd_opt_set_string(opt.get(), "vae_path", config.vae_path) != SD_STATUS_OK ||
- sd_opt_set_int(opt.get(), "upscale_factor", config.upscale_factor) != SD_STATUS_OK ||
- (config.n_threads && sd_opt_set_int(opt.get(), "n_threads", *config.n_threads) != SD_STATUS_OK) ||
- (config.backend && sd_opt_set_string(opt.get(), "backend", config.backend) != SD_STATUS_OK) ||
- (config.params_backend && sd_opt_set_string(opt.get(), "params_backend", config.params_backend) != SD_STATUS_OK) ||
- (config.flash_attn_mode && sd_opt_set_string(opt.get(), "flash_attn_mode", config.flash_attn_mode) != SD_STATUS_OK) ||
- (config.conv_direct_mode && sd_opt_set_string(opt.get(), "conv_direct_mode", config.conv_direct_mode) != SD_STATUS_OK))
- return nullptr;
- sd_vid_upscaler_t* result = nullptr;
- sd_vid_upscaler_create(opt.get(), &result);
- return result;
-}
-
-static inline bool upscale_sd_video(sd_vid_upscaler_t* ctx, const SDVideoUpscaleParams& config, sd_result_t** result) {
- if (result)
- *result = nullptr;
- SDOptBuilder opt(sd_vid_upscale_opt_create());
- opt.set_frames("frames", config.frames, config.frame_count);
- opt.set_int("fps", config.fps);
- opt.set_int("seed", config.seed);
- opt.set_int("window_size", config.window_size);
- opt.set_int("temporal_chunk_size", config.temporal_chunk_size);
- auto params = opt.finish();
- return params && sd_upscale_video(ctx, params.get(), result) == SD_STATUS_OK;
+ sd_opt_set_int(opt.get(), "repeats", repeats) == SD_STATUS_OK && sd_upscale(ctx, opt.get(), result) == SD_STATUS_OK;
}
static inline SDImage take_result_image(sd_result_t* result, size_t index) {
@@ -615,12 +521,12 @@ public:
return index < generation_infos_.size() ? generation_infos_[index] : empty;
}
- void append_generation_info(size_t index, const std::string& info, const std::string& lora_model_dir = {}) {
+ void set_generation_info(size_t index, const std::string& source) {
generation_infos_.resize(images_.size());
- generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir));
+ generation_infos_.at(index) = source;
}
- void adopt(sd_result_t* result, const std::string& lora_model_dir = {}) {
+ void adopt(sd_result_t* result) {
clear();
SDResultPtr owner(result);
if (!result)
@@ -629,13 +535,14 @@ public:
images_.reserve(count);
generation_infos_.reserve(count);
for (size_t i = 0; i < count; ++i) {
- generation_infos_.push_back(generation_info_with_relative_lora_paths(sd_result_get_generation_info(result, i), lora_model_dir));
+ const auto* source = sd_result_get_generation_info(result, i);
+ generation_infos_.emplace_back(source ? source : "");
images_.push_back(take_result_image(result, i));
}
}
- void adopt_video(sd_result_t* result, const std::string& lora_model_dir = {}) {
- adopt(result, lora_model_dir);
+ void adopt_video(sd_result_t* result) {
+ adopt(result);
}
void adopt(SDImage* images, int count) {
diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
index d01f7a7..6246a57 100644
--- a/examples/server/CMakeLists.txt
+++ b/examples/server/CMakeLists.txt
@@ -64,7 +64,6 @@ endif()
add_executable(${TARGET}
../common/common.cpp
- ../common/generation_metadata.cpp
../common/log.cpp
../common/media_io.cpp
../common/media_metadata.cpp
@@ -77,6 +76,7 @@ add_executable(${TARGET}
routes_openai.cpp
routes_sdapi.cpp
routes_sdcpp.cpp
+ routes_dgl.cpp
)
if(APPLE)
sd_set_macos_rpaths(${TARGET})
diff --git a/examples/server/README.md b/examples/server/README.md
index 4de714c..f360694 100644
--- a/examples/server/README.md
+++ b/examples/server/README.md
@@ -129,6 +129,35 @@ To apply a LoRA, open **LoRA**, select a compatible file, and set its
requests use this selector; prompt tags such as `<lora:name:1>` are CLI syntax
and are not parsed by the native server API.
+### DGL editor
+
+Open the Web UI using the local server address and select **DGL** in the header.
+Select a saved script or choose **New**. **Duplicate** saves a copy of the
+current editor text under a new name. Edit **File name** before saving to save
+under a different name. **Run** submits the editor text to the generation queue,
+so you can run unsaved edits. The results panel shows text, scalar and tensor
+information and returned encoded media. PNG appears as an image, WebM as a
+video player, and AVI as a download. WebM encoding is silent; use AVI for
+a PCM16 soundtrack.
+
+Use `image::encode` or `video::encode` and return their `{data, mime_type}`
+dicts, either directly or as named fields of the root dict. To save the same
+encoded output, call `media::save("output/cover.png", cover)` before returning
+it. Saving is optional. The job result contains the same encoded bytes, and
+polling does not encode them again. Tensor frames can be inspected separately
+with **Inspect tensor frames**; they are not fetched automatically.
+
+DGL jobs share an execution context for the lifetime of the server. Compatible
+scripts can reuse loaded model components across runs. The context can release
+idle weights under memory pressure.
+
+Scripts are listed and saved in the server's working directory under
+`dgl/` by default. Use `--dgl-dir` to select another directory. Script paths
+are relative to the server's working directory. Model files and their
+dependencies must reside under one of the configured `--model-dir` roots.
+Write results under `output/` in the working directory. For example,
+`image::save("output/result.png", image)` saves a PNG.
+
## Model management
### Model directories
@@ -251,6 +280,10 @@ Image 2.1 uses the same recipe for its full VAE and TAEQI2.1 alternatives.
### Models loaded at startup
You can pass the same model-loading arguments used by `sd-cli` to `sd-server`.
+Explicit model paths can be outside `--model-dir`. A startup model directory
+includes the dependencies inside that directory. A startup shard index also
+permits the shards it names within the index directory. Native generation accepts
+LoRAs from `--lora-model-dir` and upscalers from `--hires-upscalers-dir`.
For example, run this Z-Image Turbo configuration from the build directory:
**Windows (PowerShell)**
@@ -292,27 +325,11 @@ removed when the command-line context is unloaded or replaced.
#### Identity inputs
-The browser currently has no PhotoMaker identity-image or PuLID identity-embedding
-upload control. Prepare the inputs using the
+PhotoMaker and PuLID use the **DGL editor**. Prepare the inputs using the
[PhotoMaker](../../docs/models/photo_maker.md) or [PuLID](../../docs/models/pulid.md)
-guide, then supply them when starting the server. From the build directory, for example:
-
-**Windows (PowerShell)**
-
-```powershell
-.\bin\Release\sd-server.exe --model-dir ..\models --pm-id-images-dir ..\inputs\identity
-```
-
-**Linux/macOS**
-
-```sh
-./bin/sd-server --model-dir ../models --pm-id-images-dir ../inputs/identity
-```
-
-Select the PhotoMaker recipe and weights in the browser. For PhotoMaker v2, also
-pass `--pm-id-embed-path` with the prepared embedding file. For PuLID, use
-`--pulid-id-embedding` instead of the PhotoMaker input argument, then select the
-PuLID recipe and weights. These startup inputs are inherited by generation requests.
+guide, then set model and identity-input paths in the corresponding script.
+The former identity-input startup arguments have been removed. Scripts read
+their own inputs under the server's file-access rules.
#### Sharded checkpoints
@@ -425,6 +442,7 @@ configured defaults.
| --- | --- | --- |
| `--listen-ip <ip>` | `127.0.0.1` | Server listen address |
| `--listen-port <port>` | `1234` | Server listen port |
+| `--dgl-dir <path>` | `dgl` | Script directory for the local DGL editor |
| `--max-pending-jobs <count>` | `64` | Maximum async jobs queued or generating |
| `--completed-ttl-seconds <seconds>` | `600` | Retention time for completed async jobs |
diff --git a/examples/server/api.md b/examples/server/api.md
index 9f48e24..25aa587 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -93,7 +93,7 @@ available during loading. Model lists are empty when no model is loaded;
| GET | `/sdcpp/v1/model-files` | `{files: [{id, name, category, root, relative_path}], errors: [...]}` |
| POST | `/sdcpp/v1/models/refresh` | Rescan recipes and files; return `{errors: [...]}` |
| GET | `/sdcpp/v1/models/status` | Current state, recipe, selection, error, loading time and optional revision |
-| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion. A load is a no-op returning the current status with `already_loaded: true` only when the state is ready and the fully resolved selection, the complete recipe object, and the resolved component paths all match the active model; a catalog refresh with modified recipe metadata or a retargeted model path therefore reloads even with an unchanged recipe ID and selection |
+| POST | `/sdcpp/v1/models/load` | Prepare and validate the submitted components before reporting `ready`; failures report `model_load_failed` and leave no active context. A load is a no-op returning the current status with `already_loaded: true` only when the state is ready and the fully resolved selection, the complete recipe object, and the resolved component paths all match the active model; a catalog refresh with modified recipe metadata or a retargeted model path therefore reloads even with an unchanged recipe ID and selection |
| POST | `/sdcpp/v1/models/unload` | Release the active context; return empty status |
Load request:
@@ -507,16 +507,10 @@ Native extension fields:
Uploaded images are decoded at their original dimensions. The first decoded
image establishes the generation dimensions if `size` is omitted. Input
-geometry follows `image_preprocess`: references preserve their dimensions by
-default, while init and mask use the generation canvas preset.
+geometry follows the selected model's default input rules.
-Reference encoding then follows model presets and `ref_image_args`. To skip
-input geometry for references and disable resizing before VAE encoding, include
-this in `prompt`:
-
-```text
-edit this image <sd_cpp_extra_args>{"image_preprocess":"target=ref,mode=none","ref_image_args":"resize_before_vae=false"}</sd_cpp_extra_args>
-```
+Reference encoding follows the model defaults. Custom resizing, cropping and
+reference conditioning require a DGL program.
Response fields:
@@ -996,8 +990,6 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
| Field | Type |
| --- | --- |
| `batch_count` | `integer` |
-| `ref_image_args` | `string` |
-| `increase_ref_index` | `boolean` |
| `control_strength` | `number` |
| `control_type` | `string` |
| `ip_adapter_strength` | `number` |
@@ -1018,7 +1010,6 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
| --- | --- |
| `video_frames` | `integer` |
| `fps` | `integer` |
-| `ref_image_args` | `string` |
| `moe_boundary` | `number` |
| `vace_strength` | `number` |
| `high_noise_sample_params` | `object` |
@@ -1177,8 +1168,6 @@ Example:
"strength": 0.75,
"seed": -1,
"batch_count": 1,
- "ref_image_args": "",
- "increase_ref_index": false,
"control_strength": 0.9,
"control_type": "",
"ip_adapter_strength": 1.0,
@@ -1256,16 +1245,13 @@ Example:
### Image Encoding Rules
-Native image/video requests and SDAPI accept `image_preprocess` as a rule string
-or array of rule strings. OpenAI-compatible requests can supply it in
-`sd_cpp_extra_args`. See [Image preprocessing](../../docs/guides/image_preprocessing.md)
-for one-time input geometry, native-resolution decoding, mask alignment, and
-`canny=true` for edge detection on any supported image input.
+Native generation follows model defaults. Custom input geometry and reference
+conditioning use DGL tensor operations. See
+[Image preprocessing](../../docs/guides/image_preprocessing.md) and
+[Image editing](../../docs/guides/edit.md).
-Image generation also accepts `ref_image_args` as a string (for example,
-`"resize_before_vae=false"`) in native and SDAPI requests, or through
-`sd_cpp_extra_args` in OpenAI-compatible requests. It controls downstream
-reference encoding and is independent of input geometry rules.
+The former `image_preprocess` rule field is rejected. Use the DGL execution
+endpoint for custom preprocessing.
Any image field accepts:
@@ -1302,9 +1288,6 @@ Top-level scalar fields:
| `strength` | `number` |
| `seed` | `integer` |
| `batch_count` | `integer` |
-| `ref_image_args` | `string` |
-| `image_preprocess` | `string \| array<string>` |
-| `increase_ref_index` | `boolean` |
| `control_strength` | `number` |
| `control_type` | `string` |
| `ip_adapter_strength` | `number` |
@@ -1518,7 +1501,7 @@ Compared with `img_gen`, the `vid_gen` request body:
- `vid_gen` is a single video sequence job, so `batch_count` is not part of the request schema
- `mask_image`, `control_image`, `control_strength`, `ip_adapter_image`, `ip_adapter_strength`, and `embed_image_metadata` are not part of the request schema
- `ref_images` is accepted for MiniMax-H3 Ref2VA conditioning; other video model families currently ignore it
-- `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `ref_image_args`, `moe_boundary`, and `vace_strength`
+- `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `moe_boundary`, and `vace_strength`
Example:
@@ -1533,7 +1516,6 @@ Example:
"seed": -1,
"video_frames": 33,
"fps": 16,
- "ref_image_args": "",
"moe_boundary": 0.875,
"vace_strength": 1.0,
@@ -1615,7 +1597,6 @@ Example:
- Images retain request order and correspond to `<Picture 1>`, `<Picture 2>`, and so on in the prompt.
- For MiniMax-H3, non-empty `ref_images` cannot be combined with `init_image` or `end_image`. A conflicting API request fails during generation; the WebUI checks this before submission.
- MiniMax-H3 does not support `control_frames`; leave that array empty.
-- `ref_image_args` applies to `vid_gen`; MiniMax-H3 honors `ref_image_size` there (`match` default, `max` for a 2048 px short edge). Only references above the limit are downscaled; each axis is then snapped to the nearest 32 px multiple.
- Other video model families currently ignore `ref_images`.
- `features_by_mode.vid_gen` does not currently advertise `ref_images`; its absence is not an indication that MiniMax-H3 reference images are unsupported.
@@ -1666,7 +1647,6 @@ Top-level scalar fields:
| `seed` | `integer` |
| `video_frames` | `integer` |
| `fps` | `integer` |
-| `ref_image_args` | `string` |
| `moe_boundary` | `number` |
| `vace_strength` | `number` |
| `preview` | `string` |
@@ -1820,19 +1800,21 @@ to the loaded model:
| Model family | Frame alignment | Fixed FPS |
| --- | --- | --- |
-| Wan, HunyuanVideo, LingBot | `4n + 1`, round down | Wan 2.2 S2V: 16 |
-| LTX | `8n + 1`, round down | None |
+| Wan, HunyuanVideo, LingBot | `4n + 1`, round up | Wan 2.2 S2V: 16 |
+| LTX | `8n + 1`, round up | None |
| MiniMax-H3 | `17n + 5`, round up, minimum 5 | 24 |
-| AnimateDiff | Whole frames, maximum 32 | None |
+
+AnimateDiff requires an explicit DGL workflow rather than the native video
+generation API.
Wan examples:
- `video_frames = 33` stays `33`
-- `video_frames = 34` becomes `33`
-- `video_frames = 32` becomes `29`
+- `video_frames = 34` becomes `37`
+- `video_frames = 32` becomes `33`
-At 16 FPS, a 5-second target requests 80 frames and aligns to 77 frames for Wan,
-or approximately 4.813 seconds. Audio-driven generation and temporal upscaling
+At 16 FPS, a 5-second target requests 80 frames and aligns to 81 frames for Wan,
+or approximately 5.063 seconds. Audio-driven generation and temporal upscaling
can change the final length. Use the completed job's actual decoded `frame_count`
and `fps` for the final playback duration.
@@ -1913,13 +1895,63 @@ Example cancelled job:
- `429 Too Many Requests` when the job queue is full
- `500 Internal Server Error` for unexpected server exceptions during submission
+## DGL editor API
+
+These endpoints are available to the local, same-origin Web UI. The server uses
+the working directory's ./dgl directory unless --dgl-dir is set.
+
+| Method and path | Request or response |
+| --- | --- |
+| GET /sdcpp/v1/dgl/scripts | Returns directory and a list of scripts |
+| GET /sdcpp/v1/dgl/script?name=example.dgl | Returns the script as source |
+| POST /sdcpp/v1/dgl/script | Saves UTF-8 source with the given name. Pass previous_source to update an existing script. A conflict returns HTTP 409. |
+| POST /sdcpp/v1/dgl/jobs | Queues the source string and returns a job ID with HTTP 202 |
+| GET /sdcpp/v1/dgl/jobs/{id} | Returns the job status, error or result with tensor descriptions and inline encoded media |
+| GET /sdcpp/v1/dgl/jobs/{id}/outputs/{index}?frame=0 | Returns a PNG of an image output or a video frame. Output and frame indexes start at zero. |
+
+Model reads are restricted to the configured model directories. Script writes
+are restricted to output/ in the server's working directory.
+
+### Encoded DGL results
+
+The existing job result has `type`, `outputs` and `media`. `outputs` retains
+the flat tensor descriptions. `media` recognizes a root `{data, mime_type}`
+dict or such dicts in the root's named fields. `data` must be DGL `bytes`, and
+the supported MIME types are `image/png`, `video/webm` and `video/x-msvideo`.
+Bare bytes, U8 tensors and more deeply nested dicts are not inferred as media.
+The name of a root media dict is `result`.
+
+```json
+{
+ "type": "dict",
+ "outputs": [],
+ "media": [
+ {"name": "cover", "mime_type": "image/png", "b64_json": "..."},
+ {"name": "clip", "mime_type": "video/webm", "b64_json": "..."}
+ ]
+}
+```
+
+The script encodes media once. After execution releases the generation lock,
+the server converts returned bytes to Base64 once, before publishing
+`completed`. Polling returns the stored result; it does not encode a video,
+convert Base64 again, or require frame requests. The frame endpoint remains
+available for explicit tensor inspection. Native `/sdcpp/v1/vid_gen` is unchanged.
+
+Inline media share a 64 MiB raw-byte budget per job. An item that exceeds the
+remaining budget has no `b64_json`; it instead contains
+`error: {code: "inline_too_large", message: "..."}`. Other media and tensor
+outputs remain available, and any file already written by `media::save`
+remains intact. Encoder failures fail the script through the ordinary DGL
+execution error path. WebM encoding rejects nonempty audio instead of silently
+removing it. Use AVI encoding for a PCM16 soundtrack.
+
## Actual generation parameters
-Generation responses expose the core `sdcpp.generation/v1` record as
-`generation_info`, independently of `embed_image_metadata`. Native async image
-result items and OpenAI image items contain the parameter snapshot with each
-image's actual seed and without `batch_count`. Native video
-results contain the video request snapshot. SD API `info.generation_info` is an array
+Generation responses expose the executed DGL source as `generation_info`,
+independently of `embed_image_metadata`. Native async image result items and
+OpenAI image items contain each image's source with its resolved seed. Native
+video results contain the sequence-wide source. SD API `info.generation_info` is an array
aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field
remains the request echo. See [generation metadata](../../docs/guides/generation_metadata.md)
-for field semantics, postprocessing and container storage.
+for source semantics and container storage.
diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp
index bb88fb6..ad1bef0 100644
--- a/examples/server/async_jobs.cpp
+++ b/examples/server/async_jobs.cpp
@@ -4,13 +4,17 @@
#include <algorithm>
#include <chrono>
+#include <cwctype>
#include <iomanip>
+#include <limits>
#include <sstream>
+#include <string_view>
#include <thread>
#include "common/log.h"
#include "common/media_io.h"
#include "common/resource_owners.hpp"
+#include "model_catalog.h"
class JobCancellationMonitor {
std::mutex mutex_;
@@ -118,12 +122,10 @@ static void finish_job_progress(AsyncGenerationJob& job) {
class JobCallbacksGuard {
sd_ctx_t* ctx_;
AsyncGenerationJob& job_;
- bool detailing_ = false;
static void progress_callback(const sd_generation_progress_t* progress, void* data) {
auto& guard = *static_cast<JobCallbacksGuard*>(data);
- update_job_progress(guard.job_, guard.detailing_ ? "adetailer" : progress->stage,
- guard.detailing_ ? progress->stage : progress->detail,
+ update_job_progress(guard.job_, progress->stage, progress->detail,
progress->sample_pass, progress->completed, progress->total, progress->unit);
}
@@ -140,12 +142,6 @@ public:
clear_preview_callback();
}
- void begin_detailing() {
- clear_preview_callback();
- detailing_ = true;
- update_job_progress(job_, "adetailer");
- }
-
JobCallbacksGuard(const JobCallbacksGuard&) = delete;
JobCallbacksGuard& operator=(const JobCallbacksGuard&) = delete;
};
@@ -156,6 +152,8 @@ const char* async_job_kind_name(AsyncJobKind kind) {
return "img_gen";
case AsyncJobKind::VidGen:
return "vid_gen";
+ case AsyncJobKind::Dgl:
+ return "dgl";
default:
return "img_gen";
}
@@ -289,7 +287,9 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
}
if (job.status == AsyncJobStatus::Completed) {
- if (job.kind == AsyncJobKind::VidGen) {
+ if (job.kind == AsyncJobKind::Dgl) {
+ result["result"] = job.dgl_result_info;
+ } else if (job.kind == AsyncJobKind::VidGen) {
result["result"] = {
{"output_format", job.vid_gen.output_format},
{"mime_type", job.result_media_mime_type},
@@ -359,17 +359,10 @@ bool execute_img_gen_job(ServerRuntime& runtime,
PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.img_gen.gen_params);
sd_result_t* generated = nullptr;
generate_sd_image(runtime.sd_ctx, params, &generated);
- results.adopt(generated, runtime.ctx_params->lora_model_dir);
+ results.adopt(generated);
if (job.cancel_requested.load()) {
return false;
}
- if (!results.empty() && job.img_gen.adetailer_enabled && !runtime.default_gen_params->ad_model_path.empty()) {
- callbacks.begin_detailing();
- if (!apply_server_adetailer(runtime, job.img_gen, results)) {
- error_message = "ADetailer failed";
- return false;
- }
- }
}
const int num_results = results.count();
@@ -405,7 +398,7 @@ bool execute_img_gen_job(ServerRuntime& runtime,
continue;
}
output_images.push_back(base64_encode(image_bytes));
- output_info.push_back(json::parse(results.generation_info(i)));
+ output_info.push_back(results.generation_info(i));
update_job_progress(job, "encoding", "image", -1, i + 1, num_results, "images");
}
@@ -445,8 +438,8 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
output_fps = sd_result_get_fps(generated);
generated_audio = retain_result_audio(generated);
- results.adopt_video(generated, runtime.ctx_params->lora_model_dir);
- output_info = json::parse(results.generation_info(0));
+ results.adopt_video(generated);
+ output_info = results.generation_info(0);
if (job.vid_gen.gen_params.embed_image_metadata)
str_params = results.generation_info(0);
}
@@ -479,8 +472,234 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
return true;
}
+int64_t dgl_image_frame_count(const sd_tensor_t* tensor) {
+ if (!tensor || sd_tensor_type(tensor) != SD_TENSOR_U8)
+ return 0;
+ const auto rank = sd_tensor_rank(tensor);
+ if (rank != 3 && rank != 4)
+ return 0;
+ const auto channels = sd_tensor_dim(tensor, 0);
+ const auto width = sd_tensor_dim(tensor, 1);
+ const auto height = sd_tensor_dim(tensor, 2);
+ if (channels < 1 || channels > 4 || width <= 0 || height <= 0 ||
+ sd_tensor_byte_size(tensor) == 0 || sd_tensor_byte_size(tensor) > std::numeric_limits<int>::max())
+ return 0;
+ return rank == 3 ? 1 : sd_tensor_dim(tensor, 3);
+}
+
+static json describe_dgl_media(const sd_dgl_result_t* result) {
+ json media = json::array();
+ size_t remaining = 64 * 1024 * 1024;
+ auto append = [&](const sd_dgl_value_t* value, const std::string& name) {
+ if (sd_dgl_value_kind(value) != SD_DGL_VALUE_DICT)
+ return;
+ const auto* data = sd_dgl_value_field(value, "data");
+ const auto* mime = sd_dgl_value_field(value, "mime_type");
+ if (sd_dgl_value_kind(data) != SD_DGL_VALUE_BYTES || sd_dgl_value_kind(mime) != SD_DGL_VALUE_STRING)
+ return;
+ const char* mime_text = nullptr;
+ size_t mime_size = 0;
+ const void* bytes = nullptr;
+ size_t size = 0;
+ if (sd_dgl_value_get_string(mime, &mime_text, &mime_size) != SD_STATUS_OK ||
+ sd_dgl_value_get_bytes(data, &bytes, &size) != SD_STATUS_OK)
+ throw std::runtime_error(sd_get_last_error());
+ const std::string mime_type(mime_text, mime_size);
+ if (mime_type != "image/png" && mime_type != "video/webm" && mime_type != "video/x-msvideo")
+ return;
+ json item = {{"name", name}, {"mime_type", mime_type}};
+ if (size > remaining) {
+ item["error"] = {{"code", "inline_too_large"}, {"message", "DGL media exceeds the 64 MiB inline result budget; use media::save"}};
+ } else {
+ item["b64_json"] = base64_encode(bytes, size);
+ remaining -= size;
+ }
+ media.push_back(std::move(item));
+ };
+ const auto* root = sd_dgl_result_root(result);
+ if (sd_dgl_value_kind(root) == SD_DGL_VALUE_DICT) {
+ append(root, "result");
+ if (media.empty())
+ for (size_t i = 0; i < sd_dgl_value_count(root); ++i) {
+ const auto* value = sd_dgl_value_at(root, i);
+ if (const auto* name = sd_dgl_value_name(value))
+ append(value, name);
+ }
+ }
+ return media;
+}
+
+static json describe_dgl_result(const sd_dgl_result_t* result) {
+ const std::string type = sd_dgl_result_type(result);
+ json outputs = json::array();
+ for (size_t i = 0; i < sd_dgl_result_count(result); ++i) {
+ const auto* name = sd_dgl_result_name(result, i);
+ const auto* tensor = sd_dgl_result_tensor(result, name);
+ const auto dtype = sd_tensor_type(tensor);
+ const auto bytes = sd_tensor_byte_size(tensor);
+ std::vector<int64_t> shape;
+ for (size_t axis = 0; axis < sd_tensor_rank(tensor); ++axis)
+ shape.push_back(sd_tensor_dim(tensor, axis));
+ json output = {{"name", name}, {"dtype", dtype}, {"shape", shape}, {"bytes", bytes}, {"frames", dgl_image_frame_count(tensor)}};
+ const auto* data = sd_tensor_data(tensor);
+ if (type == "string" && std::string_view(name) == "result") {
+ output["value"] = bytes ? std::string(static_cast<const char*>(data), bytes) : "";
+ } else if (shape.empty() && bytes != 0) {
+ switch (dtype) {
+ case SD_TENSOR_U8:
+ output["value"] = *static_cast<const uint8_t*>(data);
+ break;
+ case SD_TENSOR_I32:
+ output["value"] = *static_cast<const int32_t*>(data);
+ break;
+ case SD_TENSOR_I64:
+ output["value"] = std::to_string(*static_cast<const int64_t*>(data));
+ break;
+ case SD_TENSOR_F32:
+ output["value"] = *static_cast<const float*>(data);
+ break;
+ case SD_TENSOR_F64:
+ output["value"] = *static_cast<const double*>(data);
+ break;
+ }
+ }
+ outputs.push_back(std::move(output));
+ }
+ return {{"type", type}, {"outputs", std::move(outputs)}, {"media", describe_dgl_media(result)}};
+}
+
+namespace {
+ namespace fs = std::filesystem;
+
+ struct DglExecution {
+ ServerFileAccess file_access;
+ std::unique_ptr<sd_dgl_ctx_t, decltype(&sd_dgl_ctx_free)> context{nullptr, sd_dgl_ctx_free};
+ };
+
+ bool allowed_dgl_path(const char* path, sd_dgl_file_access_t access, void* userdata) try {
+ const auto& policy = *static_cast<const ServerFileAccess*>(userdata);
+ std::error_code error;
+ const auto file = fs::u8path(path);
+ if (file.empty())
+ return false;
+#ifdef _WIN32
+ if (file.native().compare(0, 4, L"\\\\?\\") == 0 || file.native().compare(0, 4, L"\\\\.\\") == 0)
+ return false;
+ for (const auto& part : file.relative_path()) {
+ const auto name = part.native();
+ if (name.empty() || name == L"." || name == L"..")
+ continue;
+ if (name.back() == L'.' || name.back() == L' ' || name.find(L':') != std::wstring::npos)
+ return false;
+ auto device = name.substr(0, name.find(L'.'));
+ while (!device.empty() && device.back() == L' ')
+ device.pop_back();
+ std::transform(device.begin(), device.end(), device.begin(), [](wchar_t ch) { return std::towupper(ch); });
+ if (device == L"CON" || device == L"PRN" || device == L"AUX" || device == L"NUL" ||
+ device == L"CONIN$" || device == L"CONOUT$" ||
+ (device.size() == 4 && (device.compare(0, 3, L"COM") == 0 || device.compare(0, 3, L"LPT") == 0) &&
+ std::wstring_view(L"123456789\u00b9\u00b2\u00b3").find(device.back()) != std::wstring_view::npos))
+ return false;
+ }
+#endif
+ const auto absolute = fs::absolute(file, error);
+ if (error)
+ return false;
+ const auto resolved = fs::weakly_canonical(absolute, error);
+ if (error)
+ return false;
+ if (access == SD_DGL_FILE_WRITE) {
+ if (!model_path_inside(policy.output, resolved))
+ return false;
+ for (auto part = absolute; part != part.root_path(); part = part.parent_path()) {
+ const auto status = fs::symlink_status(part, error);
+ if (fs::is_symlink(status) || (error && status.type() != fs::file_type::not_found))
+ return false;
+ error.clear();
+ }
+ const auto status = fs::symlink_status(resolved, error);
+ if (status.type() != fs::file_type::not_found) {
+ return !error && fs::is_regular_file(status) && fs::hard_link_count(resolved, error) == 1 && !error;
+ }
+ error.clear();
+ fs::create_directories(resolved.parent_path(), error);
+ return !error;
+ }
+ const auto status = fs::status(resolved, error);
+ const bool readable = !error && (fs::is_regular_file(status) || fs::is_directory(status));
+ if (access == SD_DGL_FILE_READ)
+ return readable;
+ const auto key = model_path_key(resolved);
+ if (readable || status.type() == fs::file_type::not_found) {
+ if (policy.configured_files.count(key))
+ return true;
+ if (std::any_of(policy.model_bundles.begin(), policy.model_bundles.end(), [&](const fs::path& root) { return model_path_inside(root, resolved); }))
+ return true;
+ }
+ return readable && std::any_of(policy.models.begin(), policy.models.end(), [&](const ModelRoot& root) {
+ return model_path_key(root.path) == key || model_path_inside(root.path, resolved);
+ });
+ } catch (const std::exception&) {
+ return false;
+ }
+}
+
+void prepare_native_context(ServerModels& models, const SDSvrParams& params, const SDContextParams& config, bool startup) {
+ auto& policy = models.file_access;
+ policy = {get_model_roots(params), fs::weakly_canonical(fs::current_path()) / "output", {}, {}};
+ for (const auto* directory : {&config.lora_model_dir, &config.hires_upscalers_dir})
+ if (!directory->empty())
+ policy.models.push_back(make_model_root(fs::u8path(*directory), false));
+ for (const auto& entry : config.embedding_map)
+ policy.configured_files.insert(model_path_key(fs::weakly_canonical(fs::absolute(fs::u8path(entry.second)))));
+ if (startup)
+ add_startup_model_sources(policy, config);
+
+ if (sd_ctx_set_file_access_callback(models.context.get(), allowed_dgl_path, &policy) != SD_STATUS_OK)
+ throw std::runtime_error(sd_get_last_error());
+ if (sd_ctx_prepare(models.context.get()) != SD_STATUS_OK)
+ throw std::runtime_error(sd_get_last_error());
+}
+
+static bool execute_dgl_job(ServerRuntime& runtime,
+ DglExecution& execution,
+ AsyncGenerationJob& job,
+ std::shared_ptr<sd_dgl_result_t>& result,
+ json& result_info,
+ std::string& error_message) {
+ try {
+ auto source = std::move(job.dgl_source);
+ execution.file_access = {get_model_roots(*runtime.svr_params), fs::canonical(fs::current_path()) / "output", {}, {}};
+ if (!execution.context) {
+ std::unique_ptr<sd_dgl_ctx_t, decltype(&sd_dgl_ctx_free)> context(sd_dgl_ctx_create(), sd_dgl_ctx_free);
+ if (!context || !runtime.ctx_params->configure_dgl(context.get()) ||
+ !sd_dgl_ctx_set_file_access_callback(context.get(), allowed_dgl_path, &execution.file_access)) {
+ error_message = sd_dgl_error();
+ return false;
+ }
+ execution.context = std::move(context);
+ }
+ sd_dgl_result_t* raw_result = nullptr;
+ {
+ std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+ update_job_progress(job, "executing", "dgl");
+ if (sd_dgl_run(execution.context.get(), source.c_str(), nullptr, 0, &raw_result) != SD_STATUS_OK) {
+ error_message = sd_dgl_error();
+ return false;
+ }
+ }
+ result.reset(raw_result, sd_dgl_result_free);
+ result_info = describe_dgl_result(result.get());
+ return true;
+ } catch (const std::exception& error) {
+ error_message = error.what();
+ return false;
+ }
+}
+
void async_job_worker(ServerRuntime& runtime) {
AsyncJobManager& manager = *runtime.async_job_manager;
+ DglExecution dgl;
while (true) {
std::shared_ptr<AsyncGenerationJob> job;
@@ -517,6 +736,8 @@ void async_job_worker(ServerRuntime& runtime) {
std::string output_media_mime_type;
int output_frame_count = 0;
int output_fps = 0;
+ std::shared_ptr<sd_dgl_result_t> dgl_result;
+ json dgl_result_info;
std::string error_message;
bool ok = false;
@@ -531,6 +752,8 @@ void async_job_worker(ServerRuntime& runtime) {
output_fps,
output_info,
error_message);
+ } else if (job->kind == AsyncJobKind::Dgl) {
+ ok = execute_dgl_job(runtime, dgl, *job, dgl_result, dgl_result_info, error_message);
} else {
error_message = "unsupported job kind";
}
@@ -556,6 +779,8 @@ void async_job_worker(ServerRuntime& runtime) {
job->result_media_mime_type = std::move(output_media_mime_type);
job->result_frame_count = output_frame_count;
job->result_fps = output_fps;
+ job->dgl_result = std::move(dgl_result);
+ job->dgl_result_info = std::move(dgl_result_info);
job->error_code.clear();
job->error_message.clear();
} else {
diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h
index a442305..7cade97 100644
--- a/examples/server/async_jobs.h
+++ b/examples/server/async_jobs.h
@@ -17,6 +17,7 @@
enum class AsyncJobKind {
ImgGen,
VidGen,
+ Dgl,
};
enum class AsyncJobStatus {
@@ -51,6 +52,9 @@ struct AsyncGenerationJob {
int64_t completed_at = 0;
ImgGenJobRequest img_gen;
VidGenJobRequest vid_gen;
+ std::string dgl_source;
+ std::shared_ptr<sd_dgl_result_t> dgl_result;
+ json dgl_result_info;
std::vector<std::string> result_images_b64;
json result_generation_info;
std::string result_media_b64;
@@ -86,6 +90,7 @@ size_t count_pending_jobs(const AsyncJobManager& manager);
std::string make_async_job_id(AsyncJobManager& manager);
bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job);
json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job);
+int64_t dgl_image_frame_count(const sd_tensor_t* tensor);
bool execute_img_gen_job(ServerRuntime& runtime,
AsyncGenerationJob& job,
std::vector<std::string>& output_images,
diff --git a/examples/server/frontend b/examples/server/frontend
index b07665f..0166e07 160000
--- a/examples/server/frontend
+++ b/examples/server/frontend
@@ -1 +1 @@
-Subproject commit b07665f893047e7b633bbdb69d97c3a9c594076e
+Subproject commit 0166e07b6468f7cb8e1fe81b2de997869ec56b16
diff --git a/examples/server/main.cpp b/examples/server/main.cpp
index 391a1ec..af554ae 100644
--- a/examples/server/main.cpp
+++ b/examples/server/main.cpp
@@ -87,11 +87,24 @@ int main(int argc, const char** argv) {
LOG_DEBUG("%s", ctx_params.to_string().c_str());
LOG_DEBUG("%s", default_gen_params.to_string().c_str());
+ if (!ctx_params.register_rpc_devices()) {
+ return 1;
+ }
+ ctx_params.rpc_servers.reset();
+
SDOptPtr sd_ctx_params = ctx_params.to_context_opt();
ServerModels models;
if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) {
const auto started = std::chrono::steady_clock::now();
models.context.reset(create_sd_context(sd_ctx_params));
+ if (models.context) {
+ try {
+ prepare_native_context(models, svr_params, ctx_params, true);
+ } catch (const std::exception& error) {
+ LOG_ERROR("Model preparation failed: %s", error.what());
+ models.context.reset();
+ }
+ }
models.status["load_elapsed_seconds"] = std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count();
}
@@ -163,6 +176,7 @@ int main(int argc, const char** argv) {
register_openai_api_endpoints(svr, runtime);
register_sdapi_endpoints(svr, runtime);
register_sdcpp_api_endpoints(svr, runtime);
+ register_dgl_endpoints(svr, runtime);
register_model_endpoints(svr, runtime);
LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp
index 5a538fd..cbfc79d 100644
--- a/examples/server/model_catalog.cpp
+++ b/examples/server/model_catalog.cpp
@@ -132,6 +132,56 @@ std::vector<ModelRoot> get_model_roots(const SDSvrParams& params) {
return roots;
}
+static void add_startup_index_shards(ServerFileAccess& policy, const fs::path& index_path) {
+ std::error_code error;
+ if (!fs::is_regular_file(index_path, error) || error)
+ return;
+ std::ifstream file(index_path);
+ const auto index = json::parse(file, nullptr, false);
+ const auto shards = index.find("weight_map");
+ if (shards == index.end() || !shards->is_object())
+ return;
+ const auto root = fs::weakly_canonical(index_path.parent_path());
+ for (const auto& entry : *shards) {
+ if (!entry.is_string())
+ continue;
+ const auto path = fs::weakly_canonical((index_path.parent_path() / fs::u8path(entry.get_ref<const std::string&>())).lexically_normal(), error);
+ if (!error && model_path_inside(root, path))
+ policy.configured_files.insert(model_path_key(path));
+ }
+}
+
+void add_startup_model_sources(ServerFileAccess& policy, const SDContextParams& config) {
+ for (const auto& slot : k_model_slots) {
+ const auto& source = config.*slot.field;
+ if (source.empty() || slot.field == &SDContextParams::tokenizer)
+ continue;
+ const auto path = fs::absolute(fs::u8path(source));
+ const auto resolved = fs::weakly_canonical(path);
+ policy.configured_files.insert(model_path_key(resolved));
+ if (fs::is_directory(resolved))
+ policy.model_bundles.push_back(resolved);
+ else if (path.extension() == ".json")
+ add_startup_index_shards(policy, path);
+ }
+
+ auto add_tokenizer = [&](const std::string& path) {
+ if (!path.empty())
+ policy.configured_files.insert(model_path_key(fs::weakly_canonical(fs::absolute(fs::u8path(path)))));
+ };
+ if (config.tokenizer.find('=') == std::string::npos) {
+ add_tokenizer(config.tokenizer);
+ } else {
+ std::istringstream assignments(config.tokenizer);
+ std::string assignment;
+ while (std::getline(assignments, assignment, ',')) {
+ const auto equal = assignment.find('=');
+ if (equal != std::string::npos)
+ add_tokenizer(assignment.substr(equal + 1));
+ }
+ }
+}
+
std::vector<fs::path> model_category_directories(const ModelRoot& root, const std::string& category) {
std::vector<fs::path> directories;
if (!fs::exists(root.path))
@@ -370,6 +420,8 @@ json resolve_recipe_features(const json& recipe) {
features[name.get<std::string>()] = true;
for (const auto& name : recipe.value("disabled_features", json::array()))
features[name.get<std::string>()] = false;
+ for (const char* name : {"ip_adapter", "slg", "shifted_timestep"})
+ features[name] = false;
return features;
}
diff --git a/examples/server/model_catalog.h b/examples/server/model_catalog.h
index 2e45fd1..d3488e5 100644
--- a/examples/server/model_catalog.h
+++ b/examples/server/model_catalog.h
@@ -4,6 +4,7 @@
#include <chrono>
#include <map>
#include <mutex>
+#include <set>
#include <shared_mutex>
#include <string>
@@ -29,10 +30,18 @@ ModelRoot make_model_root(const std::filesystem::path& path, bool primary);
std::vector<ModelRoot> get_model_roots(const SDSvrParams& params);
std::vector<std::filesystem::path> model_category_directories(const ModelRoot& root, const std::string& category);
+struct ServerFileAccess {
+ std::vector<ModelRoot> models;
+ std::filesystem::path output;
+ std::vector<std::filesystem::path> model_bundles;
+ std::set<std::string> configured_files;
+};
+
struct ServerModels {
std::shared_mutex runtime_mutex;
std::mutex catalog_mutex;
std::mutex status_mutex;
+ ServerFileAccess file_access;
SDCtxPtr context;
SDContextParams base_context;
SDGenerationParams base_defaults;
@@ -60,5 +69,6 @@ void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& con
// stay stable if a model directory symlink is retargeted inside the configured
// root, so the idempotent-load check compares the resolved paths too.
bool component_paths_match(const SDContextParams& a, const SDContextParams& b);
+void add_startup_model_sources(ServerFileAccess& policy, const SDContextParams& config);
#endif
diff --git a/examples/server/recipes/README.md b/examples/server/recipes/README.md
index 9405fdb..6f2c744 100644
--- a/examples/server/recipes/README.md
+++ b/examples/server/recipes/README.md
@@ -81,11 +81,11 @@ for supported fields.
## Output dimension alignment
Set `size_multiple` when the runtime's output dimension alignment differs from
-the default of `8`. Match `GenerationRequest::align_image_size` in
-[`src/pipeline/request.cpp`](../../../src/pipeline/request.cpp): the VAE scale
+the default of `8`. Match the aligned dimensions in
+[`generation_program.cpp`](../../../src/api/generation_program.cpp): the VAE scale
factor multiplied by the model definition's `diffusion_down_factor`. The scale
comes from [`VAE::get_scale_factor`](../../../src/model/vae/vae.hpp), with the
-PiD override in [`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp);
+PiD override in [`geometry.hpp`](../../../src/model/vae/geometry.hpp);
model geometry is defined under [`src/model/integrations`](../../../src/model/integrations/).
Use the runtime's output alignment, not the nominal patch size: some models pad
@@ -96,13 +96,12 @@ applies. Reference-image preprocessing constraints are independent of this field
## Video length constraints
-Video recipes declare `video_constraints` to match `align_video_frames` in
-[`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp) and the FPS
-and AnimateDiff overrides in
-[`request.cpp`](../../../src/pipeline/request.cpp). Wan, HunyuanVideo and LingBot
-round down to `4n + 1`; LTX rounds down to `8n + 1`. MiniMax-H3 rounds up to
-`17n + 5`, with at least 5 frames and fixed 24 FPS. AnimateDiff accepts whole
-frames up to 32. Wan 2.2 S2V fixes FPS at 16.
+Video recipes declare `video_constraints` to match `aligned_frames` in
+[`geometry.hpp`](../../../src/model/vae/geometry.hpp) and the FPS policy in
+[`generation_program.cpp`](../../../src/api/generation_program.cpp). Wan, HunyuanVideo and LingBot
+round up to `4n + 1`; LTX rounds up to `8n + 1`. MiniMax-H3 rounds up to
+`17n + 5`, with at least 5 frames and fixed 24 FPS. AnimateDiff requires an
+explicit DGL workflow. Wan 2.2 S2V fixes FPS at 16.
Declare fixed FPS only where the runtime enforces it; recommended FPS belongs
in `defaults.fps`. These constraints guide UI conversion and estimates, not
diff --git a/examples/server/recipes/anima-edit.json b/examples/server/recipes/anima-edit.json
index f22c1b6..6918b97 100644
--- a/examples/server/recipes/anima-edit.json
+++ b/examples/server/recipes/anima-edit.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"id": "anima-edit",
"name": "Anima Edit (LoRA)",
- "description": "Requires a matching community editing LoRA. Apply it after loading and supply reference images; uses the cosmos_reference preset.",
+ "description": "Select a matching community editing LoRA separately and supply reference images. References are VAE-encoded for diffusion conditioning; the recipe does not load the LoRA automatically.",
"model_url": "https://huggingface.co/circlestone-labs/Anima",
"slots": [
{
diff --git a/examples/server/recipes/animatediff-sd15.json b/examples/server/recipes/animatediff-sd15.json
deleted file mode 100644
index e7bf07d..0000000
--- a/examples/server/recipes/animatediff-sd15.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "schema_version": 1,
- "id": "animatediff-sd15",
- "name": "AnimateDiff SD 1.5",
- "size_multiple": 64,
- "description": "Use Video Generation with an SD 1.5 checkpoint and compatible motion module.",
- "model_url": "https://huggingface.co/guoyww/animatediff",
- "slots": [
- {
- "id": "model",
- "required": true
- },
- {
- "id": "motion_module",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "video_constraints": {
- "frame_step": 1,
- "frame_offset": 0,
- "frame_rounding": "floor",
- "min_frames": 1,
- "max_frames": 32
- },
- "defaults": {
- "sampling_method": "euler",
- "video_frames": 8,
- "fps": 8,
- "cfg_scale": 7.5,
- "steps": 20,
- "width": 512,
- "height": 512
- },
- "features": [
- "cache_unet",
- "clip_skip",
- "init_image"
- ],
- "disabled_features": [
- "cache_dit",
- "img2img",
- "mask_image"
- ]
-}
diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md
index fdccc96..47434e6 100644
--- a/examples/server/recipes/defaults.md
+++ b/examples/server/recipes/defaults.md
@@ -75,7 +75,6 @@ For community edit adapters and other optional extensions, these are the base-mo
| `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) |
| `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT) |
| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/models/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. |
-| `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion) |
| `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B) |
| `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
| `pid-15` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
diff --git a/examples/server/recipes/flux1-pulid.json b/examples/server/recipes/flux1-pulid.json
deleted file mode 100644
index 8203dd1..0000000
--- a/examples/server/recipes/flux1-pulid.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
- "schema_version": 1,
- "id": "flux1-pulid",
- "name": "FLUX.1 dev PuLID",
- "description": "Supply a precomputed PuLID identity embedding with the generation request.",
- "model_url": "https://huggingface.co/guozinan/PuLID",
- "slots": [
- {
- "id": "diffusion_model",
- "required": true
- },
- {
- "id": "clip_l",
- "required": true
- },
- {
- "id": "t5xxl",
- "required": true
- },
- {
- "id": "pulid_weights",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "required_any": [
- [
- "vae",
- "tae"
- ]
- ],
- "defaults": {
- "sampling_method": "euler",
- "cfg_scale": 1,
- "steps": 20,
- "width": 1024,
- "height": 1024,
- "distilled_guidance": 3.5
- },
- "features": [
- "clip_skip",
- "distilled_guidance",
- "flow_shift",
- "slg"
- ],
- "disabled_features": [
- "slg"
- ]
-}
diff --git a/examples/server/recipes/hunyuan-video-15-i2v.json b/examples/server/recipes/hunyuan-video-15-i2v.json
index 5dc0509..8fd1e34 100644
--- a/examples/server/recipes/hunyuan-video-15-i2v.json
+++ b/examples/server/recipes/hunyuan-video-15-i2v.json
@@ -40,7 +40,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/hunyuan-video-15-t2v.json b/examples/server/recipes/hunyuan-video-15-t2v.json
index 81aacee..d7a69f7 100644
--- a/examples/server/recipes/hunyuan-video-15-t2v.json
+++ b/examples/server/recipes/hunyuan-video-15-t2v.json
@@ -40,7 +40,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/krea2-raw-edit.json b/examples/server/recipes/krea2-raw-edit.json
index 8d3eb9c..80a61cb 100644
--- a/examples/server/recipes/krea2-raw-edit.json
+++ b/examples/server/recipes/krea2-raw-edit.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"id": "krea2-raw-edit",
"name": "Krea 2 Raw Edit (LoRA)",
- "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.",
+ "description": "Select a matching Ostris-compatible community editing LoRA separately and supply reference images. References go through both the vision encoder and VAE; the recipe does not load the LoRA automatically. Select a separate vision projector when it is not bundled in the text encoder.",
"model_url": "https://huggingface.co/krea/Krea-2-Raw",
"slots": [
{
diff --git a/examples/server/recipes/krea2-turbo-edit.json b/examples/server/recipes/krea2-turbo-edit.json
index e8c521d..f26eb97 100644
--- a/examples/server/recipes/krea2-turbo-edit.json
+++ b/examples/server/recipes/krea2-turbo-edit.json
@@ -2,7 +2,7 @@
"schema_version": 1,
"id": "krea2-turbo-edit",
"name": "Krea 2 Turbo Edit (LoRA)",
- "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.",
+ "description": "Select a matching Ostris-compatible community editing LoRA separately and supply reference images. References go through both the vision encoder and VAE; the recipe does not load the LoRA automatically. Select a separate vision projector when it is not bundled in the text encoder.",
"model_url": "https://huggingface.co/krea/Krea-2-Turbo",
"slots": [
{
diff --git a/examples/server/recipes/lingbot-video-dense.json b/examples/server/recipes/lingbot-video-dense.json
index 3affb48..5fbb208 100644
--- a/examples/server/recipes/lingbot-video-dense.json
+++ b/examples/server/recipes/lingbot-video-dense.json
@@ -34,7 +34,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/lingbot-video-moe.json b/examples/server/recipes/lingbot-video-moe.json
index 55bf59f..10f65d8 100644
--- a/examples/server/recipes/lingbot-video-moe.json
+++ b/examples/server/recipes/lingbot-video-moe.json
@@ -34,7 +34,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/ltx23-dev.json b/examples/server/recipes/ltx23-dev.json
index f0bf3f2..d18854a 100644
--- a/examples/server/recipes/ltx23-dev.json
+++ b/examples/server/recipes/ltx23-dev.json
@@ -43,7 +43,7 @@
"video_constraints": {
"frame_step": 8,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/ltx23-distilled.json b/examples/server/recipes/ltx23-distilled.json
index 3e4d08e..d512a1c 100644
--- a/examples/server/recipes/ltx23-distilled.json
+++ b/examples/server/recipes/ltx23-distilled.json
@@ -43,7 +43,7 @@
"video_constraints": {
"frame_step": 8,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/ltx25-dev.json b/examples/server/recipes/ltx25-dev.json
index 011a49e..ce14efc 100644
--- a/examples/server/recipes/ltx25-dev.json
+++ b/examples/server/recipes/ltx25-dev.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 8,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/nitrosd-realism.json b/examples/server/recipes/nitrosd-realism.json
deleted file mode 100644
index ae7345d..0000000
--- a/examples/server/recipes/nitrosd-realism.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "schema_version": 1,
- "id": "nitrosd-realism",
- "name": "NitroSD Realism",
- "size_multiple": 64,
- "model_url": "https://huggingface.co/ChenDY/NitroFusion",
- "slots": [
- {
- "id": "model",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "defaults": {
- "scheduler": "sgm_uniform",
- "shifted_timestep": 250,
- "cfg_scale": 1,
- "steps": 1,
- "width": 1024,
- "height": 1024
- },
- "features": [
- "cache_unet",
- "clip_skip",
- "shifted_timestep"
- ],
- "disabled_features": [
- "cache_dit"
- ]
-}
diff --git a/examples/server/recipes/sd1-ip-adapter.json b/examples/server/recipes/sd1-ip-adapter.json
deleted file mode 100644
index 7e14c15..0000000
--- a/examples/server/recipes/sd1-ip-adapter.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "schema_version": 1,
- "id": "sd1-ip-adapter",
- "name": "SD 1.5 IP-Adapter / Plus",
- "size_multiple": 64,
- "description": "Select matching SD 1.5 IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.",
- "model_url": "https://huggingface.co/h94/IP-Adapter",
- "slots": [
- {
- "id": "model",
- "required": true
- },
- {
- "id": "clip_vision",
- "required": true,
- "label": "CLIP Vision H"
- },
- {
- "id": "ip_adapter",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "defaults": {
- "cfg_scale": 7.5,
- "steps": 20,
- "width": 512,
- "height": 512
- },
- "features": [
- "cache_unet",
- "clip_skip"
- ],
- "disabled_features": [
- "cache_dit"
- ]
-}
diff --git a/examples/server/recipes/sdxl-ip-adapter.json b/examples/server/recipes/sdxl-ip-adapter.json
deleted file mode 100644
index f0bc8cd..0000000
--- a/examples/server/recipes/sdxl-ip-adapter.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "schema_version": 1,
- "id": "sdxl-ip-adapter",
- "name": "SDXL IP-Adapter / Plus",
- "size_multiple": 64,
- "description": "Select matching SDXL ViT-H IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.",
- "model_url": "https://huggingface.co/h94/IP-Adapter",
- "slots": [
- {
- "id": "model",
- "required": true
- },
- {
- "id": "clip_vision",
- "required": true,
- "label": "CLIP Vision H"
- },
- {
- "id": "ip_adapter",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "defaults": {
- "scheduler": "discrete",
- "sampling_method": "dpm++2m",
- "cfg_scale": 5,
- "steps": 20,
- "width": 1024,
- "height": 1024
- },
- "features": [
- "cache_unet",
- "clip_skip",
- "shifted_timestep"
- ],
- "disabled_features": [
- "cache_dit"
- ]
-}
diff --git a/examples/server/recipes/sdxl-photomaker.json b/examples/server/recipes/sdxl-photomaker.json
deleted file mode 100644
index c8b5790..0000000
--- a/examples/server/recipes/sdxl-photomaker.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "schema_version": 1,
- "id": "sdxl-photomaker",
- "name": "SDXL PhotoMaker",
- "size_multiple": 64,
- "description": "Select PhotoMaker v1 and supply identity images. Include the img trigger in the prompt.",
- "model_url": "https://huggingface.co/TencentARC/PhotoMaker",
- "slots": [
- {
- "id": "model",
- "required": true
- },
- {
- "id": "photo_maker",
- "required": true
- },
- {
- "id": "vae",
- "required": false
- },
- {
- "id": "tae",
- "required": false
- }
- ],
- "defaults": {
- "width": 1024,
- "height": 1024
- },
- "features": [
- "cache_unet",
- "clip_skip",
- "shifted_timestep"
- ],
- "disabled_features": [
- "cache_dit"
- ]
-}
diff --git a/examples/server/recipes/wan21-flf2v.json b/examples/server/recipes/wan21-flf2v.json
index 6bb4c33..23c4201 100644
--- a/examples/server/recipes/wan21-flf2v.json
+++ b/examples/server/recipes/wan21-flf2v.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/wan21-i2v.json b/examples/server/recipes/wan21-i2v.json
index 32d5fc7..88b3597 100644
--- a/examples/server/recipes/wan21-i2v.json
+++ b/examples/server/recipes/wan21-i2v.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/wan21-t2v.json b/examples/server/recipes/wan21-t2v.json
index 998078c..81a487a 100644
--- a/examples/server/recipes/wan21-t2v.json
+++ b/examples/server/recipes/wan21-t2v.json
@@ -34,7 +34,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/wan21-vace.json b/examples/server/recipes/wan21-vace.json
index d7f8343..3321752 100644
--- a/examples/server/recipes/wan21-vace.json
+++ b/examples/server/recipes/wan21-vace.json
@@ -34,7 +34,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/recipes/wan22-i2v.json b/examples/server/recipes/wan22-i2v.json
index c7ab2ed..82ab7e8 100644
--- a/examples/server/recipes/wan22-i2v.json
+++ b/examples/server/recipes/wan22-i2v.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
@@ -48,9 +48,6 @@
"fps": 16,
"flow_shift": 5,
"high_noise_steps": -1,
- "high_noise_cfg_scale": 3.5,
- "high_noise_sampling_method": "euler",
- "high_noise_flow_shift": 5,
"cfg_scale": 3.5,
"steps": 20,
"width": 1280,
diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json
index 641ed6e..fd43061 100644
--- a/examples/server/recipes/wan22-s2v.json
+++ b/examples/server/recipes/wan22-s2v.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1,
"fixed_fps": 16
},
diff --git a/examples/server/recipes/wan22-t2v.json b/examples/server/recipes/wan22-t2v.json
index fba962f..9f7fc95 100644
--- a/examples/server/recipes/wan22-t2v.json
+++ b/examples/server/recipes/wan22-t2v.json
@@ -39,7 +39,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
@@ -48,9 +48,6 @@
"fps": 16,
"flow_shift": 12,
"high_noise_steps": -1,
- "high_noise_cfg_scale": 4,
- "high_noise_sampling_method": "euler",
- "high_noise_flow_shift": 12,
"cfg_scale": 3,
"steps": 20,
"width": 1280,
diff --git a/examples/server/recipes/wan22-ti2v-5b.json b/examples/server/recipes/wan22-ti2v-5b.json
index bc43361..4bcd9f0 100644
--- a/examples/server/recipes/wan22-ti2v-5b.json
+++ b/examples/server/recipes/wan22-ti2v-5b.json
@@ -34,7 +34,7 @@
"video_constraints": {
"frame_step": 4,
"frame_offset": 1,
- "frame_rounding": "floor",
+ "frame_rounding": "ceil",
"min_frames": 1
},
"defaults": {
diff --git a/examples/server/routes.h b/examples/server/routes.h
index f779391..a3093e4 100644
--- a/examples/server/routes.h
+++ b/examples/server/routes.h
@@ -9,6 +9,7 @@ void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_param
void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt);
void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
+void register_dgl_endpoints(httplib::Server& svr, ServerRuntime& rt);
void register_model_endpoints(httplib::Server& svr, ServerRuntime& rt);
class RuntimeRoutes {
diff --git a/examples/server/routes_dgl.cpp b/examples/server/routes_dgl.cpp
new file mode 100644
index 0000000..8fd0a3a
--- /dev/null
+++ b/examples/server/routes_dgl.cpp
@@ -0,0 +1,280 @@
+#include "routes.h"
+
+#include <algorithm>
+#include <charconv>
+#include <fstream>
+#include <optional>
+#include <stdexcept>
+#include <string_view>
+
+#include "async_jobs.h"
+#include "common/media_io.h"
+
+namespace {
+ namespace fs = std::filesystem;
+ using Request = httplib::Request;
+ using Response = httplib::Response;
+
+ struct DglWorkspace {
+ fs::path directory;
+ std::mutex mutex;
+
+ explicit DglWorkspace(const std::string& path) : directory(fs::u8path(path)) {}
+ };
+
+ void send_json(Response& res, int status, const json& body) {
+ res.status = status;
+ res.set_content(body.dump(), "application/json");
+ }
+
+ void send_error(Response& res, int status, const std::string& message) {
+ res.status = status;
+ res.set_content(json({{"error", message}}).dump(-1, ' ', false, json::error_handler_t::replace), "application/json");
+ }
+
+ bool local_origin(const Request& req) {
+ if (req.remote_addr != "127.0.0.1" && req.remote_addr != "::1" && req.remote_addr != "::ffff:127.0.0.1")
+ return false;
+ const auto host = req.get_header_value("Host");
+ const auto port = ":" + std::to_string(req.local_port);
+ const auto origin = req.get_header_value("Origin");
+ const auto site = req.get_header_value("Sec-Fetch-Site");
+ if ((!site.empty() && site != "same-origin" && site != "none") ||
+ (!origin.empty() && origin != "http://" + host && origin != "https://" + host))
+ return false;
+ for (const std::string address : {"localhost", "127.0.0.1", "[::1]"}) {
+ if (host == address || host == address + port)
+ return true;
+ }
+ return false;
+ }
+
+ httplib::Server::Handler local_dgl_route(httplib::Server::Handler handler) {
+ return [handler = std::move(handler)](const Request& req, Response& res) {
+ res.set_header("Cache-Control", "no-store");
+ res.set_header("Cross-Origin-Resource-Policy", "same-origin");
+ if (!local_origin(req)) {
+ send_error(res, 403, "DGL is available from the local server's Web UI");
+ return;
+ }
+ if (req.method == "POST") {
+ const auto content_type = req.get_header_value("Content-Type");
+ if (content_type.substr(0, content_type.find(';')) != "application/json") {
+ send_error(res, 415, "DGL requests require application/json");
+ return;
+ }
+ }
+ try {
+ handler(req, res);
+ } catch (const json::exception&) {
+ send_error(res, 400, "Expected valid UTF-8 JSON or script text");
+ } catch (const std::exception& error) {
+ send_error(res, 500, error.what());
+ }
+ };
+ }
+
+ bool valid_script_name(const std::string& name) {
+ return name.size() > std::string_view(".dgl").size() && name.find_first_of("/\\:*?\"<>|") == std::string::npos &&
+ std::none_of(name.begin(), name.end(), [](unsigned char c) { return c < 32; }) &&
+ fs::u8path(name).extension() == ".dgl";
+ }
+
+ bool valid_source(const std::string& source) {
+ return source.find('\0') == std::string::npos;
+ }
+
+ std::string read_script(const fs::path& path) {
+ std::ifstream file(path, std::ios::binary | std::ios::ate);
+ const auto length = file.tellg();
+ if (length < 0)
+ throw std::runtime_error("Could not open script");
+ if (static_cast<uintmax_t>(length) > std::string{}.max_size())
+ throw std::runtime_error("Script is too large to read");
+ std::string source(static_cast<size_t>(length), '\0');
+ file.seekg(0);
+ if (!file.read(source.data(), length) || file.peek() != std::char_traits<char>::eof())
+ throw std::runtime_error("Script changed while reading");
+ if (!valid_source(source))
+ throw std::runtime_error("Script must contain UTF-8 text");
+ return source;
+ }
+
+ void save_script(const fs::path& path, const std::string& source) {
+ const auto prefix = ".dgl-save-" + std::to_string(unix_timestamp_now_ms()) + "-";
+ fs::path staging;
+ for (size_t i = 0; staging.empty(); ++i) {
+ auto candidate = path.parent_path() / (prefix + std::to_string(i));
+ if (fs::create_directory(candidate))
+ staging = std::move(candidate);
+ }
+ const auto temporary = staging / "script";
+ try {
+ std::ofstream file(temporary, std::ios::binary);
+ file.write(source.data(), static_cast<std::streamsize>(source.size()));
+ file.close();
+ if (!file)
+ throw std::runtime_error("Could not write script");
+ fs::rename(temporary, path);
+ } catch (...) {
+ std::error_code ignored;
+ fs::remove(temporary, ignored);
+ fs::remove(staging, ignored);
+ throw;
+ }
+ std::error_code ignored;
+ fs::remove(staging, ignored);
+ }
+
+ std::optional<size_t> parse_index(std::string_view value) {
+ size_t index = 0;
+ const auto parsed = std::from_chars(value.data(), value.data() + value.size(), index);
+ if (parsed.ec != std::errc{} || parsed.ptr != value.data() + value.size())
+ return std::nullopt;
+ return index;
+ }
+
+ std::shared_ptr<AsyncGenerationJob> find_dgl_job(AsyncJobManager& manager, const std::string& id, Response& res) {
+ purge_expired_jobs(manager);
+ auto it = manager.jobs.find(id);
+ if (it != manager.jobs.end() && it->second->kind == AsyncJobKind::Dgl)
+ return it->second;
+ const bool expired = manager.expired_jobs.count(id) != 0;
+ send_error(res, expired ? 410 : 404, expired ? "Job expired" : "Job not found");
+ return {};
+ }
+
+ void send_output_preview(const Request& req, Response& res, const sd_dgl_result_t* result) {
+ const auto output = parse_index(req.matches[2].str());
+ if (!output || *output >= sd_dgl_result_count(result)) {
+ send_error(res, 404, "Output not found");
+ return;
+ }
+ const auto* tensor = sd_dgl_result_tensor(result, sd_dgl_result_name(result, *output));
+ const auto frames = dgl_image_frame_count(tensor);
+ const auto frame = req.has_param("frame") ? parse_index(req.get_param_value("frame")) : std::optional<size_t>(0);
+ if (!frame || *frame >= static_cast<size_t>(frames)) {
+ send_error(res, 404, "Frame not found");
+ return;
+ }
+ const size_t stride = sd_tensor_byte_size(tensor) / static_cast<size_t>(frames);
+ const auto* pixels = static_cast<const uint8_t*>(sd_tensor_data(tensor)) + *frame * stride;
+ const auto png = encode_image_to_vector(EncodedImageFormat::PNG, pixels,
+ int(sd_tensor_dim(tensor, 1)), int(sd_tensor_dim(tensor, 2)),
+ int(sd_tensor_dim(tensor, 0)));
+ if (png.empty()) {
+ send_error(res, 500, "Could not encode preview");
+ return;
+ }
+ res.set_content(reinterpret_cast<const char*>(png.data()), png.size(), "image/png");
+ }
+}
+
+void register_dgl_endpoints(httplib::Server& svr, ServerRuntime& runtime) {
+ auto workspace = std::make_shared<DglWorkspace>(runtime.svr_params->dgl_dir);
+ svr.Get("/sdcpp/v1/dgl/scripts", local_dgl_route([workspace](const Request&, Response& res) {
+ std::lock_guard<std::mutex> lock(workspace->mutex);
+ std::vector<std::string> names;
+ if (fs::exists(workspace->directory)) {
+ for (const auto& entry : fs::directory_iterator(workspace->directory)) {
+ auto name = entry.path().filename().u8string();
+ if (valid_script_name(name) && fs::is_regular_file(entry.symlink_status()))
+ names.push_back(std::move(name));
+ }
+ }
+ std::sort(names.begin(), names.end());
+ send_json(res, 200, {{"directory", workspace->directory.generic_u8string()}, {"scripts", names}});
+ }));
+
+ svr.Get("/sdcpp/v1/dgl/script", local_dgl_route([workspace](const Request& req, Response& res) {
+ const auto name = req.get_param_value("name");
+ if (!valid_script_name(name)) {
+ send_error(res, 400, "Expected a .dgl file name");
+ return;
+ }
+ std::lock_guard<std::mutex> lock(workspace->mutex);
+ const auto path = workspace->directory / fs::u8path(name);
+ if (!fs::is_regular_file(fs::symlink_status(path))) {
+ send_error(res, 404, "Script not found");
+ return;
+ }
+ send_json(res, 200, {{"source", read_script(path)}});
+ }));
+
+ svr.Post("/sdcpp/v1/dgl/script", local_dgl_route([workspace](const Request& req, Response& res) {
+ const auto body = json::parse(req.body);
+ const auto name = body.at("name").get<std::string>();
+ const auto source = body.at("source").get<std::string>();
+ if (!valid_script_name(name) || !valid_source(source)) {
+ send_error(res, 400, "Expected a .dgl file name and UTF-8 source");
+ return;
+ }
+ const bool updating = body.contains("previous_source");
+ std::lock_guard<std::mutex> lock(workspace->mutex);
+ const auto path = workspace->directory / fs::u8path(name);
+ const auto status = fs::symlink_status(path);
+ if (updating) {
+ if (!fs::is_regular_file(status) || read_script(path) != body.at("previous_source").get<std::string>()) {
+ send_error(res, 409, "The script changed on disk. Reopen it or save with a new name.");
+ return;
+ }
+ } else if (fs::exists(status)) {
+ send_error(res, 409, "A file with that name already exists");
+ return;
+ }
+ fs::create_directories(workspace->directory);
+ save_script(path, source);
+ send_json(res, updating ? 200 : 201, {{"name", name}});
+ }));
+
+ RuntimeRoutes routes(svr, runtime);
+ // Use the model transition lock during admission, as with image and video jobs.
+ routes.post("/sdcpp/v1/dgl/jobs", local_dgl_route([&runtime](const Request& req, Response& res) {
+ auto source = json::parse(req.body).at("source").get<std::string>();
+ if (source.empty() || !valid_source(source)) {
+ send_error(res, 400, "Expected non-empty UTF-8 source");
+ return;
+ }
+ auto job = std::make_shared<AsyncGenerationJob>();
+ job->kind = AsyncJobKind::Dgl;
+ job->dgl_source = std::move(source);
+ auto& manager = *runtime.async_job_manager;
+ {
+ std::lock_guard<std::mutex> lock(manager.mutex);
+ purge_expired_jobs(manager);
+ if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
+ send_error(res, 429, "Job queue is full");
+ return;
+ }
+ job->id = make_async_job_id(manager);
+ manager.jobs.emplace(job->id, job);
+ manager.queue.push_back(job->id);
+ send_json(res, 202, make_async_job_json(manager, *job));
+ }
+ manager.cv.notify_one();
+ }));
+
+ svr.Get(R"(/sdcpp/v1/dgl/jobs/([A-Za-z0-9_-]+))", local_dgl_route([&runtime](const Request& req, Response& res) {
+ auto& manager = *runtime.async_job_manager;
+ std::lock_guard<std::mutex> lock(manager.mutex);
+ if (auto job = find_dgl_job(manager, req.matches[1].str(), res))
+ send_json(res, 200, make_async_job_json(manager, *job));
+ }));
+
+ svr.Get(R"(/sdcpp/v1/dgl/jobs/([A-Za-z0-9_-]+)/outputs/([0-9]+))", local_dgl_route([&runtime](const Request& req, Response& res) {
+ auto& manager = *runtime.async_job_manager;
+ std::shared_ptr<sd_dgl_result_t> result;
+ {
+ std::lock_guard<std::mutex> lock(manager.mutex);
+ const auto job = find_dgl_job(manager, req.matches[1].str(), res);
+ if (!job)
+ return;
+ result = job->dgl_result;
+ }
+ if (!result) {
+ send_error(res, 404, "Job has no result");
+ return;
+ }
+ send_output_preview(req, res, result.get());
+ }));
+}
diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp
index 9dbd6ed..d7db0d6 100644
--- a/examples/server/routes_models.cpp
+++ b/examples/server/routes_models.cpp
@@ -117,6 +117,7 @@ static void change_model(ServerRuntime& runtime, const httplib::Request& req, ht
if (!models.context) {
throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors");
}
+ prepare_native_context(models, *runtime.svr_params, *runtime.ctx_params);
runtime.sd_ctx = models.context.get();
models.active_recipe = std::move(loaded_recipe);
}
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index e67bd20..b92d075 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -226,12 +226,8 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
PromptImageCallbackGuard prompt_images(runtime.sd_ctx, request.gen_params);
sd_result_t* generated = nullptr;
generate_sd_image(runtime.sd_ctx, img_gen_params, &generated);
- results.adopt(generated, runtime.ctx_params->lora_model_dir);
+ results.adopt(generated);
num_results = results.count();
- if (!results.empty() && !apply_server_adetailer(runtime, request, results)) {
- error_message = "ADetailer failed";
- return false;
- }
}
if (results.empty()) {
@@ -311,7 +307,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
json item;
item["b64_json"] = base64_encode(image_bytes);
- item["generation_info"] = json::parse(results.generation_info(i));
+ item["generation_info"] = results.generation_info(i);
out["data"].push_back(item);
}
@@ -375,7 +371,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
request.output_compression);
json item;
item["b64_json"] = base64_encode(image_bytes);
- item["generation_info"] = json::parse(results.generation_info(i));
+ item["generation_info"] = results.generation_info(i);
out["data"].push_back(item);
}
diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp
index e9f60c7..8539e53 100644
--- a/examples/server/routes_sdapi.cpp
+++ b/examples/server/routes_sdapi.cpp
@@ -375,16 +375,13 @@ static bool build_sdapi_img_gen_request(const json& j,
}
}
- if (j.contains("ref_image_args")) {
- if (!j["ref_image_args"].is_string()) {
- error_message = "ref_image_args must be a string";
- return false;
- }
- request.gen_params.ref_image_args = j["ref_image_args"].get<std::string>();
+ if (j.contains("ref_image_args") && j["ref_image_args"] != "") {
+ error_message = "reference overrides require a DGL script";
+ return false;
}
- if (j.contains("image_preprocess") && !request.gen_params.parse_image_preprocess_json(j["image_preprocess"].dump())) {
- error_message = "invalid image_preprocess";
+ if (j.contains("image_preprocess")) {
+ error_message = "image_preprocess requires a DGL script";
return false;
}
@@ -453,26 +450,22 @@ static bool build_sdapi_img_gen_request(const json& j,
validate_server_img_gen_request(runtime, request, error_message);
}
-static nlohmann::json prepare_info_field(const std::string& generation_info) {
- const auto info = nlohmann::json::parse(generation_info);
- const auto& sampling = info.at("sample_params");
- nlohmann::json result = {{"prompt", info.at("prompt")},
- {"negative_prompt", info.at("negative_prompt")},
- {"seed", generation_seed(generation_info)},
- {"cfg_scale", sampling.at("guidance").at("txt_cfg")},
- {"width", info.at("width")},
- {"height", info.at("height")},
- {"steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps")},
- {"sampler_name", sampling.at("sample_method")},
- {"version", "stable-diffusion.cpp"}};
- if (info.contains("clip_skip"))
- result["clip_skip"] = info.at("clip_skip");
- if (info.contains("strength"))
- result["denoising_strength"] = info.at("strength");
- if (sampling.contains("scheduler"))
- result["extra_generation_params"]["Schedule type"] = sampling.at("scheduler");
+static json generation_metadata(const std::string& source) {
+ const auto* metadata = sd_get_generation_metadata(source.c_str());
+ return metadata ? json::parse(metadata) : json::object();
+}
+
+static nlohmann::json prepare_info_field(const json& info) {
+ auto result = json::object();
+ const std::map<std::string, std::string> names{{"prompt", "prompt"}, {"negative_prompt", "negative_prompt"}, {"seed", "seed"}, {"cfg_scale", "cfg_scale"}, {"width", "width"}, {"height", "height"}, {"steps", "steps"}, {"method", "sampler_name"}, {"clip_skip", "clip_skip"}, {"strength", "denoising_strength"}};
+ for (const auto& [key, output] : names)
+ if (info.contains(key))
+ result[output] = info.at(key);
+ if (info.contains("scheduler"))
+ result["extra_generation_params"]["Schedule type"] = info.at("scheduler");
+ result["version"] = "stable-diffusion.cpp";
for (const auto* key : {"infotexts", "all_prompts", "all_negative_prompts", "all_seeds", "generation_info"})
- result[key] = nlohmann::json::array();
+ result[key] = json::array();
return result;
}
@@ -600,15 +593,10 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
SDAPIPreviewGuard preview_guard(id_task, request.gen_params, img2img ? "img2img" : "txt2img", detailing);
sd_result_t* generated = nullptr;
generate_sd_image(runtime->sd_ctx, img_gen_params, &generated);
- results.adopt(generated, runtime->ctx_params->lora_model_dir);
+ results.adopt(generated);
num_results = results.count();
if (detailing && !results.empty())
preview_guard.begin_detailing();
- if (!results.empty() && !apply_server_adetailer(*runtime, request, results)) {
- res.status = 500;
- res.set_content(R"({"error":"ADetailer failed"})", "application/json");
- return;
- }
}
if (results.empty()) {
@@ -620,7 +608,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
json out;
out["images"] = json::array();
out["parameters"] = j;
- json jsoninfo = prepare_info_field(results.generation_info(0));
+ auto metadata = generation_metadata(results.generation_info(0));
+ json jsoninfo = prepare_info_field(metadata);
for (int i = 0; i < num_results; ++i) {
if (results[i].data == nullptr) {
@@ -629,7 +618,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
bool embed_meta = request.gen_params.embed_image_metadata;
- std::string params = generation_info_to_webui(results.generation_info(i));
+ if (i != 0)
+ metadata = generation_metadata(results.generation_info(i));
+ const auto params = metadata.value("infotext", "");
auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG,
results[i].data,
@@ -645,12 +636,12 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
out["images"].push_back(base64_encode(image_bytes));
- const auto actual = json::parse(results.generation_info(i));
jsoninfo["infotexts"].push_back(params);
- jsoninfo["all_seeds"].push_back(generation_seed(results.generation_info(i)));
- jsoninfo["generation_info"].push_back(actual);
- jsoninfo["all_prompts"].push_back(actual["prompt"]);
- jsoninfo["all_negative_prompts"].push_back(actual["negative_prompt"]);
+ const auto seed = metadata.find("seed");
+ jsoninfo["all_seeds"].push_back(seed != metadata.end() && seed->is_number_integer() ? seed->get<int64_t>() : -1);
+ jsoninfo["generation_info"].push_back(results.generation_info(i));
+ jsoninfo["all_prompts"].push_back(metadata.value("prompt", ""));
+ jsoninfo["all_negative_prompts"].push_back(metadata.value("negative_prompt", ""));
}
// not a mistake: it is supposed to be a string in json format
diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp
index c6e4971..8fe235a 100644
--- a/examples/server/routes_sdcpp.cpp
+++ b/examples/server/routes_sdcpp.cpp
@@ -119,9 +119,6 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
{"seed", defaults.seed},
{"batch_count", defaults.batch_count},
{"qwen_image_layers", defaults.qwen_image_layers},
- {"ref_image_args", defaults.ref_image_args},
- {"image_preprocess", defaults.image_preprocess},
- {"increase_ref_index", defaults.increase_ref_index},
{"control_strength", defaults.control_strength},
{"control_type", defaults.control_type},
{"ip_adapter_strength", defaults.ip_adapter_strength},
@@ -147,8 +144,6 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const
{"strength", defaults.strength},
{"seed", defaults.seed},
{"video_frames", defaults.video_frames},
- {"image_preprocess", defaults.image_preprocess},
- {"ref_image_args", defaults.ref_image_args},
{"fps", defaults.fps},
{"moe_boundary", defaults.moe_boundary},
{"vace_strength", defaults.vace_strength},
@@ -170,7 +165,7 @@ static json make_img_gen_features_json() {
{"init_image", true},
{"mask_image", true},
{"control_image", true},
- {"ip_adapter_image", true},
+ {"ip_adapter_image", false},
{"ref_images", true},
{"lora", true},
{"vae_tiling", true},
@@ -191,7 +186,7 @@ static json make_vid_gen_features_json() {
{"ref_videos", true},
{"ref_audios", true},
{"ref_video_audios", true},
- {"high_noise_sample_params", true},
+ {"high_noise_sample_params", false},
{"lora", true},
{"vae_tiling", true},
{"cache", true},
@@ -332,9 +327,7 @@ static json make_capabilities_json(ServerRuntime& runtime) {
std::string(key) == "ref_video_audios")))
features[key] = generation_features.at(key);
}
- if (video)
- features["high_noise_sample_params"] = generation_features.at("high_noise");
- else {
+ if (!video) {
features["mask_image"] = generation_features.at("mask_image").get<bool>() && generation_features.at("img2img").get<bool>();
features["control_image"] = generation_features.at("controlnet").get<bool>() || generation_features.at("control_image").get<bool>();
features["ip_adapter_image"] = generation_features.at("ip_adapter");
@@ -607,21 +600,16 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
SDImageOwner current(input.release());
std::string generation_info;
- for (int i = 0; i < repeats; ++i) {
- sd_result_t* generated = nullptr;
- if (!upscale_sd_image(upscaler_ctx.get(), current.get(), &generated) ||
- !generated || sd_result_get_image_count(generated) <= 0 || sd_tensor_data(sd_result_get_image(generated, 0)) == nullptr) {
- sd_result_free(generated);
- res.status = 500;
- res.set_content(R"({"error":"upscale failed"})", "application/json");
- return;
- }
- generation_info = merge_generation_info(generation_info, sd_result_get_generation_info(generated, 0));
- SDImage produced = take_result_image(generated, 0);
-
+ sd_result_t* generated = nullptr;
+ if (!upscale_sd_image(upscaler_ctx.get(), current.get(), &generated, repeats) || !generated) {
sd_result_free(generated);
- current.reset(produced);
+ res.status = 500;
+ res.set_content(R"({"error":"upscale failed"})", "application/json");
+ return;
}
+ generation_info = sd_result_get_generation_info(generated, 0);
+ current.reset(take_result_image(generated, 0));
+ sd_result_free(generated);
const std::string& format = output_options.output_format;
const int compression = output_options.output_compression;
@@ -642,7 +630,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
}
json out;
- out["generation_info"] = json::parse(generation_info);
+ out["generation_info"] = generation_info;
out["upscaler"] = used_name;
out["scale"] = factor;
out["repeats"] = repeats;
@@ -794,6 +782,11 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
std::string job_id = req.matches[1];
auto it = manager.jobs.find(job_id);
+ if (it != manager.jobs.end() && it->second->kind == AsyncJobKind::Dgl) {
+ res.status = 404;
+ res.set_content(R"({"error":"job not found"})", "application/json");
+ return;
+ }
if (it == manager.jobs.end()) {
if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
res.status = 410;
@@ -828,6 +821,11 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
}
auto& job = *it->second;
+ if (job.kind == AsyncJobKind::Dgl) {
+ res.status = 404;
+ res.set_content(R"({"error":"job not found"})", "application/json");
+ return;
+ }
if (job.status == AsyncJobStatus::Queued) {
if (!cancel_queued_job(manager, job)) {
res.status = 409;
diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp
index 6ac3866..c473c06 100644
--- a/examples/server/runtime.cpp
+++ b/examples/server/runtime.cpp
@@ -56,139 +56,20 @@ bool validate_server_img_gen_request(const ServerRuntime& runtime,
(params.height && !within_dimension_limit(*params.height)))
return false;
if (params.hires_enabled.value_or(false)) {
- if ((params.hires_width && !within_dimension_limit(*params.hires_width)) ||
- (params.hires_height && !within_dimension_limit(*params.hires_height)))
+ const double width = params.width.value_or(runtime.default_gen_params->width.value_or(512));
+ const double height = params.height.value_or(runtime.default_gen_params->height.value_or(512));
+ const double scale = params.hires_scale.value_or(2.f);
+ const auto target_width = params.hires_width.value_or(0), target_height = params.hires_height.value_or(0);
+ if (!within_dimension_limit(target_width > 0 ? target_width : target_height > 0 ? target_height
+ : width * scale) ||
+ !within_dimension_limit(target_height > 0 ? target_height : target_width > 0 ? target_width
+ : height * scale))
return false;
- if (params.hires_width.value_or(0) <= 0 && params.hires_height.value_or(0) <= 0) {
- // Use the documented library defaults for this resource-policy check only.
- const float scale = params.hires_scale.value_or(2.f);
- if (!within_dimension_limit(std::round(params.width.value_or(512) * scale)) ||
- !within_dimension_limit(std::round(params.height.value_or(512) * scale)))
- return false;
- }
- }
- if (!request.adetailer_enabled || runtime.default_gen_params->ad_model_path.empty()) {
- return true;
}
- auto pass_valid = [&](const ServerADetailerPass& pass) {
- if ((pass.width && !within_dimension_limit(*pass.width)) ||
- (pass.height && !within_dimension_limit(*pass.height))) {
- return false;
- }
- // Match the native key normalization, but leave full option parsing to ADetailer.
- auto trim = [](const std::string& text) {
- const auto first = text.find_first_not_of(" \t\r\n");
- return first == std::string::npos ? std::string() : text.substr(first, text.find_last_not_of(" \t\r\n") - first + 1);
- };
- std::stringstream stream(pass.extra_args);
- std::string item;
- while (std::getline(stream, item, ',')) {
- const auto equals = item.find('=');
- if (equals == std::string::npos) {
- continue;
- }
- auto key = trim(item.substr(0, equals));
- std::transform(key.begin(), key.end(), key.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
- if (key != "inpaint_width" && key != "inpaint_height" && key != "steps") {
- continue;
- }
- const auto value = trim(item.substr(equals + 1));
- const int minimum = key == "steps" ? 0 : 1;
- const int maximum = key == "steps" ? k_max_adetailer_steps : k_max_generation_dimension;
- try {
- size_t used = 0;
- const int parsed = std::stoi(value, &used);
- if (used == value.size() && parsed >= minimum && parsed <= maximum) {
- continue;
- }
- } catch (const std::exception&) {
- }
- error_message = "ADetailer " + key + " must be between " + std::to_string(minimum) + " and " + std::to_string(maximum);
- return false;
- }
- return true;
- };
- if (request.adetailer_passes.empty()) {
- ServerADetailerPass pass;
- pass.extra_args = params.extra_ad_args;
- return pass_valid(pass);
- }
- return std::all_of(request.adetailer_passes.begin(), request.adetailer_passes.end(), pass_valid);
-}
-
-bool apply_server_adetailer(ServerRuntime& runtime,
- ImgGenJobRequest& request,
- SDImageVec& results) {
- auto& gen_params = request.gen_params;
- const auto& ad_model_path = runtime.default_gen_params->ad_model_path;
- if (!request.adetailer_enabled || ad_model_path.empty()) {
- return true;
- }
- std::string error_message;
- if (!validate_server_img_gen_request(runtime, request, error_message)) {
- LOG_ERROR("%s", error_message.c_str());
- return false;
- }
-
- std::vector<ServerADetailerPass> passes = request.adetailer_passes;
- if (passes.empty()) {
- ServerADetailerPass pass;
- pass.prompt = gen_params.ad_prompt;
- pass.negative_prompt = gen_params.ad_negative_prompt;
- pass.extra_args = gen_params.extra_ad_args;
- passes.push_back(std::move(pass));
- }
-
- ADetailerCtxPtr ad_ctx(create_sd_adetailer(ad_model_path.c_str(),
- runtime.ctx_params->n_threads,
- runtime.ctx_params->effective_backend.c_str(),
- runtime.ctx_params->effective_params_backend.c_str()));
- if (ad_ctx == nullptr) {
- LOG_ERROR("sd_adetailer_create failed");
+ if ((!runtime.default_gen_params->ad_model_path.empty() && request.adetailer_enabled) || !request.adetailer_passes.empty()) {
+ error_message = "ADetailer requires a DGL script";
return false;
}
-
- for (const auto& pass : passes) {
- for (int i = 0; i < results.count(); ++i) {
- if (results[i].data == nullptr) {
- continue;
- }
- SDOptBuilder ad_options(gen_params.to_image_opt(true).release());
- ad_options.unset("width");
- ad_options.unset("height");
- ad_options.unset("strength");
- ad_options.set_string("adetail.prompt", pass.prompt);
- ad_options.set_string("adetail.negative_prompt", pass.negative_prompt);
- ad_options.set_string("adetail.args", pass.extra_args);
- ad_options.set_int("seed", generation_seed(results.generation_info(i)));
- ad_options.set_int("width", pass.width);
- ad_options.set_int("height", pass.height);
- ad_options.set_float("strength", pass.strength);
- if (pass.clip_skip) {
- ad_options.set_int("clip_skip", pass.clip_skip);
- }
- if (pass.automatic_scheduler) {
- ad_options.set_string("sample.scheduler", "auto");
- }
-
- sd_result_t* detailed = nullptr;
- if (!adetail_sd_image(ad_ctx.get(),
- runtime.sd_ctx,
- results[i],
- ad_options.finish(),
- &detailed) ||
- detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) {
- sd_result_free(detailed);
- LOG_ERROR("ADetailer failed for image %d", i + 1);
- return false;
- }
- results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), runtime.ctx_params->lora_model_dir);
- sd_image_free(&results[i]);
- results[i] = take_result_image(detailed, 0);
-
- sd_result_free(detailed);
- }
- }
return true;
}
@@ -210,11 +91,18 @@ static const std::string k_base64_chars =
"0123456789+/";
std::string base64_encode(const std::vector<uint8_t>& bytes) {
+ return base64_encode(bytes.data(), bytes.size());
+}
+
+std::string base64_encode(const void* data, size_t size) {
std::string ret;
- int val = 0;
- int valb = -6;
- for (uint8_t c : bytes) {
- val = (val << 8) + c;
+ ret.reserve((size + 2) / 3 * 4);
+ uint32_t val = 0;
+ int valb = -6;
+ const auto* bytes = static_cast<const uint8_t*>(data);
+ for (size_t i = 0; i < size; ++i) {
+ const auto c = bytes[i];
+ val = (val << 8) + c;
valb += 8;
while (valb >= 0) {
ret.push_back(k_base64_chars[(val >> valb) & 0x3F]);
@@ -370,6 +258,7 @@ ArgOptions SDSvrParams::get_options() {
options.string_options = {
{"-l", "--listen-ip", "server listen ip (default: 127.0.0.1)", 0, &listen_ip},
{"", "--serve-html-path", "path to HTML file to serve at root (optional)", 0, &serve_html_path},
+ {"", "--dgl-dir", "Web UI script directory (default: dgl)", 0, &dgl_dir},
};
options.int_options = {
@@ -431,6 +320,10 @@ bool SDSvrParams::validate() {
LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
return false;
}
+ if (dgl_dir.empty()) {
+ LOG_ERROR("--dgl-dir requires a directory path");
+ return false;
+ }
return true;
}
@@ -450,6 +343,7 @@ std::string SDSvrParams::to_string() const {
<< " max_pending_jobs: " << max_pending_jobs << ",\n"
<< " completed_ttl_seconds: " << completed_ttl_seconds << ",\n"
<< " serve_html_path: \"" << serve_html_path << "\",\n"
+ << " dgl_dir: \"" << dgl_dir << "\",\n"
<< "}";
return oss.str();
}
diff --git a/examples/server/runtime.h b/examples/server/runtime.h
index abb9766..51d6e88 100644
--- a/examples/server/runtime.h
+++ b/examples/server/runtime.h
@@ -25,6 +25,7 @@ struct SDSvrParams {
int max_pending_jobs = 64;
int completed_ttl_seconds = 600;
std::string serve_html_path;
+ std::string dgl_dir = "dgl";
std::vector<std::string> model_dirs;
bool normal_exit = false;
sd_log_level_t log_level = SD_LOG_INFO;
@@ -124,6 +125,7 @@ struct VidGenJobRequest {
};
std::string base64_encode(const std::vector<uint8_t>& bytes);
+std::string base64_encode(const void* bytes, size_t size);
std::string normalize_output_format(std::string output_format);
std::vector<std::string> supported_img_output_formats(bool allow_webp = true);
std::vector<std::string> supported_vid_output_formats();
@@ -138,11 +140,9 @@ bool assign_output_options(VidGenJobRequest& request,
std::string& error_message);
std::string video_mime_type(const std::string& output_format);
bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode);
+void prepare_native_context(ServerModels& models, const SDSvrParams& params, const SDContextParams& config, bool startup = false);
std::string unsupported_generation_mode_error(SDMode mode);
void refresh_lora_cache(ServerRuntime& rt);
std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
void refresh_upscaler_cache(ServerRuntime& rt);
int64_t unix_timestamp_now_ms();
-bool apply_server_adetailer(ServerRuntime& runtime,
- ImgGenJobRequest& request,
- SDImageVec& results);
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index c5a25c6..64ae419 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -35,8 +35,9 @@ typedef struct sd_ctx_t sd_ctx_t;
typedef struct sd_result sd_result_t;
typedef struct sd_opt sd_opt_t;
typedef struct sd_upscaler sd_upscaler_t;
-typedef struct sd_vid_upscaler sd_vid_upscaler_t;
-typedef struct sd_adetailer sd_adetailer_t;
+typedef struct sd_dgl_ctx_t sd_dgl_ctx_t;
+typedef struct sd_dgl_result_t sd_dgl_result_t;
+typedef struct sd_dgl_value_t sd_dgl_value_t;
// Status and diagnostics
@@ -58,6 +59,51 @@ typedef enum {
// Valid until the next API call; free functions and tensor retain preserve it.
SD_API const char* sd_get_last_error(void);
+// Generation metadata
+
+// Returns inferred image parameters and infotext as UTF-8 JSON, or {} when no
+// image parameters can be inferred. Does not execute source or read files.
+// Borrowed until the next call to this function on the same thread.
+// Returns NULL on failure; see sd_get_last_error().
+SD_API const char* sd_get_generation_metadata(const char* source);
+
+typedef enum {
+ SD_METADATA_READ_JSON = 1u << 0,
+ SD_METADATA_READ_RAW = 1u << 1,
+ SD_METADATA_READ_BRIEF = 1u << 2,
+ SD_METADATA_READ_STRUCTURAL = 1u << 3,
+} sd_metadata_read_flags_t;
+
+// Reads container metadata without loading a model. Flags may be ORed together;
+// zero selects the default text report. Returns NULL on failure.
+// Borrowed until the next call to this function on the same thread.
+SD_API const char* sd_read_metadata(const char* path, uint32_t flags);
+
+typedef enum {
+ SD_METADATA_PNG,
+ SD_METADATA_JPEG,
+ SD_METADATA_XMP,
+ SD_METADATA_AVI,
+} sd_metadata_format_t;
+
+// Builds PNG iTXt chunks, JPEG COM segments, UTF-8 XMP or an AVI INFO chunk.
+// PNG/JPEG/XMP contain the DGL source and inferred infotext; AVI stores source.
+// Does not write files. Borrowed bytes remain valid until the next call to this
+// function on the same thread. Empty source succeeds with NULL, 0.
+// Output pointers are required and cleared on failure.
+SD_API sd_status_t sd_get_metadata_chunk(const char* source, sd_metadata_format_t format, const void** data_out, size_t* size_out);
+
+// File access for generation and DGL contexts
+
+typedef enum {
+ SD_DGL_FILE_READ,
+ SD_DGL_FILE_READ_MODEL,
+ SD_DGL_FILE_WRITE,
+} sd_dgl_file_access_t;
+
+// Return false to reject the access. The UTF-8 path is valid only during the callback.
+typedef bool (*sd_dgl_file_access_cb_t)(const char* path, sd_dgl_file_access_t access, void* userdata);
+
// Tensors and media
typedef int32_t sd_tensor_type_t;
@@ -97,6 +143,10 @@ SD_API int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis);
SD_API size_t sd_tensor_byte_size(const sd_tensor_t* tensor);
SD_API const void* sd_tensor_data(const sd_tensor_t* tensor);
+// Saves a NumPy .npy file. NumPy shape is the reverse of sd_tensor shape.
+// Failures use sd_get_last_error().
+SD_API bool sd_tensor_save(const sd_tensor_t* tensor, const char* path);
+
// Options
typedef struct {
@@ -115,10 +165,6 @@ typedef sd_opt_t sd_img_gen_opt_t;
typedef sd_opt_t sd_vid_gen_opt_t;
typedef sd_opt_t sd_upscaler_opt_t;
typedef sd_opt_t sd_upscale_opt_t;
-typedef sd_opt_t sd_vid_upscaler_opt_t;
-typedef sd_opt_t sd_vid_upscale_opt_t;
-typedef sd_opt_t sd_adetailer_opt_t;
-typedef sd_opt_t sd_adetail_opt_t;
// Parameter keys and types are documented in docs/guides/c_api.md.
// Setters replace values atomically; a failed setter preserves the previous value.
@@ -131,10 +177,6 @@ SD_API sd_img_gen_opt_t* sd_img_gen_opt_create(void);
SD_API sd_vid_gen_opt_t* sd_vid_gen_opt_create(void);
SD_API sd_upscaler_opt_t* sd_upscaler_opt_create(void);
SD_API sd_upscale_opt_t* sd_upscale_opt_create(void);
-SD_API sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void);
-SD_API sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void);
-SD_API sd_adetailer_opt_t* sd_adetailer_opt_create(void);
-SD_API sd_adetail_opt_t* sd_adetail_opt_create(void);
SD_API void sd_opt_free(sd_opt_t* params);
SD_API sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value);
@@ -168,6 +210,11 @@ SD_API sd_status_t sd_opt_unset(sd_opt_t* params, const char* key);
// its own configuration; params may be freed immediately after the call returns.
SD_API sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out);
SD_API void sd_ctx_free(sd_ctx_t* ctx);
+// Applies to generated programs and model probing. Register or clear while idle.
+SD_API sd_status_t sd_ctx_set_file_access_callback(sd_ctx_t* ctx, sd_dgl_file_access_cb_t callback, void* userdata);
+// Probe the model and prepare its configured components without generating.
+// Optional for lazy C API use; call after setting file permissions to validate loading.
+SD_API sd_status_t sd_ctx_prepare(sd_ctx_t* ctx);
SD_API sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out);
SD_API sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out);
@@ -175,12 +222,7 @@ SD_API sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* para
SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
-// ControlNet hot-swap APIs are not safe to call while generation is in flight.
-SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);
-SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx);
-SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx);
-
-// Requires a loaded context; returns a static string owned by the library, or "Unknown".
+// Probes model metadata lazily; returns a static string, or "Unknown" on error.
SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx);
enum sd_cancel_mode_t {
@@ -212,7 +254,8 @@ SD_API const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t
// Materializes contiguous U8 [C,W,H,T] storage on first access.
SD_API const sd_video_t* sd_result_get_video(const sd_result_t* result);
-// Image metadata is per image; video metadata is shared by all frame indices.
+// Returns the exact UTF-8 DGL program used for inference. Image programs are per image;
+// video programs are shared by all frame indices. Host inputs must be supplied again to replay.
SD_API const char* sd_result_get_generation_info(const sd_result_t* result, size_t index);
SD_API const sd_audio_t* sd_result_get_audio(const sd_result_t* result);
SD_API int sd_result_get_fps(const sd_result_t* result);
@@ -296,35 +339,112 @@ SD_API int sd_upscaler_get_scale(const sd_upscaler_t* upscaler);
// Reads model metadata only; returns 0 if it is not a recognized RGB upscaler.
SD_API int sd_upscaler_probe_scale(const char* model_path);
-// Video upscaling
-
-SD_API sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out);
-SD_API void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler);
-SD_API sd_status_t sd_upscale_video(
- sd_vid_upscaler_t* upscaler,
- const sd_vid_upscale_opt_t* opt,
- sd_result_t** out);
-
-// ADetailer
-
-SD_API sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out);
-SD_API void sd_adetailer_free(sd_adetailer_t* adetailer);
-SD_API sd_status_t sd_adetail(
- sd_adetailer_t* adetailer,
- sd_ctx_t* ctx,
- const sd_adetail_opt_t* opt,
- sd_result_t** out);
-
-// Image preprocessing
-
-SD_API sd_status_t sd_preprocess_canny(
- const sd_tensor_t* image,
- float high_threshold,
- float low_threshold,
- float weak,
- float strong,
- bool inverse,
- sd_tensor_t** image_out);
+// DGL (Diffusion Graph Language)
+
+SD_API const char* sd_dgl_builtin_catalog(void);
+
+// A context shares loaded components across scripts and runs one script at a time.
+// Keep the context alive until all calls using it return.
+SD_API sd_dgl_ctx_t* sd_dgl_ctx_create(void);
+// Creates a context that releases models after their last use in each run.
+SD_API sd_dgl_ctx_t* sd_dgl_ctx_create_once(void);
+SD_API void sd_dgl_ctx_free(sd_dgl_ctx_t* ctx);
+// Sets options on an idle context: max_vram, max_ram, conditioning_cache_size, batched_cfg,
+// n_threads, auto_fit, enable_mmap, eager_load, disable_prefetch, disable_segmented_compute,
+// embed_metadata (default on; controls generation metadata in DGL image/video saves).
+// Threading, placement and weight-loading options invalidate cached models when changed.
+SD_API bool sd_dgl_ctx_set(sd_dgl_ctx_t* ctx, const char* key, const char* value);
+
+// Optional per-context callback. A null callback restores unrestricted file access.
+SD_API bool sd_dgl_ctx_set_file_access_callback(sd_dgl_ctx_t* ctx, sd_dgl_file_access_cb_t callback, void* userdata);
+
+// Callbacks run synchronously on the executing thread. Set or clear while idle.
+// Pass counts start at 1 in each run. Data supplied by the library is borrowed
+// until the callback returns. Callbacks may request cancellation and must not throw.
+typedef void (*sd_dgl_preview_cb_t)(
+ const sd_preview_info_t* info,
+ int step,
+ int frame_count,
+ const sd_tensor_t* const* frames,
+ bool is_noisy,
+ void* userdata);
+SD_API sd_status_t sd_dgl_ctx_set_progress_callback(sd_dgl_ctx_t* ctx, sd_generation_progress_cb_t callback, void* userdata);
+// Modes: none, proj, tae, vae. Pass NULL as callback to clear.
+// Sampling uses projection previews by default. Positive intervals count
+// completed logical steps; supply preview(decoder=...) for tae or vae decoding.
+SD_API sd_status_t sd_dgl_ctx_set_preview_callback(
+ sd_dgl_ctx_t* ctx,
+ sd_dgl_preview_cb_t callback,
+ const char* mode,
+ int interval,
+ bool denoised,
+ bool noisy,
+ void* userdata);
+
+typedef struct {
+ const char* name;
+ const sd_tensor_t* tensor;
+} sd_dgl_tensor_input_t;
+
+// Checks syntax and types without creating a context or loading models.
+// Set allow_tensor_inputs to accept named tensor parameters in fn main(name, ...).
+SD_API sd_status_t sd_dgl_check(const char* source, bool allow_tensor_inputs);
+
+// Compiles source once per call, binds inputs and executes in the required context.
+// Supply every declared tensor parameter exactly once, or NULL, 0 for no inputs.
+// The run retains tensors as needed; result tensors can share their storage.
+// On failure, *result_out is NULL. Compare the return value with SD_STATUS_OK.
+SD_API sd_status_t sd_dgl_run(
+ sd_dgl_ctx_t* ctx,
+ const char* source,
+ const sd_dgl_tensor_input_t* inputs,
+ size_t input_count,
+ sd_dgl_result_t** result_out);
+// Thread-safe. Requests cooperative cancellation of the current run only,
+// including during compilation; it is checked before execution begins.
+// An idle context accepts the request without queuing it for the next run.
+SD_API sd_status_t sd_dgl_ctx_cancel(sd_dgl_ctx_t* ctx);
+// Alias for sd_get_last_error().
+SD_API const char* sd_dgl_error(void);
+
+SD_API void sd_dgl_result_free(sd_dgl_result_t* result);
+
+typedef enum {
+ SD_DGL_VALUE_INVALID = 0,
+ SD_DGL_VALUE_BOOL,
+ SD_DGL_VALUE_INT,
+ SD_DGL_VALUE_NUMBER,
+ SD_DGL_VALUE_STRING,
+ SD_DGL_VALUE_TENSOR,
+ SD_DGL_VALUE_LIST,
+ SD_DGL_VALUE_DICT,
+ SD_DGL_VALUE_OPAQUE = 11,
+ SD_DGL_VALUE_VOID = 12,
+ SD_DGL_VALUE_BYTES = 13,
+} sd_dgl_value_kind_t;
+
+// Structured values, names and tensor handles are borrowed from the result.
+// Retain tensors to use them after the result is freed.
+SD_API const sd_dgl_value_t* sd_dgl_result_root(const sd_dgl_result_t* result);
+SD_API sd_dgl_value_kind_t sd_dgl_value_kind(const sd_dgl_value_t* value);
+SD_API size_t sd_dgl_value_count(const sd_dgl_value_t* value);
+SD_API const sd_dgl_value_t* sd_dgl_value_at(const sd_dgl_value_t* value, size_t index);
+SD_API const char* sd_dgl_value_name(const sd_dgl_value_t* value);
+SD_API const sd_dgl_value_t* sd_dgl_value_field(const sd_dgl_value_t* value, const char* name);
+SD_API sd_status_t sd_dgl_value_get_bool(const sd_dgl_value_t* value, bool* out);
+SD_API sd_status_t sd_dgl_value_get_int(const sd_dgl_value_t* value, int64_t* out);
+SD_API sd_status_t sd_dgl_value_get_number(const sd_dgl_value_t* value, double* out);
+SD_API sd_status_t sd_dgl_value_get_string(const sd_dgl_value_t* value, const char** out, size_t* size);
+// Borrowed immutable bytes, valid until the result is freed. No NUL terminator.
+SD_API sd_status_t sd_dgl_value_get_bytes(const sd_dgl_value_t* value, const void** out, size_t* size);
+SD_API const sd_tensor_t* sd_dgl_value_tensor(const sd_dgl_value_t* value);
+
+// Flat tensor convenience access. Scalar values are materialized as tensors on demand.
+SD_API const char* sd_dgl_result_type(const sd_dgl_result_t* result);
+SD_API size_t sd_dgl_result_count(const sd_dgl_result_t* result);
+SD_API const char* sd_dgl_result_name(const sd_dgl_result_t* result, size_t index);
+// Borrowed read-only tensor. Use sd_tensor_retain() to keep it past result free.
+SD_API const sd_tensor_t* sd_dgl_result_tensor(const sd_dgl_result_t* result, const char* name);
// Weight types and model conversion
diff --git a/scripts/face_detect.py b/scripts/face_detect.py
index e7a3eae..3ce2778 100644
--- a/scripts/face_detect.py
+++ b/scripts/face_detect.py
@@ -1,13 +1,10 @@
-import os
import sys
+from pathlib import Path
import numpy as np
-import torch
from diffusers.utils import load_image
# pip install insightface==0.7.3
from insightface.app import FaceAnalysis
-from insightface.data import get_image as ins_get_image
-from safetensors.torch import save_file
###
# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543
@@ -38,51 +35,25 @@ if __name__ == "__main__":
#face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition'])
face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition'])
face_detector.prepare(ctx_id=0, det_size=(640, 640))
- #input_folder_name = './scarletthead_woman'
- input_folder_name = sys.argv[1]
- image_basename_list = os.listdir(input_folder_name)
- image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])
-
- input_id_images = []
- for image_path in image_path_list:
- input_id_images.append(load_image(image_path))
-
- id_embed_list = []
-
- for img in input_id_images:
- img = np.array(img)
- img = img[:, :, ::-1]
- faces = analyze_faces(face_detector, img)
- if len(faces) > 0:
- id_embed_list.append(torch.from_numpy((faces[0]['embedding'])))
-
- if len(id_embed_list) == 0:
- raise ValueError(f"No face detected in input image pool")
-
- id_embeds = torch.stack(id_embed_list)
-
- # for r in id_embeds:
- # print(r)
- # #torch.save(id_embeds, input_folder_name+'/id_embeds.pt');
- # weights = dict()
- # weights["id_embeds"] = id_embeds
- # save_file(weights, input_folder_name+'/id_embeds.safetensors')
-
- binary_data = id_embeds.numpy().tobytes()
- two = 4
- zero = 0
- one = 1
- tensor_name = "id_embeds"
-# Write binary data to a file
- with open(input_folder_name+'/id_embeds.bin', "wb") as f:
- f.write(two.to_bytes(4, byteorder='little'))
- f.write((len(tensor_name)).to_bytes(4, byteorder='little'))
- f.write(zero.to_bytes(4, byteorder='little'))
- f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little'))
- f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little'))
- f.write(one.to_bytes(4, byteorder='little'))
- f.write(one.to_bytes(4, byteorder='little'))
- f.write(tensor_name.encode('ascii'))
- f.write(binary_data)
-
-
\ No newline at end of file
+ input_dir = Path(sys.argv[1])
+ image_extensions = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
+ image_paths = sorted(
+ (path for path in input_dir.iterdir()
+ if path.is_file() and path.suffix.lower() in image_extensions),
+ key=lambda path: path.name,
+ )
+ if not image_paths:
+ raise ValueError(f"No input images in '{input_dir}'")
+
+ embeddings = []
+ for image_path in image_paths:
+ image = np.asarray(load_image(str(image_path)))[:, :, ::-1]
+ faces = analyze_faces(face_detector, image)
+ if not faces:
+ raise ValueError(f"No face detected in '{image_path}'")
+ embedding = np.asarray(faces[0]["embedding"], dtype=np.float32)
+ if embedding.shape != (512,):
+ raise ValueError(f"Expected a 512-value face embedding for '{image_path}'")
+ embeddings.append(embedding)
+
+ np.save(input_dir / "id_embeds.npy", np.stack(embeddings), allow_pickle=False)
diff --git a/scripts/format-code.ps1 b/scripts/format-code.ps1
index 782e092..c5f87f6 100644
--- a/scripts/format-code.ps1
+++ b/scripts/format-code.ps1
@@ -27,8 +27,16 @@ $patterns = @(
"src/extensions/*.cpp"
"src/extensions/*.h"
"src/extensions/*.hpp"
- "src/pipeline/*.cpp"
- "src/pipeline/*.h"
+ "src/dgl/*.cpp"
+ "src/dgl/*.h"
+ "src/dgl/syntax/*.cpp"
+ "src/dgl/syntax/*.h"
+ "src/dgl/bindings/*.cpp"
+ "src/dgl/bindings/*.h"
+ "src/dgl/models/*.cpp"
+ "src/dgl/models/*.h"
+ "src/dgl/ops/*.cpp"
+ "src/dgl/ops/*.h"
"src/runtime/*.cpp"
"src/runtime/*.h"
"src/runtime/*.hpp"
diff --git a/scripts/format-code.sh b/scripts/format-code.sh
index c2806f6..74ae208 100644
--- a/scripts/format-code.sh
+++ b/scripts/format-code.sh
@@ -16,7 +16,11 @@ for f in src/*.cpp src/*.h src/*.hpp \
src/core/probe/*.cpp src/core/probe/*.h \
src/core/vulkan/*.cpp src/core/vulkan/*.h src/core/vulkan/shaders/*.comp \
src/extensions/*.cpp src/extensions/*.h src/extensions/*.hpp \
- src/pipeline/*.cpp src/pipeline/*.h \
+ src/dgl/*.cpp src/dgl/*.h \
+ src/dgl/syntax/*.cpp src/dgl/syntax/*.h \
+ src/dgl/bindings/*.cpp src/dgl/bindings/*.h \
+ src/dgl/models/*.cpp src/dgl/models/*.h \
+ src/dgl/ops/*.cpp src/dgl/ops/*.h \
src/runtime/*.cpp src/runtime/*.h src/runtime/*.hpp \
src/model/*/*.cpp src/model/*/*.h src/model/*/*.hpp \
src/tokenizers/*.h src/tokenizers/*.cpp \
diff --git a/scripts/pulid_extract_id.py b/scripts/pulid_extract_id.py
index a5740b2..05199be 100644
--- a/scripts/pulid_extract_id.py
+++ b/scripts/pulid_extract_id.py
@@ -1,8 +1,8 @@
"""
Precompute a PuLID-Flux identity embedding from a single source portrait.
-Writes a gguf file (a single tensor `pulid_id`) that stable-diffusion.cpp's
-`--pulid-id-embedding` flag consumes.
+Writes a gguf file (a single tensor `pulid_id`) consumed by the DGL
+`pulid::load_embedding` function.
Dependencies (recommended: vendor rather than pip-install due to upstream
packaging quirks):
diff --git a/src/api/api.cpp b/src/api/api.cpp
index 72cd6d4..7eba378 100644
--- a/src/api/api.cpp
+++ b/src/api/api.cpp
@@ -1,4 +1,5 @@
#include "api.h"
+#include "generation.h"
#include "options.h"
#include "result.h"
#include "tensor.h"
@@ -10,11 +11,12 @@
#include "core/ggml_extend.h"
#include "core/option_types.h"
#include "core/util.h"
-#include "pipeline/diffusion_engine.h"
-#include "pipeline/generation.h"
static thread_local char last_error[512] = {};
+sd_ctx_t::sd_ctx_t() = default;
+sd_ctx_t::~sd_ctx_t() = default;
+
void sd::api::clear_error() {
last_error[0] = '\0';
}
@@ -69,177 +71,140 @@ sd_status_t sd_set_preview_callback(sd_preview_cb_t cb, const char* mode, int in
return SD_STATUS_OK;
}
-void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) {
- if (ctx == nullptr || ctx->sd == nullptr) {
+void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t callback, void* data) {
+ if (!ctx || !ctx->engine)
return;
- }
- std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
- if (!ctx->sd->executing_) {
- ctx->sd->prompt_image_callback = cb;
- ctx->sd->prompt_image_callback_data = data;
+ auto& engine = *ctx->engine;
+ std::lock_guard<std::recursive_mutex> lock(engine.mutex);
+ if (!engine.executing) {
+ engine.prompt_image = callback;
+ engine.prompt_image_data = data;
}
}
-void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) {
- if (ctx == nullptr || ctx->sd == nullptr) {
+void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t callback, void* data) {
+ if (!ctx || !ctx->engine)
return;
+ auto& engine = *ctx->engine;
+ std::lock_guard<std::recursive_mutex> lock(engine.mutex);
+ if (!engine.executing) {
+ engine.progress = callback;
+ engine.progress_data = data;
+ engine.sample_pass = 0;
}
- std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
- if (!ctx->sd->executing_) {
- ctx->sd->generation_progress.set_callback(cb, data);
- }
-}
-
-static sd_ctx_t* create_context(const sd::ContextConfig& params) {
- auto engine = std::make_unique<StableDiffusionGGML>();
- if (!engine->init(¶ms))
- return nullptr;
- auto context = std::make_unique<sd_ctx_t>();
- context->sd = engine.release();
- return context.release();
}
-sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out) {
- if (ctx_out)
- *ctx_out = nullptr;
+sd_status_t sd_ctx_create(const sd_ctx_opt_t* options, sd_ctx_t** out) {
+ if (out)
+ *out = nullptr;
return sd::api::call([&] {
- if (!ctx_out)
+ if (!out)
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "ctx_out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(params, sd::api::Context, config);
- const auto& context = config.context;
- if (status != SD_STATUS_OK)
- return status;
- *ctx_out = create_context(context);
- return *ctx_out ? SD_STATUS_OK : sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "context creation failed; see log for details");
+ auto ctx = std::make_unique<sd_ctx_t>();
+ ctx->engine = std::make_unique<sd::api::GenerationContext>(options, sd::api::Context);
+ *out = ctx.release();
+ return SD_STATUS_OK;
},
SD_STATUS_LOAD_FAILED);
}
-void sd_ctx_free(sd_ctx_t* sd_ctx) {
- if (!sd_ctx)
- return;
- if (sd_ctx->sd != nullptr) {
- delete sd_ctx->sd;
- sd_ctx->sd = nullptr;
- }
- delete sd_ctx;
+void sd_ctx_free(sd_ctx_t* context) {
+ delete context;
}
-SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) {
- if (sd_ctx && sd_ctx->sd) {
- if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) {
- mode = SD_CANCEL_ALL;
- }
- sd_ctx->sd->set_cancel_flag(mode);
- }
-}
-
-SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- return sd_version_supports_image_generation(sd_ctx->sd->version);
+sd_status_t sd_ctx_prepare(sd_ctx_t* context) {
+ return sd::api::call([&] {
+ if (!context || !context->engine)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context is required");
+ auto& engine = *context->engine;
+ sd::api::GenerationRun preparation(engine);
+ sd::FileAccessScope access(engine.execution.file_access);
+ sd::api::execute_program(engine, sd::api::component_program(engine));
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
}
-SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) {
- return true;
- }
- return sd_version_supports_video_generation(sd_ctx->sd->version);
+void sd_cancel_generation(sd_ctx_t* context, enum sd_cancel_mode_t mode) {
+ if (!context || !context->engine)
+ return;
+ if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET)
+ mode = SD_CANCEL_ALL;
+ auto& engine = *context->engine;
+ std::lock_guard<std::recursive_mutex> lock(engine.mutex);
+ if (!engine.executing)
+ return;
+ engine.cancellation.store(mode);
+ if (engine.active_run && mode != SD_CANCEL_NEW_LATENTS)
+ engine.active_run->set_cancelled(mode == SD_CANCEL_ALL);
+}
+
+static const ModelDefinition* context_definition(const sd_ctx_t* context) {
+ const ModelDefinition* definition = nullptr;
+ sd::api::call([&] {
+ if (!context || !context->engine)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context is required");
+ auto& engine = *context->engine;
+ const auto family = engine.family.load();
+ if (family != VERSION_COUNT) {
+ definition = find_model_definition(family);
+ } else {
+ std::lock_guard<std::recursive_mutex> lock(engine.mutex);
+ sd::FileAccessScope access(engine.execution.file_access);
+ definition = &engine.model_definition();
+ }
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
+ return definition;
}
-SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
- return false;
- }
- return sd_ctx->sd->load_control_net_from_file(path);
+bool sd_ctx_supports_image_generation(const sd_ctx_t* context) {
+ const auto* definition = context_definition(context);
+ return definition && !definition->traits.supports_video_generation && definition->version != VERSION_ESRGAN && definition->version != VERSION_SEEDVR2 && definition->version != VERSION_RELAY_VSR;
}
-SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- return sd_ctx->sd->unload_control_net();
+bool sd_ctx_supports_video_generation(const sd_ctx_t* context) {
+ const auto* definition = context_definition(context);
+ return definition && definition->traits.supports_video_generation && definition->version != VERSION_SEEDVR2 && definition->version != VERSION_RELAY_VSR;
}
-SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- if (sd_ctx->sd->control_net != nullptr) {
- return true;
- }
- // z-image keeps control net weights inside the diffusion runner, registered
- // under ModelComponent::ControlNet instead of a separate control_net runner.
- return sd_ctx->sd->model_manager != nullptr &&
- sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0;
+const char* sd_get_model_version_name(const sd_ctx_t* context) {
+ const auto* definition = context_definition(context);
+ return definition ? definition->name : "Unknown";
}
-const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) {
- return "Unknown";
- }
- return sd_version_name(sd_ctx->sd->version);
+sd_status_t sd_ctx_set_file_access_callback(sd_ctx_t* context, sd_dgl_file_access_cb_t callback, void* data) {
+ return sd::api::call([&] {
+ if (!context || !context->engine)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context is required");
+ auto& engine = *context->engine;
+ std::unique_lock<std::recursive_mutex> lock(engine.mutex, std::try_to_lock);
+ if (!lock.owns_lock() || engine.executing)
+ return sd::api::error(SD_STATUS_BUSY, nullptr, "context is busy");
+ engine.execution.file_access = {callback, data};
+ return SD_STATUS_OK;
+ });
}
-sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out) {
- if (result_out)
- *result_out = nullptr;
+sd_status_t sd_generate_image(sd_ctx_t* context, const sd_img_gen_opt_t* params, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
return sd::api::call([&] {
- if (!result_out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(params, sd::api::Image, config);
- const auto& image = config.image;
- if (status != SD_STATUS_OK)
- return status;
- sd::ImageGenerationResult* images = nullptr;
- status = sd::generate_image(ctx, image, &images, params);
- if (status != SD_STATUS_OK)
- return status;
- return sd::make_image_result(images, result_out) ? SD_STATUS_OK
- : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ if (!out || !context || !context->engine)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context and result_out are required");
+ return sd::api::generate_images(*context->engine, params, out);
},
SD_STATUS_GENERATION_FAILED);
}
-sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out) {
- if (result_out)
- *result_out = nullptr;
+sd_status_t sd_generate_video(sd_ctx_t* context, const sd_vid_gen_opt_t* params, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
return sd::api::call([&] {
- if (!result_out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(params, sd::api::Video, config);
- const auto& video = config.video;
- if (status != SD_STATUS_OK)
- return status;
- sd::VideoGenerationResult* frames = nullptr;
- status = sd::generate_video(ctx, video, &frames, params);
- if (status != SD_STATUS_OK)
- return status;
- return sd::make_video_result(frames, result_out) ? SD_STATUS_OK
- : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ if (!out || !context || !context->engine)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context and result_out are required");
+ return sd::api::generate_video(*context->engine, params, out);
},
SD_STATUS_GENERATION_FAILED);
}
-
-sd_status_t sd_preprocess_canny(const sd_tensor_t* image, float high_threshold, float low_threshold, float weak, float strong, bool inverse, sd_tensor_t** image_out) {
- if (image_out)
- *image_out = nullptr;
- return sd::api::call([&] {
- if (!image || !image_out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "image and image_out are required");
- sd::api::image_view(*image);
- auto output = sd::api::allocate_tensor(image->type, image->shape);
- memcpy(output.get()->mutable_data(), image->data(), image->bytes);
- if (!sd::preprocess_canny(sd::api::image_view(*output.get()), high_threshold, low_threshold, weak, strong, inverse))
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "Canny preprocessing failed");
- *image_out = sd_tensor_retain(output.get());
- return SD_STATUS_OK;
- });
-}
diff --git a/src/api/api.h b/src/api/api.h
index 545859e..5fbca07 100644
--- a/src/api/api.h
+++ b/src/api/api.h
@@ -2,14 +2,20 @@
#define SD_INTERNAL_API_H
#include <exception>
+#include <memory>
#include <new>
+#include "dgl/diagnostic.h"
#include "stable-diffusion.h"
-class StableDiffusionGGML;
+namespace sd::api {
+ struct GenerationContext;
+}
struct sd_ctx_t {
- StableDiffusionGGML* sd = nullptr;
+ std::unique_ptr<sd::api::GenerationContext> engine;
+ sd_ctx_t();
+ ~sd_ctx_t();
};
namespace sd::api {
@@ -21,6 +27,8 @@ namespace sd::api {
clear_error();
try {
return function();
+ } catch (const sd::dgl::Error& exception) {
+ return error(exception.status, nullptr, exception.what());
} catch (const std::bad_alloc&) {
return error(SD_STATUS_OUT_OF_MEMORY, nullptr, "out of memory");
} catch (const std::exception& exception) {
diff --git a/src/api/generation.cpp b/src/api/generation.cpp
new file mode 100644
index 0000000..51a3630
--- /dev/null
+++ b/src/api/generation.cpp
@@ -0,0 +1,241 @@
+#include "api/generation.h"
+
+#include <algorithm>
+#include <cmath>
+#include <filesystem>
+#include <random>
+
+#include "api/generation_program.h"
+#include "api/result.h"
+#include "api/tensor.h"
+#include "core/ggml_extend_backend.h"
+#include "core/image_geometry.h"
+#include "core/util.h"
+#include "dgl/diagnostic.h"
+#include "dgl/executor.h"
+#include "dgl/models/component_sources.h"
+#include "model/vae/geometry.hpp"
+#include "name_conversion.h"
+
+namespace sd::api {
+ GenerationContext::GenerationContext(const sd_opt_t* source, Operation purpose)
+ : options(copy_options(source)), execution(true) {
+ const auto status = resolve(options.get(), purpose, config);
+ if (status != SD_STATUS_OK)
+ throw dgl::Error(status, sd_get_last_error());
+ const auto& settings = config.context;
+ execution.set("n_threads", std::to_string(settings.n_threads));
+ execution.set("conditioning_cache_size", std::to_string(settings.conditioning_cache_size));
+ for (const auto& [key, enabled] : std::map<std::string, bool>{
+ {"auto_fit", settings.auto_fit}, {"batched_cfg", settings.batched_cfg}, {"enable_mmap", settings.enable_mmap}, {"eager_load", settings.eager_load}, {"disable_prefetch", settings.disable_prefetch}, {"disable_segmented_compute", settings.disable_segmented_compute}})
+ execution.set(key, enabled ? "true" : "false");
+ if (settings.max_vram && *settings.max_vram)
+ execution.set("max_vram", settings.max_vram);
+ if (settings.max_ram && *settings.max_ram)
+ execution.set("max_ram", settings.max_ram);
+ if (settings.rpc_servers && *settings.rpc_servers && !add_rpc_devices(settings.rpc_servers))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot register RPC devices");
+ }
+
+ static bool contains_audio_vae(const ModelLoader& metadata, SDVersion version) {
+ for (const auto& name : metadata.get_tensor_names()) {
+ const auto canonical = convert_tensor_name(name, version);
+ if (canonical.rfind("audio_vae.", 0) == 0 || canonical.rfind("vocoder.", 0) == 0)
+ return true;
+ }
+ return false;
+ }
+
+ const ModelDefinition& GenerationContext::model_definition() {
+ std::lock_guard<std::recursive_mutex> lock(mutex);
+ check_cancelled();
+ const auto& settings = config.context;
+ const auto* path = settings.diffusion_model_path && *settings.diffusion_model_path ? settings.diffusion_model_path : settings.model_path;
+ if (!path || !*path)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "model_path or diffusion_model_path is required");
+ require_file_access(path, SD_DGL_FILE_READ_MODEL);
+ if ((sd_version_is_minimax_h3(family) || sd_version_is_ltxav(family)) && settings.vae_path && *settings.vae_path)
+ require_file_access(settings.vae_path, SD_DGL_FILE_READ_MODEL);
+ bool changed = family == VERSION_COUNT;
+ if (!changed && !ModelLoader::dependencies_changed(probe_dependencies, changed))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot inspect model dependencies");
+ if (changed) {
+ try {
+ ModelLoader::Dependencies dependencies;
+ ModelLoader metadata;
+ auto detected = dgl::models::resolve_family(VERSION_COUNT, path, &dependencies, &metadata);
+ ModelLoader vae_metadata;
+ bool inspected_vae = false;
+ if (detected == VERSION_MINIMAX_H3 && settings.vae_path && *settings.vae_path && std::string(settings.vae_path) != path) {
+ require_file_access(settings.vae_path, SD_DGL_FILE_READ_MODEL);
+ if (!vae_metadata.init_from_file(settings.vae_path))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot inspect video VAE metadata");
+ inspected_vae = true;
+ const auto sources = vae_metadata.dependencies();
+ dependencies.insert(dependencies.end(), sources.begin(), sources.end());
+ for (const auto& [name, tensor] : vae_metadata.get_tensor_storage_map())
+ if (ends_with(name, "decoder.proj_out.weight") && tensor.ne[1] == 768)
+ detected = VERSION_P_H3_IMAGE;
+ }
+ const auto* definition = find_model_definition(detected);
+ if (!definition)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "unsupported model family");
+ default_prediction = definition->sampling.prediction_from_weights ? definition->sampling.prediction_from_weights(metadata.get_tensor_storage_map()) : definition->sampling.default_prediction;
+ wan_image_condition = detected == VERSION_WAN2_2_I2V;
+ wan_vision_condition = wan_end_vision = wan_vace = false;
+ if (sd_version_is_wan(detected))
+ for (const auto& [name, tensor] : metadata.get_tensor_storage_map()) {
+ wan_vision_condition |= name.find("img_emb.") != std::string::npos;
+ wan_end_vision |= name.find("img_emb.emb_pos") != std::string::npos;
+ wan_vace |= name.find("vace_blocks.") != std::string::npos;
+ }
+ wan_image_condition |= wan_vision_condition;
+ embedded_audio_vae.clear();
+ if (sd_version_is_ltxav(detected) || sd_version_is_minimax_h3(detected)) {
+ if (contains_audio_vae(metadata, detected)) {
+ embedded_audio_vae = path;
+ } else if (settings.vae_path && *settings.vae_path && std::string(settings.vae_path) != path &&
+ (!settings.audio_vae_path || !*settings.audio_vae_path)) {
+ require_file_access(settings.vae_path, SD_DGL_FILE_READ_MODEL);
+ if (!inspected_vae && !vae_metadata.init_from_file(settings.vae_path))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot inspect video VAE metadata");
+ if (!inspected_vae) {
+ const auto sources = vae_metadata.dependencies();
+ dependencies.insert(dependencies.end(), sources.begin(), sources.end());
+ }
+ if (contains_audio_vae(vae_metadata, detected))
+ embedded_audio_vae = settings.vae_path;
+ }
+ }
+ check_cancelled();
+ family = detected;
+ probe_dependencies = std::move(dependencies);
+ LOG_INFO("Detected model family: %s (%s)", sd_version_name(family), std::filesystem::u8path(path).filename().u8string().c_str());
+ } catch (const dgl::Error&) {
+ family = VERSION_COUNT;
+ throw;
+ } catch (const std::exception& error) {
+ family = VERSION_COUNT;
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, error.what());
+ }
+ }
+ const auto* definition = find_model_definition(family);
+ if (!definition)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "unsupported model family");
+ return *definition;
+ }
+
+ void GenerationContext::check_cancelled() const {
+ if (cancellation.load(std::memory_order_relaxed) == SD_CANCEL_ALL)
+ throw dgl::Error(SD_STATUS_CANCELLED, "generation cancelled");
+ }
+
+ GenerationRun::GenerationRun(GenerationContext& context)
+ : context_(context) {
+ std::unique_lock<std::recursive_mutex> lock(context_.mutex, std::try_to_lock);
+ if (!lock.owns_lock() || context_.executing)
+ throw dgl::Error(SD_STATUS_BUSY, "context is busy");
+ context_.executing = true;
+ }
+
+ GenerationRun::~GenerationRun() {
+ std::lock_guard<std::recursive_mutex> lock(context_.mutex);
+ context_.active_run = nullptr;
+ context_.execution.active_run = nullptr;
+ context_.cancellation.store(SD_CANCEL_RESET);
+ context_.executing = false;
+ }
+
+ static void generation_progress(const sd_generation_progress_t* info, void* data) {
+ auto& context = *static_cast<GenerationContext*>(data);
+ auto translated = *info;
+ translated.sample_pass += context.sample_pass;
+ if (context.progress)
+ context.progress(&translated, context.progress_data);
+ }
+
+ dgl::Value execute_program(GenerationContext& context, const GenerationProgram& program, const dgl::Graph* compiled) {
+ context.check_cancelled();
+ FileAccessScope access(context.execution.file_access);
+ dgl::Graph graph;
+ if (!compiled) {
+ graph = dgl::compile(program.source, dgl::operations(), !program.inputs.empty());
+ compiled = &graph;
+ }
+ context.check_cancelled();
+ dgl::RunCallbacks callbacks;
+ callbacks.progress = generation_progress;
+ callbacks.progress_data = &context;
+ dgl::RunEvents events(callbacks, true);
+ struct ActiveRun {
+ GenerationContext& context;
+ dgl::RunEvents& events;
+ ~ActiveRun() {
+ std::lock_guard<std::recursive_mutex> lock(context.mutex);
+ context.sample_pass += events.preview_info().sample_pass;
+ context.active_run = nullptr;
+ context.execution.active_run = nullptr;
+ }
+ } active{context, events};
+ {
+ std::lock_guard<std::recursive_mutex> lock(context.mutex);
+ context.active_run = &events;
+ context.execution.active_run = &events;
+ if (context.cancellation.load() == SD_CANCEL_ALL)
+ events.cancel();
+ }
+ dgl::RunEventsScope scope(events);
+ dgl::RunState state{context.execution, events};
+ LOG_DEBUG("Executing DGL program:\n%s", compiled->source.c_str());
+ return dgl::run(*compiled, state, program.inputs);
+ }
+
+ static int64_t resolved_seed(int64_t requested) {
+ if (requested >= 0)
+ return requested;
+ std::random_device entropy;
+ return int64_t((uint64_t(entropy()) << 32 | entropy()) & INT64_MAX);
+ }
+
+ sd_status_t generate_images(GenerationContext& context, const sd_img_gen_opt_t* options, sd_result_t** out) {
+ GenerationRun request(context);
+ FileAccessScope access(context.execution.file_access);
+ auto snapshot = copy_options(options);
+ OptConfig config;
+ const auto status = resolve(snapshot.get(), Image, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ context.check_cancelled();
+ const auto seed = resolved_seed(config.image.seed);
+ auto result = std::make_unique<sd_result_t>();
+ for (int i = 0; i < config.image.batch_count; ++i) {
+ if (context.cancellation.load() == SD_CANCEL_NEW_LATENTS)
+ break;
+ auto program = image_program(context, config.image, snapshot.get(), int64_t((uint64_t(seed) + uint64_t(i)) & INT64_MAX));
+ auto value = execute_program(context, program);
+ append_program_image(*result, value, program.source);
+ }
+ if (result->images.empty())
+ throw dgl::Error(SD_STATUS_CANCELLED, "generation cancelled before the next image");
+ context.check_cancelled();
+ *out = result.release();
+ return SD_STATUS_OK;
+ }
+ sd_status_t generate_video(GenerationContext& context, const sd_vid_gen_opt_t* options, sd_result_t** out) {
+ GenerationRun request(context);
+ FileAccessScope access(context.execution.file_access);
+ auto snapshot = copy_options(options);
+ OptConfig config;
+ const auto status = resolve(snapshot.get(), Video, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ auto program = video_program(context, config.video, snapshot.get(), resolved_seed(config.video.seed));
+ auto value = execute_program(context, program);
+ auto result = std::make_unique<sd_result_t>();
+ set_program_video(*result, value, program.source);
+ context.check_cancelled();
+ *out = result.release();
+ return SD_STATUS_OK;
+ }
+
+}
diff --git a/src/api/generation.h b/src/api/generation.h
new file mode 100644
index 0000000..cb1cc8f
--- /dev/null
+++ b/src/api/generation.h
@@ -0,0 +1,66 @@
+#ifndef __SD_API_GENERATION_H__
+#define __SD_API_GENERATION_H__
+
+#include <atomic>
+#include <map>
+#include <mutex>
+#include <string>
+
+#include "api/generation_template.h"
+#include "api/options.h"
+#include "dgl/context.h"
+#include "dgl/graph.h"
+#include "model_definition.h"
+
+namespace sd::api {
+ struct GenerationContext {
+ std::shared_ptr<sd_opt_t> options;
+ OptConfig config;
+ dgl::Context execution;
+ std::recursive_mutex mutex;
+ bool executing = false;
+ std::atomic<sd_cancel_mode_t> cancellation{SD_CANCEL_RESET};
+ dgl::RunEvents* active_run = nullptr;
+ sd_generation_progress_cb_t progress = nullptr;
+ void* progress_data = nullptr;
+ sd_prompt_image_cb_t prompt_image = nullptr;
+ void* prompt_image_data = nullptr;
+ int sample_pass = 0;
+ std::atomic<SDVersion> family{VERSION_COUNT};
+ prediction_t default_prediction = PREDICTION_COUNT;
+ bool wan_image_condition = false;
+ bool wan_vision_condition = false;
+ bool wan_end_vision = false;
+ bool wan_vace = false;
+ std::string embedded_audio_vae;
+ ModelLoader::Dependencies probe_dependencies;
+
+ GenerationContext(const sd_opt_t* source, Operation purpose);
+ const ModelDefinition& model_definition();
+ void check_cancelled() const;
+ };
+
+ class GenerationRun {
+ GenerationContext& context_;
+
+ public:
+ explicit GenerationRun(GenerationContext& context);
+ ~GenerationRun();
+ GenerationRun(const GenerationRun&) = delete;
+ GenerationRun& operator=(const GenerationRun&) = delete;
+ };
+
+ struct GenerationProgram {
+ std::string source;
+ dgl::Values inputs;
+ };
+
+ dgl::Value execute_program(GenerationContext& context, const GenerationProgram& program, const dgl::Graph* compiled = nullptr);
+ GenerationProgram component_program(GenerationContext& context);
+ GenerationProgram image_program(GenerationContext& context, const ImageConfig& image, const sd_opt_t* explicit_options, int64_t seed);
+ GenerationProgram video_program(GenerationContext& context, const VideoConfig& video, const sd_opt_t* explicit_options, int64_t seed);
+ sd_status_t generate_images(GenerationContext& context, const sd_img_gen_opt_t* options, sd_result_t** result);
+ sd_status_t generate_video(GenerationContext& context, const sd_vid_gen_opt_t* options, sd_result_t** result);
+}
+
+#endif // __SD_API_GENERATION_H__
diff --git a/src/api/generation_h3.cpp b/src/api/generation_h3.cpp
new file mode 100644
index 0000000..766c2ec
--- /dev/null
+++ b/src/api/generation_h3.cpp
@@ -0,0 +1,117 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cmath>
+
+namespace sd::api {
+ static std::string expression_list(const std::vector<std::string>& values) {
+ std::string result = "[";
+ for (size_t i = 0; i < values.size(); ++i)
+ result += (i ? ", " : "") + values[i];
+ return result + "]";
+ }
+
+ void prepare_h3_references(GenerationTemplate& request, const std::string& first, const std::string& last) {
+ const auto images = bound_tensors(request.options, "ref_images");
+ const auto videos = bound_videos(request.options, "ref_videos");
+ const auto audios = bound_audios(request.options, "ref_audios");
+ const bool references = !images.empty() || !videos.empty() || !audios.empty();
+ const bool audio_references = !audios.empty() || std::any_of(videos.begin(), videos.end(), [](const sd_video_t& video) { return video.audio.data != nullptr; });
+ if (audio_references && !request.flags.at("decode_audio").get<bool>())
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "MiniMax-H3 audio references require audio VAE weights");
+ if (references && (!first.empty() || !last.empty()))
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "MiniMax-H3 references and start/end keyframes cannot be combined");
+ if (!references && first.empty() && last.empty())
+ return;
+ auto& prepare = request.fields["references"];
+ std::vector<std::string> visuals, sounds, presentation, blocks, keys;
+ const auto encode_visual = [&](const std::string& pixels, const nlohmann::json& options = nlohmann::json::object()) {
+ const auto name = "h3_visual_" + std::to_string(visuals.size());
+ nlohmann::json parameters{{"name", name}, {"pixels", pixels}, {"encode_audio", false}, {"resize_video", false}, {"resize_image", false}};
+ parameters.update(options);
+ prepare += request.render("video_reference", parameters);
+ visuals.push_back(name);
+ };
+ const auto encode_sound = [&](const sd_audio_t& audio) {
+ const auto name = "h3_sound_" + std::to_string(sounds.size());
+ bind_tensor(request.program, name + "_samples", audio.data);
+ const auto media = "{samples = " + name + "_samples, sample_rate = " + std::to_string(audio.sample_rate) + "}";
+ prepare += request.render("video_reference", {{"encode_audio", true}, {"name", name}, {"sample_rate", audio.sample_rate}});
+ sounds.push_back(name);
+ presentation.push_back("{audio = " + media + "}");
+ };
+ if (!references) {
+ std::vector<std::string> key_images;
+ for (const auto& [pixels, index] : std::vector<std::pair<std::string, std::string>>{{first, "0"}, {last, "frames - 1"}}) {
+ if (pixels.empty())
+ continue;
+ encode_visual(pixels);
+ key_images.push_back(pixels + "[:, :, 0]");
+ keys.push_back(index);
+ }
+ request.fields["prompt_options"] += ", media = " + expression_list(key_images);
+ request.fields["negative_prompt_options"] += ", media = " + expression_list(key_images);
+ }
+ for (size_t i = 0; i < images.size(); ++i) {
+ const auto name = "h3_image_" + std::to_string(i);
+ bind_tensor(request.program, name, images[i]);
+ const int source_width = int(sd_tensor_dim(images[i], 1)), source_height = int(sd_tensor_dim(images[i], 2));
+ const double scale = std::min(1.0, std::sqrt(double(request.values.at("width").get<int>()) * request.values.at("height").get<int>() / (double(source_width) * source_height)));
+ const int width = std::max(32, int(std::round(source_width * scale / 32)) * 32), height = std::max(32, int(std::round(source_height * scale / 32)) * 32);
+ blocks.push_back(literal({{"kind", "image"}, {"video", visuals.size()}}));
+ encode_visual(name + "_pixels", {{"resize_image", true}, {"source", name}, {"width", width}, {"height", height}});
+ presentation.push_back("{image = " + name + "_pixels}");
+ }
+ for (size_t i = 0; i < videos.size(); ++i) {
+ const auto& video = videos[i];
+ const auto name = "h3_video_" + std::to_string(i);
+ bind_tensor(request.program, name, video.data);
+ const int count = int(sd_tensor_dim(video.data, 3));
+ int frames = std::min(request.values.at("frames").get<int>(), int(std::lround(double(count) * 24 / video.fps)));
+ if (frames < 5)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "MiniMax-H3 reference video needs at least 5 frames at 24 fps");
+ frames = (frames - 5) / 17 * 17 + 5;
+ std::vector<int> indices;
+ for (int frame = 0; frame < frames; ++frame)
+ indices.push_back(std::min(count - 1, int(std::floor(double(frame) * video.fps / 24))));
+ const int source_width = int(sd_tensor_dim(video.data, 1)), source_height = int(sd_tensor_dim(video.data, 2));
+ const double ratio = double(source_width) / source_height;
+ double w = ratio >= 1 ? 768 * ratio : 768, h = ratio >= 1 ? 768 : 768 / ratio;
+ if (w * h > 768.0 * 1344) {
+ const auto scale = std::sqrt(768.0 * 1344 / (w * h));
+ w *= scale;
+ h *= scale;
+ }
+ int width = std::max(32, int(std::round(w / 32)) * 32), height = std::max(32, int(std::round(h / 32)) * 32);
+ if (source_width * source_height < width * height) {
+ width = std::max(32, int(std::round(double(source_width) / 32)) * 32);
+ height = std::max(32, int(std::round(double(source_height) / 32)) * 32);
+ }
+ auto block = nlohmann::json{{"kind", video.audio.data ? "video_audio" : "video"}, {"video", visuals.size()}};
+ if (video.audio.data) {
+ block["audio"] = sounds.size();
+ encode_sound(video.audio);
+ }
+ blocks.push_back(literal(block));
+ encode_visual(name + "_pixels", {{"resize_video", true}, {"source", name}, {"indices", literal(indices)}, {"width", width}, {"height", height}});
+ presentation.push_back("{video = {frames = " + name + "_pixels, fps_num = 24, fps_den = 1}, stride = 12}");
+ }
+ for (const auto& audio : audios) {
+ blocks.push_back(literal({{"kind", "audio"}, {"audio", sounds.size()}}));
+ encode_sound(audio);
+ }
+ std::string options = ", ref_latents = " + expression_list(visuals);
+ if (!sounds.empty())
+ options += ", ref_audios = " + expression_list(sounds);
+ if (!blocks.empty())
+ options += ", reference_blocks = " + expression_list(blocks);
+ if (!keys.empty())
+ options += ", keyframe_indices = " + expression_list(keys);
+ request.fields["predictor_options"] += options;
+ request.fields["negative_predictor_options"] += options;
+ if (!presentation.empty()) {
+ request.fields["prompt_options"] += ", media = " + expression_list(presentation);
+ request.fields["negative_prompt_options"] += ", media = " + expression_list(presentation);
+ }
+ }
+}
diff --git a/src/api/generation_hires.cpp b/src/api/generation_hires.cpp
new file mode 100644
index 0000000..7d43f75
--- /dev/null
+++ b/src/api/generation_hires.cpp
@@ -0,0 +1,123 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "core/image_geometry.h"
+#include "model/vae/geometry.hpp"
+
+namespace sd::api {
+ void prepare_hires_program(GenerationTemplate& request) {
+ const auto& hires = request.image.hires;
+ const auto mode = text(hires.upscaler);
+ const bool enabled = hires.enabled && mode != "none";
+ request.flags["hires"] = enabled;
+ if (!enabled)
+ return;
+ static const std::map<std::string, std::string> modes{
+ {"latent", "bilinear"}, {"latent_nearest", "nearest"}, {"latent_nearest_exact", "nearest_exact"}, {"latent_antialiased", "bilinear"}, {"latent_bicubic", "bicubic"}, {"latent_bicubic_antialiased", "bicubic"}, {"lanczos", "lanczos"}, {"nearest", "nearest"}, {"model", "nearest"}};
+ const auto found = modes.find(mode);
+ if (found == modes.end())
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "unknown hires upscaler '" + mode + "'");
+ const bool model = mode == "model";
+ const bool pixels = !request.video && (model || mode == "lanczos" || mode == "nearest");
+ const auto version = request.definition.version;
+ const bool ltx = sd_version_is_ltxav(version);
+ if (request.video && !ltx && version != VERSION_MINIMAX_H3)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "video hires supports LTX and MiniMax-H3; other workflows can be written in DGL");
+ if (request.video && (mode == "lanczos" || mode == "nearest"))
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "video hires requires a latent interpolation mode or a latent upscaler model");
+ if (model && text(hires.model_path).empty())
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "hires model upscaling requires hires.model_path");
+ const auto& settings = request.context.config.context;
+ const auto vae_version = request.definition.traits.vae_format_override && request.fields.at("vae_loader") != "load_tae"
+ ? sd::vae::version_for_format(settings.vae_format, version)
+ : version;
+ const int scale = sd_version_is_pid(version) || sd_version_is_minit2i(version) || version == VERSION_IRIS ? 1 : sd::vae::spatial_scale(vae_version);
+ const int multiple = scale * request.definition.geometry.diffusion_down_factor;
+ const double width = hires.target_width ? hires.target_width : hires.target_height ? hires.target_height
+ : std::round(request.values.at("width").get<int>() * double(hires.scale));
+ const double height = hires.target_height ? hires.target_height : hires.target_width ? hires.target_width
+ : std::round(request.values.at("height").get<int>() * double(hires.scale));
+ if (width < 1 || height < 1 || width > 16384 || height > 16384)
+ throw dgl::Error(SD_STATUS_OUT_OF_RANGE, "hires target dimensions must be in 1..16384");
+ const auto& sampling = request.image.sample_params;
+ const int default_steps = sampling.custom_sigmas_count ? sampling.custom_sigmas_count - 1 : sampling.sample_steps;
+ const int steps = hires.steps > 0 ? hires.steps : default_steps;
+ const double scheduler_steps = steps / double(hires.denoising_strength);
+ if (!hires.custom_sigmas_count && scheduler_steps > 10000)
+ throw dgl::Error(SD_STATUS_OUT_OF_RANGE, "hires steps / denoising_strength exceeds 10000; supply hires.sigmas or increase the strength");
+ request.values.update({{"hires_target_width", sd::align_image_dimension(int(width), multiple)},
+ {"hires_target_height", sd::align_image_dimension(int(height), multiple)},
+ {"hires_steps", steps},
+ {"hires_scheduler_steps", std::max(1, int(std::min(scheduler_steps, 10000.0)))},
+ {"hires_mode", found->second},
+ {"hires_antialias", mode == "latent_antialiased" || mode == "latent_bicubic_antialiased"},
+ {"hires_vae_scale", scale}});
+ if (hires.custom_sigmas_count)
+ request.values["hires_custom_sigmas"] = std::vector<float>(hires.custom_sigmas, hires.custom_sigmas + hires.custom_sigmas_count);
+ if (model) {
+ request.values["hires_model_path"] = text(hires.model_path);
+ auto options = loader_options(settings, request.video ? SDBackendModule::DIFFUSION : SDBackendModule::UPSCALER, request.definition);
+ if (!request.video) {
+ auto placement = nlohmann::json::object();
+ for (const auto* key : {"backend", "params_backend"})
+ if (options.contains(key))
+ placement[key] = options.at(key);
+ options = std::move(placement);
+ options["tile_size"] = hires.upscale_tile_size;
+ } else
+ options["family"] = ltx ? "ltxav" : "minimax_h3";
+ request.fields["hires_model_options"] = arguments(options);
+ }
+ request.flags.update({{"hires_model", model}, {"hires_pixels", pixels}, {"hires_ltx", ltx}, {"hires_pixel_model", request.fields.at("entry") == "image_pixel"}, {"hires_sefi", sd_version_is_sefi_image(version)}, {"hires_h3_image", sd_version_is_minimax_h3_image(version)}, {"hires_custom_sigmas", hires.custom_sigmas_count > 0}});
+ request.fields["result_latent"] = "hires_latent";
+ request.fields["hires_sample_options"] = request.fields.at("sample_core");
+ request.fields["hires_schedule_predictor"] = request.fields.at("schedule_predictor");
+ if (request.video) {
+ request.fields["output_video_frames"] = "hires_video_shape[2]";
+ request.fields["output_audio_shape"] = "hires_audio_shape";
+ request.fields["hires_schedule_predictor"] = ltx ? "hires_guided" : "guided";
+ request.fields["hires_positions_option"] = request.flags.at("last").get<bool>() ? ", video_positions = hires_positions" : "";
+ if (ltx && request.flags.at("conditioned").get<bool>())
+ request.fields["hires_sample_options"] += ", base = hires_base, denoise_mask = hires_packed_mask, mask_input = true";
+ return;
+ }
+ const bool rebuild = request.flags.value("inpaint_model", false) || request.flags.value("unet_edit", false) || request.flags.value("native_control", false) || request.flags.value("z_control", false);
+ request.flags["hires_rebuild"] = rebuild;
+ request.fields["hires_positive_condition"] = request.fields.at("positive_condition");
+ request.fields["hires_negative_condition"] = request.fields.at("negative_condition");
+ request.fields["hires_guidance_options"] = request.fields.at("guidance_core");
+ if (request.flags.value("image_guidance", false)) {
+ request.fields["hires_guidance_options"] += ", image_only = hires_image_only, image_scale = image_cfg_scale";
+ request.fields["hires_image_only_condition"] = request.fields.at("image_only_condition");
+ }
+ if (request.flags.value("z_control", false)) {
+ request.fields["hires_positive_condition"] = "hires_controlled_positive";
+ request.fields["hires_negative_condition"] = "hires_controlled_negative";
+ if (request.flags.value("reference_image_guidance", false)) {
+ request.fields["hires_negative_condition"] = "hires_controlled_unconditional";
+ request.fields["hires_image_only_condition"] = "hires_controlled_negative";
+ }
+ }
+ if (rebuild) {
+ auto options = request.fields.at("predictor_core") + request.fields.at("reference_predictor_options");
+ if (request.flags.value("inpaint_model", false))
+ options += ", concat = hires_inpaint_condition";
+ else if (request.flags.value("unet_edit", false))
+ options += ", concat = hires_edit_latent";
+ else if (version == VERSION_FLUX_CONTROLS)
+ options += ", concat = hires_control_latent";
+ request.fields["hires_predictor_options"] = options;
+ request.fields["hires_image_only_options"] = options;
+ request.fields["hires_negative_predictor_options"] = request.flags.value("image_guidance", false)
+ ? request.fields.at("predictor_core") + (request.flags.value("reference_image_guidance", false) ? "" : ", concat = hires_image_unconditional_concat")
+ : options;
+ request.fields["hires_schedule_predictor"] = "hires_guided";
+ }
+ if (request.flags.value("mask_stage", false) || request.flags.value("has_mask", false)) {
+ request.fields["hires_mask"] = request.flags.value("inpaint_model", false) ? "denoise_mask" : "mask_latent";
+ request.fields["hires_sample_options"] += ", base = hires_base, denoise_mask = hires_mask";
+ }
+ }
+}
diff --git a/src/api/generation_image.cpp b/src/api/generation_image.cpp
new file mode 100644
index 0000000..cdaa3c6
--- /dev/null
+++ b/src/api/generation_image.cpp
@@ -0,0 +1,250 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "core/file_access.h"
+#include "core/image_geometry.h"
+#include "core/util.h"
+#include "model_loader.h"
+
+namespace sd::api {
+ static bool supports_reference_image_guidance(SDVersion version) {
+ return version == VERSION_FLUX || version == VERSION_CHROMA || sd_version_is_flux2(version) ||
+ (sd_version_is_qwen_image(version) && version != VERSION_QWEN_IMAGE_2_1) ||
+ sd_version_is_mage_flow(version) || sd_version_is_longcat(version) ||
+ sd_version_is_z_image(version) || sd_version_is_boogu_image(version);
+ }
+
+ static std::string reference_pixels(GenerationTemplate& request, const sd_tensor_t* tensor, size_t index, bool encode) {
+ const auto version = request.definition.version;
+ const auto name = "reference_" + std::to_string(index);
+ bind_tensor(request.program, name, tensor);
+ int width = int(sd_tensor_dim(tensor, 1)), height = int(sd_tensor_dim(tensor, 2));
+ const int target_width = request.values.at("width"), target_height = request.values.at("height");
+ const int factor = sd_version_is_qwen_image(version) ? 32 : 16;
+ if (version == VERSION_P_H3_IMAGE || version == VERSION_MINIMAX_H3_IMAGE) {
+ const double scale = version == VERSION_P_H3_IMAGE ? std::min(1.0, std::sqrt(double(target_width) * target_height / (double(width) * height))) : double(std::clamp(std::min(width, height), 1024, 2048)) / std::min(width, height);
+ width = std::max(32, int(std::nearbyint(width * scale / 32)) * 32);
+ height = std::max(32, int(std::nearbyint(height * scale / 32)) * 32);
+ } else if (version != VERSION_ANIMA && !sd_version_is_pid(version)) {
+ if (sd_version_is_llada_image(version) || sd_version_is_mage_flow(version) || (version == VERSION_FIBO2 && request.image.ref_images_count == 1)) {
+ width = target_width;
+ height = target_height;
+ } else {
+ const int pixels = version == VERSION_QWEN_IMAGE_2_1 ? target_width * target_height : 1024 * 1024;
+ if (version == VERSION_FIBO2) {
+ std::tie(width, height) = sd::select_image_size_bucket(width, height, pixels, factor, target_width, target_height);
+ } else {
+ const double w = std::sqrt(double(std::min(pixels, target_width * target_height)) * width / height);
+ height = std::max(factor, int(std::round(w * height / width / factor)) * factor);
+ width = std::max(factor, int(std::round(w / factor)) * factor);
+ }
+ }
+ }
+ const auto pixels = name + "_pixels";
+ const auto scale = std::max(double(width) / sd_tensor_dim(tensor, 1), double(height) / sd_tensor_dim(tensor, 2));
+ const auto resize_width = int(std::ceil(sd_tensor_dim(tensor, 1) * scale));
+ const auto resize_height = int(std::ceil(sd_tensor_dim(tensor, 2) * scale));
+ request.fields["references"] += request.render("image_reference", {{"name", name}, {"channels", request.definition.geometry.image_channels}, {"width", width}, {"height", height}, {"center_crop", version == VERSION_FIBO2}, {"resize_width", resize_width}, {"resize_height", resize_height}, {"crop_x", (resize_width - width) / 2}, {"crop_y", (resize_height - height) / 2}, {"mode", literal(sd_version_is_minimax_h3_image(version) ? "lanczos" : "bilinear")}, {"encode", encode}, {"augment", version == VERSION_MINIMAX_H3_IMAGE}, {"latent", "reference_latent_" + std::to_string(index)}});
+ return pixels;
+ }
+
+ static bool qwen_prompt_has_vision(const GenerationTemplate& request) {
+ if (request.definition.version == VERSION_QWEN_IMAGE_LAYERED)
+ return false;
+ const auto* vision_path = request.context.config.context.llm_vision_path;
+ if (vision_path && *vision_path)
+ return true;
+ const auto path = request.values.at("text_path").get<std::string>();
+ require_file_access(path, SD_DGL_FILE_READ_MODEL);
+ ModelLoader metadata;
+ if (!metadata.init_from_file(path))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot inspect text encoder metadata: " + path);
+ const auto names = metadata.get_tensor_names();
+ return std::any_of(names.begin(), names.end(), [](const auto& name) {
+ return name.find("visual.") != std::string::npos;
+ });
+ }
+
+ static void image_references(GenerationTemplate& request) {
+ const auto version = request.definition.version;
+ const auto refs = bound_tensors(request.options, "ref_images");
+ if (refs.empty() && !sd_version_is_unet_edit(version))
+ return;
+ std::string pixels, latents;
+ const bool encode = version != VERSION_HIDREAM_O1;
+ for (size_t i = 0; i < refs.size(); ++i) {
+ const auto image = reference_pixels(request, refs[i], i, encode);
+ const auto latent = "reference_latent_" + std::to_string(i);
+ if (encode)
+ latents += (latents.empty() ? "" : ", ") + latent;
+ const auto prompt_image = version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_FIBO2 || sd_version_is_llada_image(version) ? image : "reference_" + std::to_string(i) + "_original";
+ pixels += (pixels.empty() ? "" : ", ") + prompt_image;
+ }
+ if (sd_version_is_minimax_h3_image(version)) {
+ auto blocks = nlohmann::json::array();
+ for (size_t i = 0; i < refs.size(); ++i)
+ blocks.push_back({{"kind", "image"}, {"video", i}});
+ request.fields["predictor_options"] += ", reference_blocks = " + literal(blocks);
+ request.fields["negative_predictor_options"] += ", reference_blocks = " + literal(blocks);
+ request.fields["reference_predictor_options"] += ", reference_blocks = " + literal(blocks);
+ }
+ if ((sd_version_is_qwen_image(version) && qwen_prompt_has_vision(request)) || sd_version_is_minimax_h3_image(version) ||
+ version == VERSION_FIBO2 || sd_version_is_llada_image(version) || sd_version_is_krea2(version) ||
+ sd_version_is_mage_flow(version) || sd_version_is_longcat(version) || sd_version_is_boogu_image(version)) {
+ request.fields["prompt_options"] += ", media = [" + pixels + "]";
+ if (!sd_version_is_llada_image(version) && version != VERSION_FIBO2)
+ request.fields["negative_prompt_options"] += ", media = [" + pixels + "]";
+ }
+ if (encode && !sd_version_is_unet_edit(version)) {
+ const bool increase = sd_version_is_flux2(version) || sd_version_is_flux3(version) ||
+ sd_version_is_qwen_image(version) || sd_version_is_minimax_h3_image(version) ||
+ version == VERSION_FIBO2 || sd_version_is_krea2(version) ||
+ version == VERSION_ANIMA || sd_version_is_mage_flow(version);
+ std::string options = ", ref_latents = [" + latents + "], ref_index_mode = " +
+ literal(version == VERSION_QWEN_IMAGE_LAYERED ? "decrease" : increase ? "increase"
+ : "fixed") +
+ ", force_ref_timestep_zero = " + literal(sd_version_is_llada_image(version) || sd_version_is_krea2(version));
+ request.fields["predictor_options"] += options;
+ request.fields["negative_predictor_options"] += options;
+ request.fields["reference_predictor_options"] += options;
+ }
+ if (sd_version_is_unet_edit(version)) {
+ request.fields["predictor_options"] += ", concat = edit_latent";
+ request.fields["negative_predictor_options"] += ", concat = edit_latent";
+ }
+ }
+
+ static void image_inpaint(GenerationTemplate& request) {
+ const bool inpaint = sd_version_is_inpaint(request.definition.version);
+ const auto* mask = bound_tensor(request.options, "mask_image");
+ if (!mask && !inpaint)
+ return;
+ if (mask) {
+ if (!request.image.init_image.data)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "mask_image requires init_image");
+ bind_tensor(request.program, "input_mask", mask);
+ }
+ if (inpaint) {
+ request.fields["predictor_options"] += ", concat = inpaint_condition";
+ request.fields["negative_predictor_options"] += ", concat = inpaint_condition";
+ }
+ request.fields["sample_options"] += inpaint ? ", base = base, denoise_mask = denoise_mask" : ", base = base, denoise_mask = mask_latent";
+ }
+
+ static void image_control(GenerationTemplate& request) {
+ const auto version = request.definition.version;
+ if (!request.image.control_image.data) {
+ if (version == VERSION_FLUX_CONTROLS) {
+ request.fields["predictor_options"] += ", concat = tensor::zeros_like(base)";
+ request.fields["negative_predictor_options"] += ", concat = tensor::zeros_like(base)";
+ }
+ return;
+ }
+ bind_tensor(request.program, "input_control", bound_tensor(request.options, "control_image"));
+ if (version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2) {
+ if (version == VERSION_FLUX_CONTROLS) {
+ request.fields["predictor_options"] += ", concat = control_latent";
+ request.fields["negative_predictor_options"] += ", concat = control_latent";
+ }
+ return;
+ }
+ const auto& settings = request.context.config.context;
+ if (!settings.control_net_path || !*settings.control_net_path)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "control_image requires control_net_path");
+ request.values["control_path"] = settings.control_net_path;
+ request.values["control_strength"] = request.image.control_strength;
+ request.fields["control_options"] = arguments(loader_options(settings, SDBackendModule::CONTROL_NET, request.definition));
+ request.fields["control_hint"] = sd_version_is_z_image(version) ? "control_hint" : "control_pixels";
+ request.fields["control_type"] = literal(sd_control_type_name(request.image.control_type));
+ request.fields["positive_condition"] = "controlled_positive";
+ request.fields["negative_condition"] = "controlled_negative";
+ }
+
+ void prepare_image_program(GenerationTemplate& request) {
+ const auto version = request.definition.version;
+ const auto& settings = request.context.config.context;
+ const auto prediction = settings.prediction == PREDICTION_COUNT ? request.context.default_prediction : settings.prediction;
+ const bool inpaint = sd_version_is_inpaint(version);
+ const bool mask = bound_tensor(request.options, "mask_image");
+ const bool refs = request.image.ref_images_count > 0;
+ const bool control = request.image.control_image.data;
+ const bool native_control = version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2;
+ const auto image_scale = request.image.sample_params.guidance.img_cfg;
+ const bool reference_guidance = refs && supports_reference_image_guidance(version);
+ if (std::isfinite(image_scale) && image_scale != 1.f &&
+ !sd_version_is_inpaint_or_unet_edit(version) && !reference_guidance)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "sample.guidance.img_cfg is not supported by this model with these inputs");
+ if (version != VERSION_QWEN_IMAGE_LAYERED && request.options && scalar_values(request.options).contains("qwen_image_layers"))
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "qwen_image_layers requires Qwen Image Layered");
+ const bool image_guidance = (inpaint || sd_version_is_control(version) || (sd_version_is_unet_edit(version) && refs) || reference_guidance) && std::isfinite(image_scale) && image_scale != 1.f;
+ request.flags = {
+ {"has_init", bool(request.image.init_image.data)}, {"has_mask", mask}, {"mask_stage", mask || inpaint}, {"references", refs}, {"unet_edit", sd_version_is_unet_edit(version)}, {"inpaint_model", inpaint}, {"flux_fill", version == VERSION_FLUX_FILL}, {"flex", version == VERSION_FLEX_2}, {"control_image", control}, {"native_control", native_control}, {"controlnet", control && !native_control}, {"z_control", control && sd_version_is_z_image(version)}, {"image_guidance", image_guidance}, {"minit2i", sd_version_is_minit2i(version)}, {"ming", version == VERSION_MING_IMAGE}, {"ideogram", sd_version_is_ideogram4(version)}, {"separate_negative_model", false}, {"sdxl_zero", sd_version_is_sdxl(version) && text(request.image.negative_prompt).empty() && prediction != EDM_V_PRED}, {"layered", version == VERSION_QWEN_IMAGE_LAYERED}, {"ph3", version == VERSION_P_H3_IMAGE}, {"h3_reference_noise", version == VERSION_MINIMAX_H3_IMAGE && refs}, {"custom_sigmas", request.image.sample_params.custom_sigmas_count > 0}};
+ request.fields["predictor_core"] = request.fields["predictor_options"];
+ request.flags["raw_reference"] = version == VERSION_SD1_PIX2PIX;
+ request.flags["reference_image_guidance"] = reference_guidance && image_guidance;
+ request.flags["hidream_o1"] = version == VERSION_HIDREAM_O1;
+ request.flags["control_image_guidance"] = image_guidance && version == VERSION_FLUX_CONTROLS;
+ if (version == VERSION_MOSS_IMAGE)
+ request.fields["negative_predictor_options"] += ", skip_layers = [22]";
+ if (sd_version_is_sensenova_u1(version)) {
+ request.fields["negative_prompt_input"] = "negative_prompt_raw";
+ request.fields["negative_prompt_options"] += ", raw = true";
+ }
+ request.fields["entry"] = mask || inpaint ? "image_inpaint" : refs || sd_version_is_unet_edit(version) ? "image_edit"
+ : "image";
+ if (version == VERSION_QWEN_IMAGE_LAYERED) {
+ if (request.image.init_image.data)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "Qwen layered uses ref_images rather than init_image");
+ request.fields["entry"] = "image_layered";
+ request.values["layers"] = request.image.qwen_image_layers + 1;
+ }
+ if (sd_version_is_minimax_h3_image(version)) {
+ request.fields["entry"] = "image_h3";
+ const auto options = version == VERSION_P_H3_IMAGE ? ", video_sigma_shift = 1, audio_length = 2, audio_sigma_shift = 1" : ", video_sigma_shift = 1";
+ request.fields["predictor_options"] += options;
+ request.fields["negative_predictor_options"] += options;
+ }
+ if (sd_version_is_ideogram4(version) && !text(settings.uncond_diffusion_model_path).empty()) {
+ request.flags["separate_negative_model"] = true;
+ request.values["negative_model_path"] = settings.uncond_diffusion_model_path;
+ request.fields["negative_model"] = "negative_model";
+ request.fields["schedule_predictor"] = "pos";
+ }
+ if (sd_version_is_pid(version) || sd_version_is_minit2i(version) || version == VERSION_IRIS) {
+ request.fields["entry"] = "image_pixel";
+ if (sd_version_is_pid(version) && !refs)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "PiD requires a reference image");
+ }
+ request.fields["unconditional_prompt_options"] = request.fields["negative_prompt_options"];
+ image_references(request);
+ image_control(request);
+ image_inpaint(request);
+ if (image_guidance) {
+ request.values["image_cfg_scale"] = image_scale;
+ request.fields["guidance_options"] += ", image_only = image_only, image_scale = image_cfg_scale";
+ request.fields["image_only_condition"] = request.fields["negative_condition"];
+ request.fields["image_only_options"] = request.fields["negative_predictor_options"];
+ request.fields["negative_predictor_options"] = request.fields["predictor_core"];
+ if (reference_guidance)
+ request.fields["negative_condition"] = request.flags.at("controlnet").get<bool>() ? "controlled_unconditional" : "unconditional_text";
+ else
+ request.fields["negative_predictor_options"] += ", concat = image_unconditional_concat";
+ }
+ if (sd_version_is_sefi_image(version)) {
+ request.fields["entry"] = "image_sefi";
+ auto options = nlohmann::json::object();
+ for (const auto& [key, value] : parse_key_value_args(request.image.sample_params.extra_sample_args, "SeFi scheduler arg")) {
+ if (key != "sefi_alpha" && key != "sefi_delta_t")
+ continue;
+ float number;
+ if (parse_strict_float(value, number))
+ options[key == "sefi_alpha" ? "alpha" : "delta"] = number;
+ else
+ LOG_WARN("ignoring invalid SeFi scheduler arg '%s=%s'", key.c_str(), value.c_str());
+ }
+ request.fields["sefi_schedule_options"] = arguments(options);
+ }
+ }
+}
diff --git a/src/api/generation_metadata.cpp b/src/api/generation_metadata.cpp
new file mode 100644
index 0000000..386ad9b
--- /dev/null
+++ b/src/api/generation_metadata.cpp
@@ -0,0 +1,202 @@
+#include "api/api.h"
+#include "api/metadata.h"
+
+#include <map>
+#include <set>
+#include <tuple>
+
+#include "dgl/syntax/syntax.h"
+
+namespace sd::api {
+ using JSON = nlohmann::json;
+ using Expression = std::shared_ptr<dgl::Expr>;
+ using Bindings = std::map<std::string, Expression>;
+
+ static Expression resolve_expression(Expression expr, const Bindings& bindings) {
+ std::set<std::string> visited;
+ while (expr && expr->form == dgl::Expr::Form::Reference && bindings.count(expr->name)) {
+ if (!visited.insert(expr->name).second)
+ return {};
+ expr = bindings.at(expr->name);
+ }
+ return expr;
+ }
+
+ static JSON constant(Expression expr, const Bindings& bindings, int depth = 0) {
+ if (depth > 32)
+ return {};
+ expr = resolve_expression(std::move(expr), bindings);
+ if (!expr)
+ return {};
+ if (expr->form == dgl::Expr::Form::Literal)
+ return expr->literal;
+ if (expr->form == dgl::Expr::Form::List) {
+ auto values = JSON::array();
+ for (const auto& item : expr->arguments) {
+ auto value = constant(item.second, bindings, depth + 1);
+ if (value.is_null())
+ return {};
+ values.push_back(std::move(value));
+ }
+ return values;
+ }
+ return {};
+ }
+
+ static Expression argument(const dgl::Expr& call, const char* name, size_t position = SIZE_MAX) {
+ size_t index = 0;
+ for (const auto& [key, expr] : call.arguments) {
+ if (key == name || (key.empty() && index == position))
+ return expr;
+ if (key.empty())
+ ++index;
+ }
+ return {};
+ }
+
+ static void collect_calls(const Expression& expr, std::map<std::string, std::vector<Expression>>& calls) {
+ if (!expr)
+ return;
+ if (expr->form == dgl::Expr::Form::Call)
+ calls[expr->name].push_back(expr);
+ for (const auto& item : expr->arguments)
+ collect_calls(item.second, calls);
+ for (const auto& statement : expr->body)
+ collect_calls(statement.value, calls);
+ }
+
+ static std::string filename(const std::string& path) {
+ const auto offset = path.find_last_of("/\\");
+ return offset == std::string::npos ? path : path.substr(offset + 1);
+ }
+
+ nlohmann::json generation_metadata(const std::string& source) noexcept {
+ try {
+ const auto syntax = dgl::parse_syntax(source, true);
+ Bindings bindings;
+ std::map<std::string, std::vector<Expression>> calls;
+ for (const auto& statement : syntax.entry.body) {
+ if (!statement.name.empty())
+ bindings.emplace(statement.name, statement.value);
+ collect_calls(statement.value, calls);
+ }
+ if (calls["sample"].empty() && calls["sefi::sample"].empty())
+ return JSON::object();
+ for (const auto& shape : calls["latent_shape"])
+ if (argument(*shape, "frames"))
+ return JSON::object();
+ auto info = JSON::object();
+ for (const auto* key : {"prompt", "negative_prompt", "seed", "width", "height", "steps", "cfg_scale", "method", "scheduler", "eta", "strength", "clip_skip", "model_path", "vae_path"}) {
+ if (bindings.count(key)) {
+ auto value = constant(bindings.at(key), bindings);
+ if (!value.is_null())
+ info[key] = std::move(value);
+ }
+ }
+ for (const auto* key : {"prompt", "negative_prompt"})
+ if (bindings.count(std::string("source_") + key))
+ info[key] = constant(bindings.at(std::string("source_") + key), bindings);
+ const auto select = [&](const std::vector<Expression>& candidates, const char* parameter, size_t position, const char* output) {
+ JSON selected;
+ for (const auto& call : candidates) {
+ const auto value = constant(argument(*call, parameter, position), bindings);
+ if (value.is_null() || (!selected.is_null() && selected != value)) {
+ info.erase(output);
+ return;
+ }
+ selected = value;
+ }
+ if (!selected.is_null())
+ info[output] = std::move(selected);
+ };
+ select(calls["load_dit"], "path", 0, "model_path");
+ select(calls["load_unet"], "path", 0, "model_path");
+ select(calls["load_vae"], "path", 0, "vae_path");
+ select(calls["cfg"], "scale", 2, "cfg_scale");
+ select(calls["make_sigmas"], "steps", SIZE_MAX, "steps");
+ select(calls["make_sigmas"], "scheduler", SIZE_MAX, "scheduler");
+ const auto samples = calls["sample"].size() + calls["sefi::sample"].size();
+ if (samples == 1) {
+ auto& sample = calls["sample"].empty() ? calls["sefi::sample"] : calls["sample"];
+ select(sample, "method", SIZE_MAX, "method");
+ select(sample, "eta", SIZE_MAX, "eta");
+ } else {
+ for (const auto* key : {"steps", "method", "eta", "scheduler", "strength"})
+ info.erase(key);
+ }
+ auto loras = JSON::array();
+ std::set<std::string> patches;
+ for (const auto& call : calls["apply_lora"]) {
+ const auto patch = resolve_expression(argument(*call, "lora", 1), bindings);
+ if (!patch || patch->form != dgl::Expr::Form::Call || patch->name != "load_lora")
+ continue;
+ const auto path = constant(argument(*patch, "path", 0), bindings);
+ auto weight = constant(argument(*call, "weight"), bindings);
+ if (path.is_string() && !weight.is_null()) {
+ auto item = JSON{{"path", path}, {"weight", weight}};
+ if (patches.insert(item.dump()).second)
+ loras.push_back(std::move(item));
+ }
+ }
+ info["loras"] = std::move(loras);
+ std::string infotext = info.value("prompt", "");
+ for (const auto& lora : info.value("loras", JSON::array())) {
+ auto name = filename(lora.at("path"));
+ const auto extension = name.find_last_of('.');
+ if (extension != std::string::npos)
+ name.erase(extension);
+ if (!name.empty() && name.find_first_of(":<>\r\n") == std::string::npos)
+ infotext += (infotext.empty() ? "" : " ") + std::string("<lora:") + name + ":" + lora.at("weight").dump() + ">";
+ }
+ const auto negative = info.value("negative_prompt", "");
+ if (!negative.empty())
+ infotext += "\nNegative prompt: " + negative;
+ infotext += "\n";
+ bool first = true;
+ const auto field = [&](const std::string& key, const JSON& value) {
+ std::string text = value.is_string() ? value.get<std::string>() : value.dump();
+ if (text.find_first_of(",\n\r:\"") != std::string::npos)
+ text = JSON(text).dump();
+ infotext += (first ? "" : ", ") + key + ": " + text;
+ first = false;
+ };
+ for (const auto& [key, label] : std::map<std::string, std::string>{{"steps", "Steps"}, {"cfg_scale", "CFG scale"}, {"seed", "Seed"}, {"eta", "Eta"}, {"strength", "Denoising strength"}})
+ if (info.contains(key))
+ field(label, info.at(key));
+ const std::map<std::string, std::string> samplers{{"euler", "Euler"}, {"euler_a", "Euler a"}, {"heun", "Heun"}, {"dpm2", "DPM2"}, {"dpm++2s_a", "DPM++ 2S a"}, {"dpm++2m", "DPM++ 2M"}, {"dpm++2m_sde", "DPM++ 2M SDE"}, {"lcm", "LCM"}};
+ const std::map<std::string, std::string> schedulers{{"discrete", "Normal"}, {"karras", "Karras"}, {"exponential", "Exponential"}, {"sgm_uniform", "SGM Uniform"}, {"simple", "Simple"}, {"ddim_uniform", "DDIM"}, {"beta", "Beta"}};
+ for (const auto& [key, label, unknown, names] : {std::tuple{"method", "Sampler", "SDCPP Sampler", &samplers}, {"scheduler", "Schedule type", "SDCPP Schedule", &schedulers}})
+ if (info.contains(key)) {
+ const auto name = info.at(key).get<std::string>();
+ const auto found = names->find(name);
+ field(found == names->end() ? unknown : label, found == names->end() ? name : found->second);
+ }
+ if (info.contains("width") && info.contains("height"))
+ field("Size", info.at("width").dump() + "x" + info.at("height").dump());
+ for (const auto& [key, label] : std::map<std::string, std::string>{{"model_path", "Model"}, {"vae_path", "VAE"}})
+ if (info.contains(key))
+ field(label, filename(info.at(key)));
+ if (info.value("clip_skip", 0) > 0)
+ field("Clip skip", info.at("clip_skip"));
+ field("Version", "stable-diffusion.cpp");
+ info["infotext"] = std::move(infotext);
+ return info;
+ } catch (...) {
+ return JSON::object();
+ }
+ }
+}
+
+static thread_local std::string generation_metadata_json;
+
+const char* sd_get_generation_metadata(const char* source) {
+ const char* result = nullptr;
+ sd::api::call([&] {
+ if (!source)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "source is required");
+ generation_metadata_json = sd::api::generation_metadata(source).dump();
+ result = generation_metadata_json.c_str();
+ return SD_STATUS_OK;
+ });
+ return result;
+}
diff --git a/src/api/generation_program.cpp b/src/api/generation_program.cpp
new file mode 100644
index 0000000..92b69a4
--- /dev/null
+++ b/src/api/generation_program.cpp
@@ -0,0 +1,377 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "conditioning/prompt_input.h"
+#include "core/image_geometry.h"
+#include "model/vae/geometry.hpp"
+#include "name_conversion.h"
+#include "runtime/guidance.h"
+
+namespace sd::api {
+ std::string GenerationTemplate::render(const std::string& name, const nlohmann::json& overrides) const {
+ nlohmann::json parameters = fields;
+ parameters["flags"] = flags;
+ parameters["vae_run_options"] = vae_options;
+ for (auto it = overrides.begin(); it != overrides.end(); ++it)
+ parameters[it.key()] = it.value();
+ return render_template(name, parameters);
+ }
+
+ static std::string variables(const nlohmann::json& values) {
+ std::string result;
+ for (auto it = values.begin(); it != values.end(); ++it)
+ result += " " + it.key() + " = " + literal(it.value()) + "\n";
+ return result;
+ }
+
+ std::string text(const char* value) {
+ return value ? value : "";
+ }
+
+ static std::string module_option(const char* value, SDBackendModule module, bool (*parse)(const std::string&, SDBackendAssignment*, std::string*) = sd_parse_backend_assignment) {
+ if (!value || !*value)
+ return "";
+ SDBackendAssignment assignment;
+ std::string error;
+ if (!parse(value, &assignment, &error))
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, error);
+ return assignment.get(module);
+ }
+
+ nlohmann::json loader_options(const ContextConfig& settings, SDBackendModule module, const ModelDefinition& definition) {
+ nlohmann::json options{{"family", definition.id}};
+ for (const auto& [key, value] : std::map<std::string, const char*>{{"backend", settings.backend}, {"params_backend", settings.params_backend}, {"split_mode", settings.split_mode}}) {
+ const auto selected = module_option(value, module);
+ if (!selected.empty())
+ options[key] = selected;
+ }
+ auto flash = module_option(settings.flash_attn_mode, module, sd_parse_flash_attention_assignment);
+ if (flash.empty()) {
+ if (settings.flash_attn)
+ flash = "on";
+ else if (settings.diffusion_flash_attn)
+ flash = module == SDBackendModule::DIFFUSION ? "on" : "off";
+ }
+ if (!flash.empty())
+ options["flash_attn_mode"] = flash;
+ auto conv = module_option(settings.conv_direct_mode, module, sd_parse_conv_direct_assignment);
+ if (conv.empty() && (module == SDBackendModule::VAE ? settings.vae_conv_direct : settings.diffusion_conv_direct))
+ conv = "on";
+ if (!conv.empty())
+ options["conv_direct_mode"] = conv;
+ if (settings.wtype != SD_TYPE_COUNT)
+ options["wtype"] = sd_type_name(settings.wtype);
+ for (const auto& [key, value] : std::map<std::string, const char*>{{"model_args", settings.model_args}, {"tensor_type_rules", settings.tensor_type_rules}})
+ if (value && *value)
+ options[key] = value;
+ if (settings.linear_scale)
+ options["linear_scale"] = settings.linear_scale;
+ if (settings.attn_scale)
+ options["attn_scale"] = settings.attn_scale;
+ if (module == SDBackendModule::DIFFUSION && settings.sage_attn)
+ options["sage_attn"] = true;
+ return options;
+ }
+
+ static std::string tiling_options(const TilingConfig& config, bool circular_x, bool circular_y) {
+ std::string options;
+ if (circular_x)
+ options += ", circular_x = true";
+ if (circular_y)
+ options += ", circular_y = true";
+ if (config.enabled || config.temporal_tiling) {
+ auto fields = nlohmann::json{{"enabled", config.enabled}, {"temporal_tiling", config.temporal_tiling}, {"tile_size_w", config.tile_size_w}, {"tile_size_h", config.tile_size_h}, {"target_overlap", config.target_overlap}, {"rel_size_w", config.rel_size_w}, {"rel_size_h", config.rel_size_h}};
+ if (config.extra_tiling_args && *config.extra_tiling_args)
+ fields["extra_args"] = config.extra_tiling_args;
+ const auto args = arguments(fields);
+ options += ", tiling = tiling(" + args.substr(2) + ")";
+ }
+ return options;
+ }
+
+ static std::string cache_options(const sd_opt_t* options) {
+ const auto values = scalar_values(options);
+ if (!values.contains("cache.mode") || values.at("cache.mode") == "disabled")
+ return "";
+ auto fields = nlohmann::json::object();
+ for (auto it = values.begin(); it != values.end(); ++it) {
+ if (it.key().rfind("cache.", 0) != 0)
+ continue;
+ const auto name = it.key().substr(6);
+ if (name == "fn_compute_blocks" || name == "bn_compute_blocks")
+ continue;
+ fields[name] = it.value();
+ }
+ const auto args = arguments(fields);
+ return ", cache = cache(" + args.substr(2) + ")";
+ }
+
+ void bind_tensor(GenerationProgram& program, const std::string& name, const sd_tensor_t* tensor) {
+ if (!tensor)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "missing host tensor '" + name + "'");
+ program.inputs.emplace(name, dgl::Value::object(std::shared_ptr<sd_tensor_t>(sd_tensor_retain(tensor), sd_tensor_free)));
+ }
+
+ static std::string input_names(const GenerationProgram& program) {
+ std::string names;
+ for (const auto& [name, value] : program.inputs)
+ names += (names.empty() ? "" : ", ") + name;
+ return names;
+ }
+
+ struct LoraBindings {
+ std::string source;
+ std::string model = "base_model";
+ std::string text = "base_text";
+ };
+
+ static LoraBindings lora_bindings(const ContextConfig& settings, SDVersion family, const sd_lora_t* loras, uint32_t count) {
+ LoraBindings result;
+ for (uint32_t i = 0; i < count; ++i) {
+ if (loras[i].is_high_noise)
+ continue;
+ ModelLoader metadata;
+ if (!metadata.init_from_file_and_convert_name(text(loras[i].path), "lora.", family))
+ throw dgl::Error(SD_STATUS_LOAD_FAILED, "cannot inspect LoRA '" + text(loras[i].path) + "'");
+ bool diffusion = false, conditioning = false;
+ for (const auto& name : metadata.get_tensor_names()) {
+ diffusion |= is_diffusion_model_name(name);
+ conditioning |= is_cond_stage_model_name(name);
+ }
+ if (!diffusion && !conditioning)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "LoRA has no diffusion or text-encoder targets");
+ const auto number = std::to_string(i);
+ const auto model = "model_" + number, encoder = "text_" + number;
+ result.source += render_template("lora", {{"name", "lora_" + number}, {"path", text(loras[i].path)}, {"diffusion", diffusion}, {"conditioning", conditioning}, {"model_result", model}, {"text_result", encoder}, {"model_source", result.model}, {"text_source", result.text}, {"weight", loras[i].multiplier}, {"mode", sd_lora_apply_mode_name(settings.lora_apply_mode)}});
+ if (diffusion)
+ result.model = model;
+ if (conditioning)
+ result.text = encoder;
+ }
+ return result;
+ }
+
+ struct ComponentBindings {
+ bool tae;
+ nlohmann::json values;
+ std::map<std::string, std::string> fields;
+ };
+
+ static ComponentBindings component_bindings(const ContextConfig& settings, const ModelDefinition& definition) {
+ const auto primary = text(settings.diffusion_model_path).empty() ? text(settings.model_path) : text(settings.diffusion_model_path);
+ auto text_path = text(settings.llm_path);
+ if (text_path.empty())
+ text_path = text(settings.model_path);
+ if (text_path.empty())
+ for (const auto* path : {settings.clip_l_path, settings.t5xxl_path, settings.clip_g_path})
+ if (path && *path) {
+ text_path = path;
+ break;
+ }
+ if (text_path.empty())
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "a checkpoint or text encoder path is required");
+ const bool mandatory_tae = definition.version == VERSION_SDXS_512_DS || definition.version == VERSION_SDXS_09;
+ const bool tae = settings.taesd_path && *settings.taesd_path && (!settings.tae_preview_only || mandatory_tae);
+ auto vae_path = tae ? text(settings.taesd_path) : text(settings.vae_path);
+ if (vae_path.empty())
+ vae_path = text(settings.model_path).empty() ? primary : text(settings.model_path);
+ auto text_options = loader_options(settings, SDBackendModule::TE, definition);
+ for (const auto& [key, path] : std::map<std::string, const char*>{{"tokenizer", settings.tokenizer}, {"vision_path", settings.llm_vision_path}, {"connectors_path", settings.embeddings_connectors_path}, {"t5_path", settings.t5xxl_path}, {"clip_l_path", settings.clip_l_path}, {"clip_g_path", settings.clip_g_path}})
+ if (path && *path)
+ text_options[key] = path;
+ if (sd_version_is_ltxav(definition.version) || definition.version == VERSION_P_H3_IMAGE)
+ text_options["projection_path"] = primary;
+ auto vae_options = loader_options(settings, SDBackendModule::VAE, definition);
+ if (sd_version_is_sdxl(definition.version) && !tae &&
+ (settings.force_sdxl_vae_conv_scale || text(settings.vae_path).empty()))
+ vae_options["conv_scale"] = 0.03125;
+ if (!tae && settings.vae_format != SD_VAE_FORMAT_AUTO) {
+ static const char* formats[]{"flux", "sd3", "flux2", "wan"};
+ vae_options["vae_format"] = formats[int(settings.vae_format)];
+ }
+ return {tae,
+ {{"model_path", primary}, {"text_path", text_path}, {"vae_path", vae_path}},
+ {{"model_loader", sd_version_is_unet(definition.version) ? "load_unet" : "load_dit"},
+ {"model_options", arguments(loader_options(settings, SDBackendModule::DIFFUSION, definition))},
+ {"text_options", arguments(text_options)},
+ {"vae_loader", tae ? "load_tae" : "load_vae"},
+ {"vae_options", arguments(vae_options)}}};
+ }
+
+ GenerationProgram component_program(GenerationContext& context) {
+ const auto& definition = context.model_definition();
+ const auto& settings = context.config.context;
+ if (definition.version == VERSION_ESRGAN || definition.version == VERSION_SEEDVR2 || definition.version == VERSION_RELAY_VSR)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "this model requires a dedicated DGL workflow");
+ auto components = component_bindings(settings, definition);
+ nlohmann::json flags;
+ const auto optional_component = [&](const char* name, const std::string& path, SDBackendModule module) {
+ flags[name] = !path.empty();
+ if (!path.empty()) {
+ components.values[std::string(name) + "_path"] = path;
+ components.fields[std::string(name) + "_options"] = arguments(loader_options(settings, module, definition));
+ }
+ };
+ optional_component("high_model", text(settings.high_noise_diffusion_model_path), SDBackendModule::DIFFUSION);
+ optional_component("negative_model", sd_version_is_ideogram4(definition.version) ? text(settings.uncond_diffusion_model_path) : "", SDBackendModule::DIFFUSION);
+ optional_component("control", text(settings.control_net_path), SDBackendModule::CONTROL_NET);
+ optional_component("vision", text(settings.clip_vision_path), SDBackendModule::TE);
+ optional_component("audio_encoder", text(settings.audio_encoder_path), SDBackendModule::TE);
+ optional_component("audio_vae", text(settings.audio_vae_path).empty() ? context.embedded_audio_vae : text(settings.audio_vae_path), SDBackendModule::VAE);
+ optional_component("preview", !components.tae && settings.tae_preview_only ? text(settings.taesd_path) : "", SDBackendModule::VAE);
+ nlohmann::json parameters = components.fields;
+ parameters["variables"] = variables(components.values);
+ parameters["flags"] = std::move(flags);
+ return {render_template("load_components", parameters), {}};
+ }
+
+ static GenerationProgram generation_program(GenerationContext& context, const ImageConfig& image, const sd_opt_t* explicit_options, int64_t seed, const VideoConfig* video) {
+ const auto& definition = context.model_definition();
+ const auto& settings = context.config.context;
+ if (definition.traits.supports_video_generation != bool(video) || definition.version == VERSION_ESRGAN || definition.version == VERSION_SEEDVR2 || definition.version == VERSION_RELAY_VSR)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "this model requires a video or dedicated DGL workflow");
+ const auto components = component_bindings(settings, definition);
+ const auto method = image.sample_params.sample_method == SAMPLE_METHOD_COUNT ? definition.sampling.default_sample_method : image.sample_params.sample_method;
+ auto scheduler = image.sample_params.scheduler;
+ if (scheduler == SCHEDULER_COUNT) {
+ scheduler = (settings.prediction == EDM_V_PRED || (settings.prediction == PREDICTION_COUNT && context.default_prediction == EDM_V_PRED)) ? EXPONENTIAL_SCHEDULER : method == LCM_SAMPLE_METHOD || method == TCD_SAMPLE_METHOD ? LCM_SCHEDULER
+ : method == DDIM_TRAILING_SAMPLE_METHOD ? SIMPLE_SCHEDULER
+ : definition.sampling.default_scheduler;
+ }
+ const auto predictor_options = arguments({{"prediction", settings.prediction == PREDICTION_COUNT ? "auto" : sd_prediction_name(settings.prediction)},
+ {"guidance", image.sample_params.guidance.distilled_guidance},
+ {"circular_x", image.circular_x},
+ {"circular_y", image.circular_y}});
+ const double eta = std::isfinite(image.sample_params.eta) ? image.sample_params.eta : definition.version == VERSION_HIDREAM_O1 ? 8.0
+ : method == EULER_A_SAMPLE_METHOD || method == DPMPP2S_A_SAMPLE_METHOD || method == ER_SDE_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD || method == DPMPP2M_SDE_SAMPLE_METHOD || method == DPMPP2M_SDE_BT_SAMPLE_METHOD ? 1.0
+ : 0.0;
+ auto values = components.values;
+ values.update({{"prompt", text(image.prompt)}, {"negative_prompt", text(image.negative_prompt)}, {"seed", seed}, {"width", image.width}, {"height", image.height}, {"steps", image.sample_params.sample_steps}, {"cfg_scale", image.sample_params.guidance.txt_cfg}, {"method", sd_sample_method_name(method)}, {"scheduler", sd_scheduler_name(scheduler)}, {"eta", eta}, {"rng_type", sd_rng_type_name(settings.rng_type)}});
+ const auto vae_version = definition.traits.vae_format_override && !components.tae ? sd::vae::version_for_format(settings.vae_format, definition.version) : definition.version;
+ const int scale = sd_version_is_pid(definition.version) ? 1 : sd::vae::spatial_scale(vae_version);
+ const int multiple = scale * definition.geometry.diffusion_down_factor;
+ values["width"] = sd_version_is_minit2i(definition.version) ? 512 : sd::align_image_dimension(image.width, multiple);
+ values["height"] = sd_version_is_minit2i(definition.version) ? 512 : sd::align_image_dimension(image.height, multiple);
+ const auto axes = tiling_options(image.vae_tiling_params, image.circular_x, image.circular_y);
+ auto schedule_options = nlohmann::json::object();
+ if (std::isfinite(image.sample_params.flow_shift))
+ schedule_options["flow_shift"] = image.sample_params.flow_shift;
+ if (image.sample_params.extra_sample_args && *image.sample_params.extra_sample_args)
+ schedule_options["extra_args"] = image.sample_params.extra_sample_args;
+ if (image.sample_params.custom_sigmas_count) {
+ values["custom_sigmas"] = std::vector<float>(image.sample_params.custom_sigmas, image.sample_params.custom_sigmas + image.sample_params.custom_sigmas_count);
+ }
+ GenerationProgram program;
+ if (video) {
+ values["frames"] = sd::vae::aligned_frames(definition.version, video->video_frames);
+ values["fps"] = sd_version_is_minimax_h3(definition.version) ? 24 : definition.version == VERSION_WAN2_2_S2V ? 16
+ : video->fps;
+ }
+ const bool init = image.init_image.data;
+ if (!video && init) {
+ bind_tensor(program, "input_image", bound_tensor(explicit_options, "init_image"));
+ values["strength"] = image.strength;
+ }
+ const auto sampler_type = settings.sampler_rng_type == RNG_TYPE_COUNT ? settings.rng_type : settings.sampler_rng_type;
+ std::string preview_decoder = "vae";
+ if (settings.tae_preview_only && settings.taesd_path && *settings.taesd_path) {
+ auto options = loader_options(settings, SDBackendModule::VAE, definition);
+ options["decode_only"] = true;
+ preview_decoder = "load_tae(" + literal(settings.taesd_path) + arguments(options) + ")";
+ }
+ auto sample_options = cache_options(explicit_options) + ", observer = preview(decoder = " + preview_decoder + arguments({{"circular_x", image.circular_x}, {"circular_y", image.circular_y}}) + ")";
+ if (image.sample_params.extra_sample_args && *image.sample_params.extra_sample_args)
+ sample_options += ", extra_args = " + literal(image.sample_params.extra_sample_args);
+ const auto to_vae = sd_version_is_unet(definition.version) ? "unet_to_vae_latents" : "dit_to_vae_latents";
+ const auto patches = lora_bindings(settings, definition.version, image.loras, image.lora_count);
+ const auto apg = sd::guidance::parse_adaptive_projected_guidance_args(image.sample_params.extra_sample_args);
+ const bool use_apg = sd::guidance::is_adaptive_projected_guidance_enabled(apg);
+ const auto guidance_schedule = sd::guidance::parse_guidance_schedule(image.sample_params.extra_sample_args);
+ auto guidance_options = nlohmann::json::object();
+ if (!guidance_schedule.empty())
+ guidance_options["schedule"] = guidance_schedule;
+ if (use_apg)
+ guidance_options.update({{"eta", apg.eta}, {"momentum", apg.momentum}, {"norm_threshold", apg.norm_threshold}, {"norm_threshold_smoothing", apg.norm_threshold_smoothing}});
+ std::map<std::string, std::string> fields{{"patched_model", patches.model},
+ {"patched_text", patches.text},
+ {"references", ""},
+ {"loras", patches.source},
+ {"positive_condition", "positive"},
+ {"negative_condition", "negative"},
+ {"negative_model", "model"},
+ {"prompt_options", ", clip_skip = " + std::to_string(image.clip_skip)},
+ {"negative_prompt_options", ", clip_skip = " + std::to_string(image.clip_skip)},
+ {"negative_prompt_input", "negative_prompt"},
+ {"predictor_options", predictor_options},
+ {"predictor_core", predictor_options},
+ {"reference_predictor_options", ""},
+ {"negative_predictor_options", predictor_options},
+ {"guidance_options", arguments(guidance_options)},
+ {"guidance_core", arguments(guidance_options)},
+ {"guidance_function", use_apg ? "apg" : "cfg"},
+ {"sample_options", sample_options},
+ {"sample_core", sample_options},
+ {"result_latent", "latent"},
+ {"output_audio_shape", "audio_shape"},
+ {"output_video_frames", "tensor::shape(video_base)[2]"},
+ {"to_vae", to_vae},
+ {"channels", std::to_string(definition.geometry.image_channels)},
+ {"from_vae", sd_version_is_unet(definition.version) ? "vae_to_unet_latents" : "vae_to_dit_latents"},
+ {"schedule_options", arguments(schedule_options)},
+ {"schedule_predictor", "guided"},
+ {"audio", "{}"}};
+ fields.insert(components.fields.begin(), components.fields.end());
+ GenerationTemplate request{context, image, explicit_options, definition, program, values, fields, video, axes, seed};
+ if (video)
+ prepare_video_program(request);
+ else
+ prepare_image_program(request);
+ prepare_prompt_images(request);
+ if (sd_version_is_sensenova_u1(definition.version)) {
+ const auto input = sd::parse_prompt_input(values.at("negative_prompt").get<std::string>(), true);
+ if (std::any_of(input.begin(), input.end(), [](const auto& part) { return part.type != sd::PromptPart::Type::TEXT; }))
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "SenseNova U1.5 does not support inline prompt images or embeddings");
+ values["negative_prompt_raw"] = "<|im_start|>user\n" + sd::prompt_input_text(input) + "<|im_end|>\n<|im_start|>assistant\n<img>";
+ }
+ request.flags["encode_negative"] = image.sample_params.guidance.txt_cfg != 1.f || use_apg || request.flags.value("image_guidance", false) ||
+ method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD ||
+ std::any_of(guidance_schedule.begin(), guidance_schedule.end(), [](float scale) { return scale != 1.f; });
+ request.flags["shared_rng"] = sampler_type == settings.rng_type;
+ if (sampler_type != settings.rng_type)
+ values["sampler_rng_type"] = sd_rng_type_name(sampler_type);
+ prepare_hires_program(request);
+ fields["variables"] = variables(values);
+ fields["inputs"] = input_names(program);
+ program.source = request.render(fields.at("entry"));
+ return program;
+ }
+
+ GenerationProgram image_program(GenerationContext& context, const ImageConfig& image, const sd_opt_t* explicit_options, int64_t seed) {
+ return generation_program(context, image, explicit_options, seed, nullptr);
+ }
+
+ GenerationProgram video_program(GenerationContext& context, const VideoConfig& video, const sd_opt_t* explicit_options, int64_t seed) {
+ ImageConfig common;
+ common.loras = video.loras;
+ common.lora_count = video.lora_count;
+ common.prompt = video.prompt;
+ common.negative_prompt = video.negative_prompt;
+ common.clip_skip = video.clip_skip;
+ common.init_image = video.init_image;
+ common.ref_images = video.ref_images;
+ common.ref_images_count = video.ref_images_count;
+ common.width = video.width;
+ common.height = video.height;
+ common.sample_params = video.sample_params;
+ common.strength = video.strength;
+ common.vae_tiling_params = video.vae_tiling_params;
+ common.cache = video.cache;
+ common.hires = video.hires;
+ common.circular_x = video.circular_x;
+ common.circular_y = video.circular_y;
+ return generation_program(context, common, explicit_options, seed, &video);
+ }
+
+}
diff --git a/src/api/generation_program.h b/src/api/generation_program.h
new file mode 100644
index 0000000..714e981
--- /dev/null
+++ b/src/api/generation_program.h
@@ -0,0 +1,33 @@
+#ifndef __SD_API_GENERATION_PROGRAM_H__
+#define __SD_API_GENERATION_PROGRAM_H__
+
+#include "api/generation.h"
+
+namespace sd::api {
+ struct GenerationTemplate {
+ GenerationContext& context;
+ const ImageConfig& image;
+ const sd_opt_t* options;
+ const ModelDefinition& definition;
+ GenerationProgram& program;
+ nlohmann::json& values;
+ std::map<std::string, std::string>& fields;
+ const VideoConfig* video;
+ std::string vae_options;
+ int64_t seed;
+ nlohmann::json flags = nlohmann::json::object();
+
+ std::string render(const std::string& name, const nlohmann::json& overrides = nlohmann::json::object()) const;
+ };
+
+ std::string text(const char* value);
+ nlohmann::json loader_options(const ContextConfig& settings, SDBackendModule module, const ModelDefinition& definition);
+ void bind_tensor(GenerationProgram& program, const std::string& name, const sd_tensor_t* tensor);
+ void prepare_image_program(GenerationTemplate& request);
+ void prepare_video_program(GenerationTemplate& request);
+ void prepare_hires_program(GenerationTemplate& request);
+ void prepare_prompt_images(GenerationTemplate& request);
+ void prepare_h3_references(GenerationTemplate& request, const std::string& first, const std::string& last);
+}
+
+#endif // __SD_API_GENERATION_PROGRAM_H__
diff --git a/src/api/generation_prompt.cpp b/src/api/generation_prompt.cpp
new file mode 100644
index 0000000..5481751
--- /dev/null
+++ b/src/api/generation_prompt.cpp
@@ -0,0 +1,123 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cctype>
+
+#include "api/tensor.h"
+
+namespace sd::api {
+ static std::string embedding_lookup(std::string value) {
+ std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); });
+ return value;
+ }
+
+ static void prepare_textual_inversion(GenerationTemplate& request) {
+ const auto& config = request.context.config.context;
+ if (!config.embedding_count)
+ return;
+ std::vector<std::pair<std::string, std::string>> embeddings;
+ for (uint32_t i = 0; i < config.embedding_count; ++i) {
+ auto path = text(config.embeddings[i].path);
+ std::replace(path.begin(), path.end(), '\\', '/');
+ if (path.find('>') != std::string::npos)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "embedding path cannot contain '>'");
+ embeddings.emplace_back(embedding_lookup(config.embeddings[i].name), std::move(path));
+ }
+ std::stable_sort(embeddings.begin(), embeddings.end(), [](const auto& a, const auto& b) { return a.first.size() > b.first.size(); });
+ const auto word = [](unsigned char c) { return std::isalnum(c) || c == '_' || c >= 128; };
+ for (const auto* key : {"prompt", "negative_prompt"}) {
+ const auto original = request.values.at(key).get<std::string>();
+ const auto lookup = embedding_lookup(original);
+ std::string rewritten;
+ for (size_t cursor = 0; cursor < original.size();) {
+ if (original[cursor] == '<') {
+ const auto end = original.find('>', cursor);
+ if (end != std::string::npos) {
+ rewritten += original.substr(cursor, end + 1 - cursor);
+ cursor = end + 1;
+ continue;
+ }
+ }
+ bool matched = false;
+ for (const auto& [name, path] : embeddings) {
+ const auto end = cursor + name.size();
+ if ((cursor && word(original[cursor - 1])) ||
+ lookup.compare(cursor, name.size(), name) || (end < original.size() && word(original[end])))
+ continue;
+ rewritten += "<embedding:" + path + ">";
+ cursor = end;
+ matched = true;
+ break;
+ }
+ if (!matched)
+ rewritten += original[cursor++];
+ }
+ if (rewritten != original) {
+ request.values[std::string("source_") + key] = original;
+ request.values[key] = std::move(rewritten);
+ }
+ }
+ }
+
+ void prepare_prompt_images(GenerationTemplate& request) {
+ prepare_textual_inversion(request);
+ auto& context = request.context;
+ if (!context.prompt_image)
+ return;
+ std::map<std::string, std::string> aliases;
+ for (const auto* key : {"prompt", "negative_prompt"}) {
+ const auto original = request.values.at(key).get<std::string>();
+ std::string rewritten;
+ size_t cursor = 0;
+ while (cursor < original.size()) {
+ const auto start = original.find("<image:", cursor);
+ if (start == std::string::npos) {
+ rewritten += original.substr(cursor);
+ break;
+ }
+ size_t slashes = 0;
+ while (start > slashes && original[start - slashes - 1] == '\\')
+ ++slashes;
+ rewritten += original.substr(cursor, start - cursor);
+ if (slashes % 2) {
+ rewritten += "<image:";
+ cursor = start + 7;
+ continue;
+ }
+ const auto end = original.find('>', start + 7);
+ if (end == std::string::npos || end == start + 7)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "invalid inline prompt image");
+ const auto source = original.substr(start + 7, end - start - 7);
+ auto found = aliases.find(source);
+ if (found == aliases.end()) {
+ if (aliases.size() >= 32)
+ throw dgl::Error(SD_STATUS_OUT_OF_RANGE, "a request supports at most 32 prompt images");
+ context.check_cancelled();
+ const sd_tensor_t* tensor = nullptr;
+ if (!context.prompt_image(source.c_str(), context.prompt_image_data, &tensor) || !tensor)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "cannot resolve prompt image '" + source + "'");
+ image_view(*tensor);
+ auto copy = std::make_shared<sd_tensor_t>(tensor->type, tensor->shape(), tensor->data(), tensor->bytes);
+ const auto alias = "prompt_image_" + std::to_string(aliases.size());
+ request.program.inputs.emplace(alias, dgl::Value::object(std::move(copy)));
+ found = aliases.emplace(source, alias).first;
+ }
+ rewritten += "<image:" + found->second + ">";
+ cursor = end + 1;
+ }
+ if (rewritten != original) {
+ request.values[std::string("source_") + key] = original;
+ request.values[key] = rewritten;
+ }
+ }
+ if (aliases.empty())
+ return;
+ std::string bindings;
+ for (const auto& [source, alias] : aliases)
+ bindings += (bindings.empty() ? "" : ", ") + alias + " = " + alias;
+ const auto argument = ", image_bindings = {" + bindings + "}";
+ request.fields["prompt_options"] += argument;
+ request.fields["negative_prompt_options"] += argument;
+ request.fields["unconditional_prompt_options"] += argument;
+ }
+}
diff --git a/src/api/generation_template.cpp b/src/api/generation_template.cpp
new file mode 100644
index 0000000..f56e09a
--- /dev/null
+++ b/src/api/generation_template.cpp
@@ -0,0 +1,222 @@
+#include "api/generation_template.h"
+
+#include <algorithm>
+#include <map>
+#include <stdexcept>
+#include <string_view>
+#include <vector>
+
+#include "api_templates.h"
+
+namespace sd::api {
+ std::string literal(const nlohmann::json& value) {
+ if (value.is_object()) {
+ std::string result = "{";
+ for (auto it = value.begin(); it != value.end(); ++it) {
+ if (it != value.begin())
+ result += ", ";
+ result += it.key() + " = " + literal(it.value());
+ }
+ return result + "}";
+ }
+ if (value.is_array()) {
+ std::string result = "[";
+ for (size_t i = 0; i < value.size(); ++i)
+ result += (i ? ", " : "") + literal(value[i]);
+ return result + "]";
+ }
+ return value.dump();
+ }
+
+ std::string arguments(const nlohmann::json& values) {
+ if (!values.is_object())
+ throw std::logic_error("template argument options must be an object");
+ std::string result;
+ for (auto it = values.begin(); it != values.end(); ++it)
+ result += ", " + it.key() + " = " + literal(it.value());
+ return result;
+ }
+
+ struct TemplatePart {
+ enum class Kind { Text,
+ Variable,
+ Conditional,
+ Include } kind;
+ std::string value;
+ std::vector<std::pair<std::string, std::vector<TemplatePart>>> branches;
+ std::vector<TemplatePart> otherwise;
+ };
+
+ static std::string trim_template(std::string_view text) {
+ const auto start = text.find_first_not_of(" \t\r\n");
+ if (start == std::string_view::npos)
+ return "";
+ return std::string(text.substr(start, text.find_last_not_of(" \t\r\n") - start + 1));
+ }
+
+ class TemplateParser {
+ std::string name_;
+ std::string_view source_;
+ size_t offset_ = 0;
+
+ [[noreturn]] void fail(const std::string& reason) const {
+ size_t line = 1;
+ for (size_t i = 0; i < offset_; ++i)
+ line += source_[i] == '\n';
+ throw std::logic_error("template '" + name_ + "', line " + std::to_string(line) + ": " + reason);
+ }
+
+ std::vector<TemplatePart> sequence(std::string& boundary) {
+ std::vector<TemplatePart> parts;
+ while (offset_ < source_.size()) {
+ const auto variable = source_.find("{{", offset_);
+ const auto control = source_.find("{%", offset_);
+ const auto begin = std::min(variable, control);
+ if (begin == std::string_view::npos) {
+ parts.push_back({TemplatePart::Kind::Text, std::string(source_.substr(offset_))});
+ offset_ = source_.size();
+ break;
+ }
+ if (begin > offset_)
+ parts.push_back({TemplatePart::Kind::Text, std::string(source_.substr(offset_, begin - offset_))});
+ const bool is_variable = begin == variable;
+ const auto end = source_.find(is_variable ? "}}" : "%}", begin + 2);
+ if (end == std::string_view::npos)
+ fail("unterminated tag");
+ const auto command = trim_template(source_.substr(begin + 2, end - begin - 2));
+ offset_ = end + 2;
+ if (!is_variable) {
+ if (source_.substr(offset_, 2) == "\r\n")
+ offset_ += 2;
+ else if (offset_ < source_.size() && source_[offset_] == '\n')
+ ++offset_;
+ }
+ if (is_variable) {
+ if (command.empty())
+ fail("empty variable");
+ parts.push_back({TemplatePart::Kind::Variable, command});
+ } else if (command == "else" || command == "endif" || command.rfind("elif ", 0) == 0) {
+ boundary = command;
+ return parts;
+ } else if (command.rfind("if ", 0) == 0) {
+ TemplatePart conditional{TemplatePart::Kind::Conditional};
+ auto condition = trim_template(std::string_view(command).substr(3));
+ std::string closing;
+ do {
+ if (condition.empty())
+ fail("empty condition");
+ closing.clear();
+ conditional.branches.emplace_back(condition, sequence(closing));
+ condition = closing.rfind("elif ", 0) == 0 ? trim_template(std::string_view(closing).substr(5)) : "";
+ } while (!condition.empty());
+ if (closing == "else") {
+ closing.clear();
+ conditional.otherwise = sequence(closing);
+ }
+ if (closing != "endif")
+ fail("expected endif");
+ parts.push_back(std::move(conditional));
+ } else if (command.rfind("include ", 0) == 0) {
+ const auto path = trim_template(std::string_view(command).substr(8));
+ if (path.size() < 3 || path.front() != '"' || path.back() != '"')
+ fail("include requires a quoted template name");
+ parts.push_back({TemplatePart::Kind::Include, path.substr(1, path.size() - 2)});
+ } else {
+ fail("unknown directive '" + command + "'");
+ }
+ }
+ return parts;
+ }
+
+ public:
+ TemplateParser(std::string name, std::string_view source)
+ : name_(std::move(name)), source_(source) {}
+
+ std::vector<TemplatePart> parse() {
+ std::string boundary;
+ auto result = sequence(boundary);
+ if (!boundary.empty())
+ fail("unexpected '" + boundary + "'");
+ return result;
+ }
+ };
+
+ static const nlohmann::json& template_value(const nlohmann::json& values, const std::string& name) {
+ const auto* value = &values;
+ size_t start = 0;
+ do {
+ const auto dot = name.find('.', start);
+ const auto field = name.substr(start, dot - start);
+ if (!value->is_object() || !value->contains(field))
+ throw std::logic_error("missing template variable '" + name + "'");
+ value = &value->at(field);
+ if (dot == std::string::npos)
+ return *value;
+ start = dot + 1;
+ } while (start < name.size());
+ throw std::logic_error("invalid template variable '" + name + "'");
+ }
+
+ static bool template_condition(const nlohmann::json& values, std::string name) {
+ const bool invert = name.rfind("not ", 0) == 0;
+ if (invert)
+ name = trim_template(std::string_view(name).substr(4));
+ const auto& value = template_value(values, name);
+ if (!value.is_boolean())
+ throw std::logic_error("template condition '" + name + "' must be boolean");
+ return value.get<bool>() != invert;
+ }
+
+ using ParsedTemplates = std::map<std::string, std::vector<TemplatePart>>;
+
+ static void render_parts(std::string& output, const std::vector<TemplatePart>& parts, const nlohmann::json& values, const ParsedTemplates& sources, int depth) {
+ if (depth > 16)
+ throw std::logic_error("template include depth exceeds 16");
+ for (const auto& part : parts) {
+ if (part.kind == TemplatePart::Kind::Text) {
+ output += part.value;
+ } else if (part.kind == TemplatePart::Kind::Variable) {
+ const auto separator = part.value.find('|');
+ const auto name = trim_template(std::string_view(part.value).substr(0, separator));
+ const auto& value = template_value(values, name);
+ const auto filter = separator == std::string::npos ? "" : trim_template(std::string_view(part.value).substr(separator + 1));
+ if (filter == "literal")
+ output += literal(value);
+ else if (filter == "args")
+ output += arguments(value);
+ else if (filter.empty())
+ output += value.is_string() ? value.get<std::string>() : literal(value);
+ else
+ throw std::logic_error("unknown template filter '" + filter + "'");
+ } else if (part.kind == TemplatePart::Kind::Include) {
+ const auto include = sources.find(part.value);
+ if (include == sources.end())
+ throw std::logic_error("unknown included template '" + part.value + "'");
+ render_parts(output, include->second, values, sources, depth + 1);
+ } else {
+ const auto* selected = &part.otherwise;
+ for (const auto& [condition, branch] : part.branches)
+ if (template_condition(values, condition)) {
+ selected = &branch;
+ break;
+ }
+ render_parts(output, *selected, values, sources, depth);
+ }
+ }
+ }
+
+ std::string render_template(const std::string& name, const nlohmann::json& values) {
+ static const auto sources = [] {
+ ParsedTemplates result;
+ for (const auto& [name, source] : templates::sources)
+ result.emplace(name, TemplateParser(name, source).parse());
+ return result;
+ }();
+ const auto source = sources.find(name);
+ if (source == sources.end())
+ throw std::logic_error("unknown generation template '" + name + "'");
+ std::string result;
+ render_parts(result, source->second, values, sources, 0);
+ return result;
+ }
+}
diff --git a/src/api/generation_template.h b/src/api/generation_template.h
new file mode 100644
index 0000000..2b7445e
--- /dev/null
+++ b/src/api/generation_template.h
@@ -0,0 +1,14 @@
+#ifndef __SD_API_GENERATION_TEMPLATE_H__
+#define __SD_API_GENERATION_TEMPLATE_H__
+
+#include <string>
+
+#include "json.hpp"
+
+namespace sd::api {
+ std::string literal(const nlohmann::json& value);
+ std::string arguments(const nlohmann::json& values);
+ std::string render_template(const std::string& name, const nlohmann::json& values);
+}
+
+#endif
diff --git a/src/api/generation_video.cpp b/src/api/generation_video.cpp
new file mode 100644
index 0000000..ef9c894
--- /dev/null
+++ b/src/api/generation_video.cpp
@@ -0,0 +1,155 @@
+#include "api/generation_program.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "model/vae/audio_geometry.hpp"
+
+namespace sd::api {
+ static void predictor_option(GenerationTemplate& request, const std::string& option) {
+ request.fields["predictor_options"] += option;
+ request.fields["negative_predictor_options"] += option;
+ }
+
+ static bool keyframe(GenerationTemplate& request, const char* key, const char* name) {
+ const auto* tensor = bound_tensor(request.options, key);
+ if (!tensor)
+ return false;
+ bind_tensor(request.program, name, tensor);
+ return true;
+ }
+
+ static void driving_audio(GenerationTemplate& request) {
+ const auto tracks = bound_audios(request.options, "ref_audios");
+ if (tracks.size() > 1)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "Wan S2V accepts one driving audio track");
+ request.flags["driving_audio"] = !tracks.empty();
+ if (tracks.empty())
+ return;
+ const auto& config = request.context.config.context;
+ if (!config.audio_encoder_path || !*config.audio_encoder_path)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "Wan S2V audio requires audio_encoder_path");
+ bind_tensor(request.program, "driving_audio", tracks[0].data);
+ request.values["audio_encoder_path"] = config.audio_encoder_path;
+ request.values["sample_rate"] = tracks[0].sample_rate;
+ request.fields["audio_encoder_options"] = arguments(loader_options(config, SDBackendModule::TE, request.definition));
+ request.fields["audio"] = "{samples = driving_audio[:, :math::floor(sample_rate * frames / fps)], sample_rate = sample_rate}";
+ predictor_option(request, ", audio_embedding = audio_embedding");
+ }
+
+ static void audiovisual(GenerationTemplate& request) {
+ const bool h3 = sd_version_is_minimax_h3(request.definition.version);
+ const auto& settings = request.context.config.context;
+ auto path = text(settings.audio_vae_path);
+ if (path.empty())
+ path = request.context.embedded_audio_vae;
+ request.flags["decode_audio"] = !path.empty();
+ if (!path.empty())
+ request.values["audio_vae_path"] = path;
+ else
+ request.values["empty_audio_shape"] = sd::vae::audio_latent_shape(request.definition.version, request.values.at("frames").get<int>(), request.values.at("fps").get<int>());
+ const auto video_shift = std::isfinite(request.image.sample_params.flow_shift) ? request.image.sample_params.flow_shift : request.definition.sampling.default_flow_shift;
+ if (h3 && video_shift <= 0)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "MiniMax-H3 flow shift must be positive");
+ request.values["audio_scale"] = h3 ? video_shift / 3.f : 1.f;
+ request.fields["audio_vae_options"] = arguments(loader_options(settings, SDBackendModule::VAE, request.definition));
+ predictor_option(request, h3 ? ", audio_length = audio_shape[0], fps = fps" : ", audio_length = audio_shape[1], fps = fps");
+ if (h3)
+ predictor_option(request, ", video_sigma_shift = " + literal(video_shift));
+ }
+
+ static void dual_expert(GenerationTemplate& request) {
+ const auto& settings = request.context.config.context;
+ request.flags["dual"] = !text(settings.high_noise_diffusion_model_path).empty();
+ if (!request.flags.at("dual").get<bool>())
+ return;
+ if (!sd_version_is_wan(request.definition.version) && !sd_version_is_lingbot_video(request.definition.version))
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "two-model sampling requires Wan or LingBot; use DGL for other compositions");
+ request.values["high_model_path"] = settings.high_noise_diffusion_model_path;
+ request.values["moe_boundary"] = request.video->moe_boundary;
+ request.flags["fixed_high_steps"] = request.video->high_noise_steps >= 0;
+ if (request.video->high_noise_steps >= 0) {
+ const int64_t high_steps = request.video->high_noise_steps;
+ const int64_t total_steps = request.image.sample_params.sample_steps + high_steps;
+ if (!request.image.sample_params.custom_sigmas_count && total_steps > 10000)
+ throw dgl::Error(SD_STATUS_OUT_OF_RANGE, "combined low/high-noise steps exceed 10000");
+ request.values["steps"] = total_steps;
+ request.values["high_noise_steps"] = high_steps;
+ if (request.image.sample_params.custom_sigmas_count) {
+ request.values["steps"] = request.image.sample_params.custom_sigmas_count - 1;
+ if (high_steps > 0)
+ request.values["high_noise_steps"] = std::max<int64_t>(0, request.image.sample_params.custom_sigmas_count - 1 - request.image.sample_params.sample_steps);
+ }
+ }
+ std::string high = "high_base", patches;
+ for (uint32_t i = 0; i < request.video->lora_count; ++i) {
+ const auto& lora = request.video->loras[i];
+ if (!lora.is_high_noise)
+ continue;
+ patches += request.render("lora", {{"name", "high_lora_" + std::to_string(i)}, {"path", text(lora.path)}, {"diffusion", true}, {"conditioning", false}, {"model_result", "high_model_" + std::to_string(i)}, {"model_source", high}, {"weight", lora.multiplier}, {"mode", sd_lora_apply_mode_name(settings.lora_apply_mode)}});
+ high = "high_model_" + std::to_string(i);
+ }
+ request.fields["high_loras"] = patches;
+ request.fields["high_model"] = high;
+ }
+
+ void prepare_video_program(GenerationTemplate& request) {
+ const auto version = request.definition.version;
+ const auto& settings = request.context.config.context;
+ const bool first = keyframe(request, "init_image", "first_frame");
+ const bool last = keyframe(request, "end_image", "last_frame");
+ const bool vace = request.context.wan_vace;
+ const bool s2v = version == VERSION_WAN2_2_S2V;
+ const bool frozen = (version == VERSION_WAN2_2_TI2V || sd_version_is_lingbot_video(version)) && (first || last);
+ const bool ltx = sd_version_is_ltxav(version);
+ const bool h3 = version == VERSION_MINIMAX_H3;
+ if (!h3 && !bound_videos(request.options, "ref_videos").empty())
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "reference videos require MiniMax-H3");
+ if (!h3 && !s2v && !bound_audios(request.options, "ref_audios").empty())
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "reference audio requires MiniMax-H3 or Wan2.2 S2V");
+ if (s2v && last)
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "Wan2.2 S2V does not support an end image");
+ const bool image_condition = !vace && !s2v && request.context.wan_image_condition;
+ request.flags = {{"first", first}, {"last", last}, {"conditioned", first || last}, {"vace", vace}, {"vace_reference", vace && first}, {"s2v_reference", s2v && first}, {"driving_audio", false}, {"image_condition", image_condition}, {"vision", image_condition && request.context.wan_vision_condition}, {"end_vision", request.context.wan_end_vision}, {"frozen", frozen}, {"hunyuan", sd_version_is_hunyuan_video(version) && (first || last)}, {"controls", false}, {"dual", false}, {"custom_sigmas", request.image.sample_params.custom_sigmas_count > 0}};
+ request.fields["entry"] = ltx ? "video_ltx" : h3 ? "video_h3"
+ : "video_wan";
+ if (image_condition || request.flags.at("hunyuan").get<bool>())
+ predictor_option(request, ", concat = video_concat");
+ if (frozen)
+ request.fields["sample_options"] += ", base = base, denoise_mask = video_mask, mask_input = true";
+ if (vace) {
+ const auto* frames = bound_tensor(request.options, "control_frames");
+ if (frames) {
+ bind_tensor(request.program, "control_video", frames);
+ request.values["control_count"] = std::min(sd_tensor_dim(frames, 3), request.values.at("frames").get<int64_t>());
+ }
+ request.flags["controls"] = frames != nullptr;
+ predictor_option(request, ", vace_context = vace_context, vace_strength = " + literal(request.video->vace_strength));
+ } else if (bound_tensor(request.options, "control_frames")) {
+ throw dgl::Error(SD_STATUS_UNSUPPORTED, "control frames require a VACE model or an explicit DGL workflow");
+ }
+ if (s2v) {
+ if (first)
+ predictor_option(request, ", ref_latents = [reference_latent]");
+ driving_audio(request);
+ }
+ if (request.flags.at("vision").get<bool>()) {
+ if (!settings.clip_vision_path || !*settings.clip_vision_path)
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, "Wan image conditioning requires clip_vision_path");
+ request.values["vision_path"] = settings.clip_vision_path;
+ request.fields["vision_options"] = arguments(loader_options(settings, SDBackendModule::TE, request.definition));
+ }
+ if (ltx || h3) {
+ audiovisual(request);
+ if (ltx && (first || last)) {
+ request.values["strength"] = request.image.strength;
+ request.fields["sample_options"] += ", base = base, denoise_mask = packed_mask, mask_input = true";
+ if (last)
+ predictor_option(request, ", video_positions = positions");
+ }
+ if (h3)
+ prepare_h3_references(request, first ? "first_frame_pixels" : "", last ? "last_frame_pixels" : "");
+ }
+ dual_expert(request);
+ }
+}
diff --git a/examples/cli/image_metadata.cpp b/src/api/image_metadata.cpp
similarity index 96%
rename from examples/cli/image_metadata.cpp
rename to src/api/image_metadata.cpp
index bb5ab7c..5924800 100644
--- a/examples/cli/image_metadata.cpp
+++ b/src/api/image_metadata.cpp
@@ -1,4 +1,4 @@
-#include "image_metadata.h"
+#include "api/api.h"
#include <algorithm>
#include <cctype>
@@ -26,7 +26,7 @@ extern "C" int mz_uncompress(unsigned char* pDest,
const unsigned char* pSource,
unsigned long source_len);
-namespace {
+namespace sd::metadata_reader {
constexpr int MZ_OK = 0;
constexpr int MZ_BUF_ERROR = -5;
@@ -988,13 +988,13 @@ namespace {
(marker == 0xFE) || (marker >= 0xE0 && marker <= 0xEF);
if (marker == 0xFE) {
- std::string comment = bytes_to_string(payload, payload + payload_size);
- if (comment.rfind("parameters", 0) == 0 &&
- comment.size() > std::string("parameters").size() &&
- comment[std::string("parameters").size()] == '\0') {
+ std::string comment = bytes_to_string(payload, payload + payload_size);
+ const auto separator = comment.find('\0');
+ const auto label = comment.substr(0, separator);
+ if (separator != std::string::npos && (label == "parameters" || label == "dgl")) {
entry["data"] = json{
- {"label", "parameters"},
- {"text", comment.substr(std::string("parameters").size() + 1)},
+ {"label", label},
+ {"text", trim_trailing_nuls(comment.substr(separator + 1))},
};
} else {
entry["data"] = json{{"text", trim_trailing_nuls(comment)}};
@@ -1430,24 +1430,31 @@ namespace {
return false;
}
-} // namespace
-
-bool print_image_metadata(const std::string& image_path,
- const MetadataReadOptions& options,
- std::ostream& out,
- std::string& error) {
- json report;
- if (!build_metadata_report(image_path, options.include_raw, report, error)) {
- return false;
- }
-
- json visible_report = filter_visible_entries(report, options.include_structural);
-
- if (options.output_format == MetadataOutputFormat::JSON) {
- visible_report["file"] = image_path;
- out << visible_report.dump(2, ' ', false, json::error_handler_t::replace) << "\n";
- } else {
- print_text_report(image_path, visible_report, options.include_structural, options.brief, out);
- }
- return true;
+} // namespace sd::metadata_reader
+
+static thread_local std::string metadata_report_text;
+
+const char* sd_read_metadata(const char* path, uint32_t flags) {
+ const char* result = nullptr;
+ sd::api::call([&] {
+ constexpr uint32_t allowed = SD_METADATA_READ_JSON | SD_METADATA_READ_RAW | SD_METADATA_READ_BRIEF | SD_METADATA_READ_STRUCTURAL;
+ if (!path || (flags & ~allowed))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "path is required and metadata flags must be recognized");
+ json report;
+ std::string error;
+ if (!sd::metadata_reader::build_metadata_report(path, flags & SD_METADATA_READ_RAW, report, error))
+ return sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, error.c_str());
+ auto visible_report = sd::metadata_reader::filter_visible_entries(report, flags & SD_METADATA_READ_STRUCTURAL);
+ std::ostringstream output;
+ if (flags & SD_METADATA_READ_JSON) {
+ visible_report["file"] = path;
+ output << visible_report.dump(2, ' ', false, json::error_handler_t::replace) << "\n";
+ } else {
+ sd::metadata_reader::print_text_report(path, visible_report, flags & SD_METADATA_READ_STRUCTURAL, flags & SD_METADATA_READ_BRIEF, output);
+ }
+ metadata_report_text = output.str();
+ result = metadata_report_text.c_str();
+ return SD_STATUS_OK;
+ });
+ return result;
}
diff --git a/src/api/media_metadata.cpp b/src/api/media_metadata.cpp
new file mode 100644
index 0000000..ce3e5fc
--- /dev/null
+++ b/src/api/media_metadata.cpp
@@ -0,0 +1,173 @@
+#include "api/metadata.h"
+
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+
+#include "api/api.h"
+
+namespace sd::metadata {
+ static std::string xml_text(const std::string& info) {
+ std::string escaped;
+ for (char c : info) {
+ switch (c) {
+ case '&':
+ escaped += "&";
+ break;
+ case '<':
+ escaped += "<";
+ break;
+ case '>':
+ escaped += ">";
+ break;
+ case '"':
+ escaped += """;
+ break;
+ case '\'':
+ escaped += "'";
+ break;
+ default:
+ escaped += c;
+ break;
+ }
+ }
+ return escaped;
+ }
+
+ std::string generation_xmp(const std::string& source, const std::string& parameters) {
+ if (source.empty())
+ return "";
+ return "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">"
+ "<rdf:Description rdf:about=\"\" xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">"
+ "<sdcpp:dgl>" +
+ xml_text(source) + "</sdcpp:dgl><sdcpp:parameters>" + xml_text(parameters) +
+ "</sdcpp:parameters></rdf:Description></rdf:RDF></x:xmpmeta>";
+ }
+
+ static bool jpeg_comment(std::vector<uint8_t>& comments, const std::string& label, const std::string& value) {
+ if (value.empty())
+ return true;
+ const size_t size = label.size() + 1 + value.size() + 1;
+ if (size > jpeg_comment_max_bytes)
+ return false;
+ const size_t length = size + 2;
+ std::vector<uint8_t> segment{0xff, 0xfe, uint8_t(length >> 8), uint8_t(length)};
+ segment.insert(segment.end(), label.begin(), label.end());
+ segment.push_back(0);
+ segment.insert(segment.end(), value.begin(), value.end());
+ segment.push_back(0);
+ comments.insert(comments.end(), segment.begin(), segment.end());
+ return true;
+ }
+
+ bool jpeg_metadata(std::vector<uint8_t>& comments, const std::string& source, const std::string& parameters) {
+ return source.empty() || (jpeg_comment(comments, "parameters", parameters) && jpeg_comment(comments, "dgl", source));
+ }
+
+ bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& source, const std::string& parameters) {
+ if (source.empty())
+ return true;
+ std::vector<uint8_t> comments;
+ if (image.size() < 2 || !jpeg_metadata(comments, source, parameters))
+ return false;
+ image.insert(image.begin() + 2, comments.begin(), comments.end());
+ return true;
+ }
+
+ static void append_u32(std::vector<uint8_t>& bytes, uint32_t value) {
+ for (int i = 3; i >= 0; --i)
+ bytes.push_back(uint8_t(value >> (8 * i)));
+ }
+
+ static bool png_text(std::vector<uint8_t>& chunks, const std::string& label, const std::string& value) {
+ if (value.empty())
+ return true;
+ const size_t size = label.size() + 5 + value.size();
+ if (size > std::numeric_limits<uint32_t>::max())
+ return false;
+ append_u32(chunks, uint32_t(size));
+ const size_t start = chunks.size();
+ chunks.insert(chunks.end(), {'i', 'T', 'X', 't'});
+ chunks.insert(chunks.end(), label.begin(), label.end());
+ chunks.insert(chunks.end(), 5, 0);
+ chunks.insert(chunks.end(), value.begin(), value.end());
+ uint32_t crc = 0xffffffff;
+ for (size_t i = start; i < chunks.size(); ++i) {
+ crc ^= chunks[i];
+ for (int bit = 0; bit < 8; ++bit)
+ crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1)));
+ }
+ append_u32(chunks, ~crc);
+ return true;
+ }
+
+ bool png_metadata(std::vector<uint8_t>& chunks, const std::string& source, const std::string& parameters) {
+ return source.empty() || (png_text(chunks, "dgl", source) && png_text(chunks, "parameters", parameters));
+ }
+
+ bool embed_png_metadata(std::vector<uint8_t>& image, const std::string& source, const std::string& parameters) {
+ if (source.empty())
+ return true;
+ if (image.size() < 33 || std::memcmp(image.data() + 12, "IHDR", 4))
+ return false;
+ std::vector<uint8_t> chunks;
+ if (!png_metadata(chunks, source, parameters))
+ return false;
+ image.insert(image.begin() + 33, chunks.begin(), chunks.end());
+ return true;
+ }
+
+ std::vector<uint8_t> avi_info(const std::string& source) {
+ if (source.empty())
+ return {};
+ if (source.size() > std::numeric_limits<uint32_t>::max() - 22)
+ throw std::runtime_error("generation metadata exceeds AVI container capacity");
+ const auto size = uint32_t(source.size() + 1);
+ std::vector<uint8_t> bytes{'L', 'I', 'S', 'T'};
+ auto append_le = [&](uint32_t value) {
+ for (int i = 0; i < 4; ++i)
+ bytes.push_back(uint8_t(value >> (8 * i)));
+ };
+ append_le(12 + size + (size & 1));
+ bytes.insert(bytes.end(), {'I', 'N', 'F', 'O', 'I', 'C', 'M', 'T'});
+ append_le(size);
+ bytes.insert(bytes.end(), source.begin(), source.end());
+ bytes.push_back(0);
+ if (size & 1)
+ bytes.push_back(0);
+ return bytes;
+ }
+}
+
+static thread_local std::vector<uint8_t> metadata_chunk_bytes;
+
+sd_status_t sd_get_metadata_chunk(const char* source, sd_metadata_format_t format, const void** data_out, size_t* size_out) {
+ if (data_out)
+ *data_out = nullptr;
+ if (size_out)
+ *size_out = 0;
+ return sd::api::call([&] {
+ if (!source || !data_out || !size_out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "source, data_out and size_out are required");
+ auto& bytes = metadata_chunk_bytes;
+ bytes.clear();
+ if (format == SD_METADATA_AVI) {
+ bytes = sd::metadata::avi_info(source);
+ } else {
+ if (format != SD_METADATA_PNG && format != SD_METADATA_JPEG && format != SD_METADATA_XMP)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "unknown metadata format");
+ const auto info = sd::api::generation_metadata(source);
+ const auto parameters = info.value("infotext", "");
+ if (format == SD_METADATA_XMP) {
+ const auto text = sd::metadata::generation_xmp(source, parameters);
+ bytes.assign(text.begin(), text.end());
+ } else if (!(format == SD_METADATA_PNG ? sd::metadata::png_metadata(bytes, source, parameters)
+ : sd::metadata::jpeg_metadata(bytes, source, parameters))) {
+ return sd::api::error(SD_STATUS_OUT_OF_RANGE, nullptr, "generation metadata exceeds container capacity");
+ }
+ }
+ *data_out = bytes.empty() ? nullptr : bytes.data();
+ *size_out = bytes.size();
+ return SD_STATUS_OK;
+ });
+}
diff --git a/src/api/metadata.h b/src/api/metadata.h
new file mode 100644
index 0000000..b33837c
--- /dev/null
+++ b/src/api/metadata.h
@@ -0,0 +1,25 @@
+#ifndef __SD_API_METADATA_H__
+#define __SD_API_METADATA_H__
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "json.hpp"
+
+namespace sd::api {
+ nlohmann::json generation_metadata(const std::string& source) noexcept;
+}
+
+namespace sd::metadata {
+ constexpr std::size_t jpeg_comment_max_bytes = 65533;
+ std::string generation_xmp(const std::string& source, const std::string& parameters = {});
+ bool png_metadata(std::vector<uint8_t>& chunks, const std::string& source, const std::string& parameters);
+ bool jpeg_metadata(std::vector<uint8_t>& comments, const std::string& source, const std::string& parameters);
+ bool embed_png_metadata(std::vector<uint8_t>& image, const std::string& source, const std::string& parameters = {});
+ bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& source, const std::string& parameters = {});
+ std::vector<uint8_t> avi_info(const std::string& source);
+}
+
+#endif // __SD_API_METADATA_H__
diff --git a/src/api/options.cpp b/src/api/options.cpp
index 591cce3..be4a2a7 100644
--- a/src/api/options.cpp
+++ b/src/api/options.cpp
@@ -1,5 +1,6 @@
#include "options.h"
#include "core/option_types.h"
+#include "core/util.h"
#include "tensor.h"
#include <algorithm>
@@ -8,6 +9,7 @@
#include <limits>
#include <map>
#include <new>
+#include <set>
#include <stdexcept>
#include <string>
#include <tuple>
@@ -97,16 +99,6 @@ static sd_vae_format_t parse_vae_format(const char* text) {
return SD_VAE_FORMAT_COUNT;
}
-static sd_hires_upscaler_t parse_hires_upscaler(const char* text) {
- static const char* names[] = {"none", "latent", "latent_nearest", "latent_nearest_exact",
- "latent_antialiased", "latent_bicubic", "latent_bicubic_antialiased",
- "lanczos", "nearest", "model"};
- for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; ++i)
- if (strcmp(text, names[i]) == 0)
- return static_cast<sd_hires_upscaler_t>(i);
- return SD_HIRES_UPSCALER_COUNT;
-}
-
static bool valid_sigmas(const ParamValue& value) {
const auto& sigmas = std::get<std::vector<float>>(value);
if (sigmas.empty())
@@ -119,11 +111,6 @@ static bool valid_sigmas(const ParamValue& value) {
return true;
}
-static bool valid_layers(const ParamValue& value) {
- const auto& layers = std::get<std::vector<int>>(value);
- return std::all_of(layers.begin(), layers.end(), [](int index) { return index >= 0; });
-}
-
template <typename Value, typename Apply>
static ParamSpec parameter(const char* key, ParamType type, int operations, Apply apply) {
return {key, type, operations, [apply](const ParamValue& value, sd::OptConfig& config) {
@@ -311,6 +298,7 @@ static void add_context_specs(std::vector<ParamSpec>& specs) {
ctx.string("clip_vision_path", &sd::ContextConfig::clip_vision_path);
ctx.integer("conditioning_cache_size", &sd::ContextConfig::conditioning_cache_size, 0, INT32_MAX);
ctx.string("control_net_path", &sd::ContextConfig::control_net_path);
+ ctx.string("max_ram", &sd::ContextConfig::max_ram);
ctx.boolean("diffusion_conv_direct", &sd::ContextConfig::diffusion_conv_direct);
ctx.boolean("diffusion_flash_attn", &sd::ContextConfig::diffusion_flash_attn);
ctx.boolean("disable_prefetch", &sd::ContextConfig::disable_prefetch);
@@ -322,18 +310,14 @@ static void add_context_specs(std::vector<ParamSpec>& specs) {
ctx.boolean("flash_attn", &sd::ContextConfig::flash_attn);
ctx.boolean("force_sdxl_vae_conv_scale", &sd::ContextConfig::force_sdxl_vae_conv_scale);
ctx.string("high_noise_diffusion_model_path", &sd::ContextConfig::high_noise_diffusion_model_path);
- ctx.string("ip_adapter_path", &sd::ContextConfig::ip_adapter_path);
ctx.number("linear_scale", &sd::ContextConfig::linear_scale, 0, std::numeric_limits<float>::max());
ctx.string("llm_path", &sd::ContextConfig::llm_path);
ctx.string("llm_vision_path", &sd::ContextConfig::llm_vision_path);
ctx.choice("lora_apply_mode", &sd::ContextConfig::lora_apply_mode, str_to_lora_apply_mode, LORA_APPLY_MODE_COUNT);
ctx.string("max_vram", &sd::ContextConfig::max_vram);
ctx.string("model_args", &sd::ContextConfig::model_args);
- ctx.for_operations(Context | Upscaler | ADetailer).string("model_path", &sd::ContextConfig::model_path);
- ctx.string("motion_module_path", &sd::ContextConfig::motion_module_path);
- ctx.string("photo_maker_path", &sd::ContextConfig::photo_maker_path);
+ ctx.for_operations(Context | Upscaler).string("model_path", &sd::ContextConfig::model_path);
ctx.choice("prediction", &sd::ContextConfig::prediction, str_to_prediction, PREDICTION_COUNT, true);
- ctx.string("pulid_weights_path", &sd::ContextConfig::pulid_weights_path);
ctx.choice("rng_type", &sd::ContextConfig::rng_type, str_to_rng_type, RNG_TYPE_COUNT, true);
ctx.string("rpc_servers", &sd::ContextConfig::rpc_servers);
ctx.boolean("sage_attn", &sd::ContextConfig::sage_attn);
@@ -349,116 +333,78 @@ static void add_context_specs(std::vector<ParamSpec>& specs) {
ctx.choice("vae_format", &sd::ContextConfig::vae_format, parse_vae_format, SD_VAE_FORMAT_COUNT);
ctx.choice("wtype", &sd::ContextConfig::wtype, str_to_sd_type, SD_TYPE_COUNT, true);
- specs.push_back(parameter<std::string>("backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.backend = value.c_str(); config.post.video_upscaler.backend = value.c_str(); }));
- specs.push_back(parameter<std::string>("conv_direct_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.conv_direct_mode = value.c_str(); config.post.video_upscaler.conv_direct_mode = value.c_str(); }));
- specs.push_back(parameter<std::string>("diffusion_model_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.diffusion_model_path = value.c_str(); config.post.video_upscaler.diffusion_model_path = value.c_str(); }));
- specs.push_back(parameter<std::string>("flash_attn_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.flash_attn_mode = value.c_str(); config.post.video_upscaler.flash_attn_mode = value.c_str(); }));
+ specs.push_back(parameter<std::string>("backend", ParamType::String, Context | Upscaler, [](const std::string& value, sd::OptConfig& config) { config.context.backend = value.c_str(); }));
+ specs.push_back(parameter<std::string>("conv_direct_mode", ParamType::String, Context, [](const std::string& value, sd::OptConfig& config) { config.context.conv_direct_mode = value.c_str(); }));
+ specs.push_back(parameter<std::string>("diffusion_model_path", ParamType::String, Context, [](const std::string& value, sd::OptConfig& config) { config.context.diffusion_model_path = value.c_str(); }));
+ specs.push_back(parameter<std::string>("flash_attn_mode", ParamType::String, Context, [](const std::string& value, sd::OptConfig& config) { config.context.flash_attn_mode = value.c_str(); }));
specs.push_back(integer_parameter(
- "n_threads", Context | Upscaler | VidUpscaler | ADetailer, [](const int64_t& value, sd::OptConfig& config) { config.context.n_threads = static_cast<int>(value); config.post.video_upscaler.n_threads = static_cast<int>(value); }, 1, INT32_MAX));
- specs.push_back(parameter<std::string>("params_backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.params_backend = value.c_str(); config.post.video_upscaler.params_backend = value.c_str(); }));
- specs.push_back(parameter<std::string>("vae_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.vae_path = value.c_str(); config.post.video_upscaler.vae_path = value.c_str(); }));
+ "n_threads", Context | Upscaler, [](const int64_t& value, sd::OptConfig& config) { config.context.n_threads = static_cast<int>(value); }, 1, INT32_MAX));
+ specs.push_back(parameter<std::string>("params_backend", ParamType::String, Context | Upscaler, [](const std::string& value, sd::OptConfig& config) { config.context.params_backend = value.c_str(); }));
+ specs.push_back(parameter<std::string>("vae_path", ParamType::String, Context, [](const std::string& value, sd::OptConfig& config) { config.context.vae_path = value.c_str(); }));
}
static void add_generation_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
ParamGroup img(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image); });
ParamGroup vid(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video); });
- ParamGroup preprocess(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.image_preprocess, config.video.image_preprocess); });
- ParamGroup photomaker(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pm_params); });
- ParamGroup pulid(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pulid_params); });
img.integer("batch_count", &sd::ImageConfig::batch_count, 1, INT32_MAX);
img.image("control_image", &sd::ImageConfig::control_image);
img.number("control_strength", &sd::ImageConfig::control_strength, 0, std::numeric_limits<float>::max());
img.choice("control_type", &sd::ImageConfig::control_type, str_to_sd_control_type, SD_CONTROL_TYPE_COUNT);
- img.image("ip_adapter_image", &sd::ImageConfig::ip_adapter_image);
- img.number("ip_adapter_strength", &sd::ImageConfig::ip_adapter_strength, 0, std::numeric_limits<float>::max());
img.image("mask_image", &sd::ImageConfig::mask_image);
img.integer("qwen_image_layers", &sd::ImageConfig::qwen_image_layers, 0, INT32_MAX);
vid.frames("control_frames", &sd::VideoConfig::control_frames, &sd::VideoConfig::control_frames_size);
vid.image("end_image", &sd::VideoConfig::end_image);
vid.number("moe_boundary", &sd::VideoConfig::moe_boundary, 0, 1);
+ vid.integer("high_noise_sample.steps", &sd::VideoConfig::high_noise_steps, -1, INT32_MAX);
vid.array("ref_audios", &sd::VideoConfig::ref_audios, &sd::VideoConfig::ref_audios_count);
vid.array("ref_videos", &sd::VideoConfig::ref_videos, &sd::VideoConfig::ref_videos_count);
vid.number("vace_strength", &sd::VideoConfig::vace_strength, 0, std::numeric_limits<float>::max());
vid.integer("video_frames", &sd::VideoConfig::video_frames, 1, INT32_MAX);
- preprocess.string("image_preprocess.rules", &sd::ImagePreprocessConfig::rules);
-
- photomaker.string("photomaker.id_embed_path", &sd::PhotoMakerConfig::id_embed_path);
- photomaker.array("photomaker.id_images", &sd::PhotoMakerConfig::id_images, &sd::PhotoMakerConfig::id_images_count);
- photomaker.number("photomaker.style_strength", &sd::PhotoMakerConfig::style_strength, 0, 100);
-
- pulid.string("pulid.id_embedding_path", &sd::PuLIDConfig::id_embedding_path);
- pulid.number("pulid.id_weight", &sd::PuLIDConfig::id_weight, 0, std::numeric_limits<float>::max());
-
- specs.push_back(parameter<bool>("circular_x", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_x = value; config.video.circular_x = value; }));
- specs.push_back(parameter<bool>("circular_y", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_y = value; config.video.circular_y = value; }));
+ specs.push_back(parameter<bool>("circular_x", ParamType::Bool, Image | Video, [](const bool& value, sd::OptConfig& config) { config.image.circular_x = value; config.video.circular_x = value; }));
+ specs.push_back(parameter<bool>("circular_y", ParamType::Bool, Image | Video, [](const bool& value, sd::OptConfig& config) { config.image.circular_y = value; config.video.circular_y = value; }));
specs.push_back(integer_parameter(
- "clip_skip", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.clip_skip = static_cast<int>(value); config.video.clip_skip = static_cast<int>(value); }, INT32_MIN, INT32_MAX));
+ "clip_skip", Image | Video, [](const int64_t& value, sd::OptConfig& config) { config.image.clip_skip = static_cast<int>(value); config.video.clip_skip = static_cast<int>(value); }, INT32_MIN, INT32_MAX));
specs.push_back(integer_parameter(
- "fps", Video | VidUpscale, [](const int64_t& value, sd::OptConfig& config) { config.video.fps = static_cast<int>(value); config.post.video_upscale.fps = static_cast<int>(value); }, 1, INT32_MAX));
+ "fps", Video, [](const int64_t& value, sd::OptConfig& config) { config.video.fps = static_cast<int>(value); }, 1, INT32_MAX));
specs.push_back(integer_parameter(
- "height", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.height = static_cast<int>(value); config.video.height = static_cast<int>(value); }, 1, INT32_MAX));
+ "height", Image | Video, [](const int64_t& value, sd::OptConfig& config) { config.image.height = static_cast<int>(value); config.video.height = static_cast<int>(value); }, 1, INT32_MAX));
specs.push_back(parameter<TensorValue>("init_image", ParamType::Tensor, Image | Video, [](const TensorValue& value, sd::OptConfig& config) { config.image.init_image = sd::api::image_view(*value.tensor.get()); config.video.init_image = config.image.init_image; }));
- specs.push_back(parameter<LoraValues>("loras", ParamType::Loras, Image | Video | ADetail, [](const LoraValues& value, sd::OptConfig& config) { config.image.loras = value.items.data(); config.image.lora_count = static_cast<decltype(config.image.lora_count)>(value.items.size()); config.video.loras = value.items.data(); config.video.lora_count = static_cast<decltype(config.video.lora_count)>(value.items.size()); }));
- specs.push_back(parameter<std::string>("negative_prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.negative_prompt = value.c_str(); config.video.negative_prompt = value.c_str(); }));
- specs.push_back(parameter<std::string>("prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.prompt = value.c_str(); config.video.prompt = value.c_str(); }));
- specs.push_back(parameter<std::string>("ref_image_args", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.ref_image_args = value.c_str(); config.video.ref_image_args = value.c_str(); }));
+ specs.push_back(parameter<LoraValues>("loras", ParamType::Loras, Image | Video, [](const LoraValues& value, sd::OptConfig& config) { config.image.loras = value.items.data(); config.image.lora_count = static_cast<decltype(config.image.lora_count)>(value.items.size()); config.video.loras = value.items.data(); config.video.lora_count = static_cast<decltype(config.video.lora_count)>(value.items.size()); }));
+ specs.push_back(parameter<std::string>("negative_prompt", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.negative_prompt = value.c_str(); config.video.negative_prompt = value.c_str(); }));
+ specs.push_back(parameter<std::string>("prompt", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.prompt = value.c_str(); config.video.prompt = value.c_str(); }));
specs.push_back(parameter<ImageValues>("ref_images", ParamType::Tensors, Image | Video, [](const ImageValues& value, sd::OptConfig& config) { config.image.ref_images = const_cast<sd::Image*>(value.items.data()); config.image.ref_images_count = static_cast<int>(value.items.size()); config.video.ref_images = config.image.ref_images; config.video.ref_images_count = config.image.ref_images_count; }));
- specs.push_back(integer_parameter("seed", Image | Video | VidUpscale | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.seed = static_cast<int64_t>(value); config.video.seed = static_cast<int64_t>(value); config.post.video_upscale.seed = value; }));
+ specs.push_back(integer_parameter("seed", Image | Video, [](const int64_t& value, sd::OptConfig& config) { config.image.seed = static_cast<int64_t>(value); config.video.seed = static_cast<int64_t>(value); }));
specs.push_back(float_parameter(
- "strength", Image | Video | ADetail, [](const double& value, sd::OptConfig& config) { config.image.strength = static_cast<float>(value); config.video.strength = static_cast<float>(value); }, 0, 1));
+ "strength", Image | Video, [](const double& value, sd::OptConfig& config) { config.image.strength = static_cast<float>(value); config.video.strength = static_cast<float>(value); }, 0, 1));
specs.push_back(integer_parameter(
- "width", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.width = static_cast<int>(value); config.video.width = static_cast<int>(value); }, 1, INT32_MAX));
+ "width", Image | Video, [](const int64_t& value, sd::OptConfig& config) { config.image.width = static_cast<int>(value); config.video.width = static_cast<int>(value); }, 1, INT32_MAX));
}
static void add_sampling_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
- ParamGroup high_noise(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params); });
- ParamGroup high_noise_guidance(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance); });
- ParamGroup high_noise_slg(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance.slg); });
- ParamGroup sample(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params, config.video.sample_params); });
- ParamGroup guidance(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance, config.video.sample_params.guidance); });
- ParamGroup slg(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance.slg, config.video.sample_params.guidance.slg); });
-
- high_noise.string("high_noise_sample.args", &sd::SampleConfig::extra_sample_args);
- high_noise.number("high_noise_sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits<float>::max());
- high_noise.choice("high_noise_sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true);
- high_noise.integer("high_noise_sample.steps", &sd::SampleConfig::sample_steps, -1, INT32_MAX);
-
- high_noise_guidance.number("high_noise_sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance);
- high_noise_guidance.number("high_noise_sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg);
- high_noise_guidance.number("high_noise_sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg);
-
- high_noise_slg.number("high_noise_sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1);
- high_noise_slg.number("high_noise_sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1);
- high_noise_slg.array("high_noise_sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative");
- high_noise_slg.number("high_noise_sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits<float>::max());
+ ParamGroup sample(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.sample_params, config.video.sample_params); });
+ ParamGroup guidance(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance, config.video.sample_params.guidance); });
sample.string("sample.args", &sd::SampleConfig::extra_sample_args);
sample.number("sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits<float>::max());
sample.number("sample.flow_shift", &sd::SampleConfig::flow_shift, 0, std::numeric_limits<float>::max());
sample.choice("sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true);
sample.choice("sample.scheduler", &sd::SampleConfig::scheduler, str_to_scheduler, SCHEDULER_COUNT, true);
- sample.integer("sample.shifted_timestep", &sd::SampleConfig::shifted_timestep, 0, INT32_MAX);
sample.array("sample.sigmas", &sd::SampleConfig::custom_sigmas, &sd::SampleConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array");
sample.integer("sample.steps", &sd::SampleConfig::sample_steps, 1, INT32_MAX);
guidance.number("sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance);
guidance.number("sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg);
guidance.number("sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg);
-
- slg.number("sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1);
- slg.number("sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1);
- slg.array("sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative");
- slg.number("sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits<float>::max());
}
static void add_cache_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
- ParamGroup cache(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.cache, config.video.cache); });
+ ParamGroup cache(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.cache, config.video.cache); });
cache.integer("cache.bn_compute_blocks", &sd::CacheConfig::Bn_compute_blocks, 0, INT32_MAX);
cache.number("cache.end_percent", &sd::CacheConfig::end_percent, 0, 1);
@@ -489,22 +435,21 @@ static void add_cache_specs(std::vector<ParamSpec>& specs) {
static void add_hires_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
ParamGroup hires(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.hires, config.video.hires); });
-
- hires.number("hires.denoising_strength", &sd::HiresConfig::denoising_strength, 0, 1);
hires.boolean("hires.enabled", &sd::HiresConfig::enabled);
+ hires.string("hires.upscaler", &sd::HiresConfig::upscaler);
hires.string("hires.model_path", &sd::HiresConfig::model_path);
- hires.number("hires.scale", &sd::HiresConfig::scale, 0, std::numeric_limits<float>::max());
- hires.array("hires.sigmas", &sd::HiresConfig::custom_sigmas, &sd::HiresConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array");
- hires.integer("hires.steps", &sd::HiresConfig::steps, 0, INT32_MAX);
- hires.integer("hires.target_height", &sd::HiresConfig::target_height, 0, INT32_MAX);
- hires.integer("hires.target_width", &sd::HiresConfig::target_width, 0, INT32_MAX);
+ hires.number("hires.scale", &sd::HiresConfig::scale, std::numeric_limits<float>::min(), 64);
+ hires.integer("hires.target_width", &sd::HiresConfig::target_width, 0, 16384);
+ hires.integer("hires.target_height", &sd::HiresConfig::target_height, 0, 16384);
+ hires.integer("hires.steps", &sd::HiresConfig::steps, 0, 10000);
+ hires.number("hires.denoising_strength", &sd::HiresConfig::denoising_strength, 0.0001, 1);
hires.integer("hires.upscale_tile_size", &sd::HiresConfig::upscale_tile_size, 1, INT32_MAX);
- hires.choice("hires.upscaler", &sd::HiresConfig::upscaler, parse_hires_upscaler, SD_HIRES_UPSCALER_COUNT);
+ hires.array("hires.sigmas", &sd::HiresConfig::custom_sigmas, &sd::HiresConfig::custom_sigmas_count, valid_sigmas, "hires sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array");
}
static void add_tiling_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
- ParamGroup tiling(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.vae_tiling_params, config.video.vae_tiling_params); });
+ ParamGroup tiling(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.vae_tiling_params, config.video.vae_tiling_params); });
tiling.boolean("vae_tiling.enabled", &sd::TilingConfig::enabled);
tiling.string("vae_tiling.extra_tiling_args", &sd::TilingConfig::extra_tiling_args);
@@ -518,26 +463,23 @@ static void add_tiling_specs(std::vector<ParamSpec>& specs) {
static void add_postprocess_specs(std::vector<ParamSpec>& specs) {
using namespace sd::api;
- ParamGroup input(specs, Upscale | ADetail, [](sd::OptConfig& config) { return std::tie(config.post); });
+ ParamGroup input(specs, Upscale, [](sd::OptConfig& config) { return std::tie(config.post); });
ParamGroup upscaler(specs, Upscaler, [](sd::OptConfig& config) { return std::tie(config.post); });
- ParamGroup video_upscaler(specs, VidUpscaler, [](sd::OptConfig& config) { return std::tie(config.post.video_upscaler); });
- ParamGroup video_upscale(specs, VidUpscale, [](sd::OptConfig& config) { return std::tie(config.post.video_upscale); });
- ParamGroup adetail(specs, ADetail, [](sd::OptConfig& config) { return std::tie(config.post.adetail); });
-
input.image("image", &sd::PostprocessConfig::image);
-
+ input.integer("repeats", &sd::PostprocessConfig::repeats, 1, 16);
upscaler.boolean("conv_direct", &sd::PostprocessConfig::conv_direct);
upscaler.integer("tile_size", &sd::PostprocessConfig::tile_size, 1, INT32_MAX);
+}
- video_upscaler.integer("upscale_factor", &sd::VideoUpscalerConfig::upscale_factor, 1, INT32_MAX);
-
- video_upscale.frames("frames", &sd::VideoUpscaleConfig::frames, &sd::VideoUpscaleConfig::frame_count);
- video_upscale.integer("window_size", &sd::VideoUpscaleConfig::window_size, 1, INT32_MAX);
- video_upscale.integer("temporal_chunk_size", &sd::VideoUpscaleConfig::temporal_chunk_size, 1, INT32_MAX);
-
- adetail.string("adetail.prompt", &sd::ADetailConfig::prompt);
- adetail.string("adetail.negative_prompt", &sd::ADetailConfig::negative_prompt);
- adetail.string("adetail.args", &sd::ADetailConfig::extra_ad_args);
+static bool dgl_only_parameter(const char* key) {
+ const std::string name = key;
+ if (name == "high_noise_sample.steps")
+ return false;
+ for (const auto* prefix : {"photomaker.", "pulid.", "high_noise_sample.", "sample.guidance.slg.", "adetail."})
+ if (name.rfind(prefix, 0) == 0)
+ return true;
+ static const std::set<std::string> names{"ip_adapter_path", "ip_adapter_image", "ip_adapter_strength", "motion_module_path", "photo_maker_path", "pulid_weights_path", "image_preprocess.rules", "sample.shifted_timestep"};
+ return names.count(name) != 0;
}
static std::vector<ParamSpec> make_specs() {
@@ -565,6 +507,8 @@ static const ParamSpec* find_spec(const char* key) {
static sd_status_t check_key(sd_opt_t* params, const char* key, ParamType type, const ParamSpec** out) {
if (!params || !key)
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required");
+ if (dgl_only_parameter(key))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, key, "this parameter requires an explicit DGL workflow (-M dgl --script FILE)");
*out = find_spec(key);
if (!*out)
return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter");
@@ -589,6 +533,8 @@ static sd_status_t store_value(sd_opt_t* params, const ParamSpec& spec, ParamVal
if (spec.valid && !spec.valid(value))
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, spec.key, spec.invalid_reason);
params->values.insert_or_assign(spec.key, std::move(value));
+ if (strcmp(spec.key, "cache.fn_compute_blocks") == 0 || strcmp(spec.key, "cache.bn_compute_blocks") == 0)
+ LOG_WARN("parameter '%s' is ignored: condition-level caching does not support front/back block counts", spec.key);
return SD_STATUS_OK;
}
@@ -615,22 +561,6 @@ sd_upscale_opt_t* sd_upscale_opt_create(void) {
return create_opt(sd::api::Upscale);
}
-sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void) {
- return create_opt(sd::api::VidUpscaler);
-}
-
-sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void) {
- return create_opt(sd::api::VidUpscale);
-}
-
-sd_adetailer_opt_t* sd_adetailer_opt_create(void) {
- return create_opt(sd::api::ADetailer);
-}
-
-sd_adetail_opt_t* sd_adetail_opt_create(void) {
- return create_opt(sd::api::ADetail);
-}
-
void sd_opt_free(sd_opt_t* params) {
delete params;
}
@@ -820,6 +750,8 @@ sd_status_t sd_opt_unset(sd_opt_t* params, const char* key) {
return sd::api::call([&] {
if (!params || !key)
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required");
+ if (dgl_only_parameter(key))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, key, "this parameter requires an explicit DGL workflow (-M dgl --script FILE)");
const auto* spec = find_spec(key);
if (!spec)
return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter");
@@ -834,6 +766,96 @@ bool sd::api::is_set(const sd_opt_t* params, const char* key) {
return params && params->values.count(key) != 0;
}
+std::shared_ptr<sd_opt_t> sd::api::copy_options(const sd_opt_t* params) {
+ if (!params)
+ throw sd::dgl::Error(SD_STATUS_INVALID_ARGUMENT, "params is required");
+ auto copy = std::make_shared<sd_opt_t>(*params);
+ for (auto& [key, value] : copy->values) {
+ if (auto* loras = std::get_if<LoraValues>(&value)) {
+ for (size_t i = 0; i < loras->items.size(); ++i)
+ loras->items[i].path = loras->paths[i].c_str();
+ } else if (auto* embeddings = std::get_if<EmbeddingValues>(&value)) {
+ for (size_t i = 0; i < embeddings->items.size(); ++i) {
+ embeddings->items[i].name = embeddings->strings[i * 2].c_str();
+ embeddings->items[i].path = embeddings->strings[i * 2 + 1].c_str();
+ }
+ } else if (auto* videos = std::get_if<VideoValues>(&value)) {
+ for (size_t i = 0; i < videos->items.size(); ++i)
+ videos->items[i].frames = videos->frames[i].data();
+ }
+ }
+ return copy;
+}
+
+std::vector<std::string> sd::api::explicit_keys(const sd_opt_t* params) {
+ std::vector<std::string> keys;
+ if (params)
+ for (const auto& [key, value] : params->values)
+ keys.push_back(key);
+ return keys;
+}
+
+const sd_tensor_t* sd::api::bound_tensor(const sd_opt_t* params, const char* key) {
+ if (!params)
+ return nullptr;
+ const auto found = params->values.find(key);
+ if (found == params->values.end())
+ return nullptr;
+ const auto* value = std::get_if<TensorValue>(&found->second);
+ return value ? value->tensor.get() : nullptr;
+}
+
+std::vector<const sd_tensor_t*> sd::api::bound_tensors(const sd_opt_t* params, const char* key) {
+ std::vector<const sd_tensor_t*> tensors;
+ if (!params)
+ return tensors;
+ const auto found = params->values.find(key);
+ if (found == params->values.end())
+ return tensors;
+ if (const auto* values = std::get_if<ImageValues>(&found->second))
+ for (const auto& tensor : values->tensors)
+ tensors.push_back(tensor.get());
+ return tensors;
+}
+
+std::vector<sd_audio_t> sd::api::bound_audios(const sd_opt_t* params, const char* key) {
+ std::vector<sd_audio_t> audios;
+ if (params && params->values.count(key))
+ if (const auto* values = std::get_if<AudioValues>(¶ms->values.at(key)))
+ for (size_t i = 0; i < values->items.size(); ++i)
+ audios.push_back({uint32_t(values->items[i].sample_rate), values->tensors[i].get()});
+ return audios;
+}
+
+std::vector<sd_video_t> sd::api::bound_videos(const sd_opt_t* params, const char* key) {
+ std::vector<sd_video_t> videos;
+ if (params && params->values.count(key))
+ if (const auto* values = std::get_if<VideoValues>(¶ms->values.at(key))) {
+ size_t tensor = 0;
+ for (const auto& item : values->items) {
+ sd_video_t video{values->tensors[tensor++].get(), item.fps, {}};
+ if (item.audio.data)
+ video.audio = {uint32_t(item.audio.sample_rate), values->tensors[tensor++].get()};
+ videos.push_back(video);
+ }
+ }
+ return videos;
+}
+
+nlohmann::json sd::api::scalar_values(const sd_opt_t* params) {
+ auto values = nlohmann::json::object();
+ if (params)
+ for (const auto& [key, value] : params->values)
+ std::visit([&](const auto& item) {
+ using T = std::decay_t<decltype(item)>;
+ if constexpr (std::is_arithmetic_v<T> || std::is_same_v<T, std::string> ||
+ std::is_same_v<T, std::vector<float>> || std::is_same_v<T, std::vector<int>>)
+ values[key] = item;
+ },
+ value);
+ return values;
+}
+
bool sd::api::has_prefix(const sd_opt_t* params, const char* prefix) {
if (!params)
return false;
@@ -847,51 +869,18 @@ sd_status_t sd::api::resolve(const sd_opt_t* params, Operation operation, OptCon
if (params->operation != operation)
return error(SD_STATUS_UNSUPPORTED, nullptr, "option object has the wrong purpose for this operation");
config = OptConfig{};
- if (operation == ADetail)
- config.image.strength = 0.4f;
for (const auto& entry : params->values) {
const auto* spec = find_spec(entry.first.c_str());
- if (!(spec->operations & operation))
- return error(SD_STATUS_UNSUPPORTED, spec->key, "parameter does not apply to this operation");
+ if (!spec || !(spec->operations & operation))
+ return error(SD_STATUS_UNSUPPORTED, entry.first.c_str(), "parameter does not apply to this operation");
spec->apply(entry.second, config);
}
- const auto& image = config.image;
- const auto& video = config.video;
- const bool sampling = operation == Image || operation == Video || operation == ADetail;
- const auto& cache = operation == Video ? video.cache : image.cache;
- if (sampling && cache.mode != SD_CACHE_DISABLED) {
- if (cache.start_percent >= cache.end_percent)
- return error(SD_STATUS_INVALID_ARGUMENT, "cache", "start_percent must precede end_percent");
- }
- if (sampling) {
- const auto& sample = operation == Video ? video.sample_params : image.sample_params;
- if (sample.guidance.slg.layer_start > sample.guidance.slg.layer_end)
- return error(SD_STATUS_INVALID_ARGUMENT, "sample.guidance.slg", "layer_start must not exceed layer_end");
- const auto& hires = operation == Video ? video.hires : image.hires;
- if (hires.enabled && hires.upscaler != SD_HIRES_UPSCALER_NONE &&
- hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0)
- return error(SD_STATUS_INVALID_ARGUMENT, "hires", "enabled hires requires a positive scale or target size");
- }
- if (operation == Video && video.high_noise_sample_params.guidance.slg.layer_start > video.high_noise_sample_params.guidance.slg.layer_end)
- return error(SD_STATUS_INVALID_ARGUMENT, "high_noise_sample.guidance.slg", "layer_start must not exceed layer_end");
- if ((operation == Upscaler || operation == ADetailer) && (!config.context.model_path || !*config.context.model_path))
+ const auto& cache = operation == Video ? config.video.cache : config.image.cache;
+ if ((operation == Image || operation == Video) && cache.mode != SD_CACHE_DISABLED && cache.start_percent >= cache.end_percent)
+ return error(SD_STATUS_INVALID_ARGUMENT, "cache", "start_percent must precede end_percent");
+ if (operation == Upscaler && (!config.context.model_path || !*config.context.model_path))
return error(SD_STATUS_INVALID_ARGUMENT, "model_path", "model path is required");
- if (operation == VidUpscaler) {
- const auto& load = config.post.video_upscaler;
- if (!load.diffusion_model_path || !*load.diffusion_model_path || !load.vae_path || !*load.vae_path || load.upscale_factor <= 0)
- return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "diffusion_model_path, vae_path and upscale_factor are required");
- }
- if ((operation == Upscale || operation == ADetail) && !config.post.image.data)
- return error(SD_STATUS_INVALID_ARGUMENT, "image", "input image is required");
- if (operation == Upscale && config.post.image.channel != 3)
+ if (operation == Upscale && (!config.post.image.data || config.post.image.channel != 3))
return error(SD_STATUS_INVALID_ARGUMENT, "image", "upscaling requires an RGB image");
- if (operation == VidUpscale) {
- const auto& input = config.post.video_upscale;
- if (!input.frames || input.frame_count <= 0)
- return error(SD_STATUS_INVALID_ARGUMENT, "frames", "video upscaling requires nonempty frames");
- for (int i = 0; i < input.frame_count; ++i)
- if (input.frames[i].channel != 3 || input.frames[i].width != input.frames[0].width || input.frames[i].height != input.frames[0].height)
- return error(SD_STATUS_INVALID_ARGUMENT, "frames", "frames must be RGB and have equal dimensions");
- }
return SD_STATUS_OK;
}
diff --git a/src/api/options.h b/src/api/options.h
index 6517829..aab4f7b 100644
--- a/src/api/options.h
+++ b/src/api/options.h
@@ -3,23 +3,31 @@
#include "api.h"
#include "config.h"
+#include "json.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
namespace sd::api {
enum Operation {
- Context = 1,
- Image = 2,
- Video = 4,
- Upscaler = 8,
- Upscale = 16,
- VidUpscaler = 32,
- VidUpscale = 64,
- ADetailer = 128,
- ADetail = 256
+ Context = 1,
+ Image = 2,
+ Video = 4,
+ Upscaler = 8,
+ Upscale = 16
};
sd_status_t resolve(const sd_opt_t* params, Operation operation, OptConfig& config);
bool is_set(const sd_opt_t* params, const char* key);
bool has_prefix(const sd_opt_t* params, const char* prefix);
+ std::shared_ptr<sd_opt_t> copy_options(const sd_opt_t* params);
+ std::vector<std::string> explicit_keys(const sd_opt_t* params);
+ const sd_tensor_t* bound_tensor(const sd_opt_t* params, const char* key);
+ std::vector<const sd_tensor_t*> bound_tensors(const sd_opt_t* params, const char* key);
+ std::vector<sd_audio_t> bound_audios(const sd_opt_t* params, const char* key);
+ std::vector<sd_video_t> bound_videos(const sd_opt_t* params, const char* key);
+ nlohmann::json scalar_values(const sd_opt_t* params);
}
#endif // SD_OPTIONS_H
diff --git a/src/api/program_result.cpp b/src/api/program_result.cpp
new file mode 100644
index 0000000..6d314c9
--- /dev/null
+++ b/src/api/program_result.cpp
@@ -0,0 +1,68 @@
+#include "api/result.h"
+
+#include <cstring>
+
+#include "dgl/diagnostic.h"
+#include "dgl/value.h"
+
+namespace sd::api {
+ static const dgl::Value& field(const dgl::Value& value, const char* name) {
+ if (value.type().kind != dgl::Kind::Dict || !value.fields.count(name))
+ throw dgl::Error(SD_STATUS_TYPE_MISMATCH, "generation result is missing '" + std::string(name) + "'");
+ return value.fields.at(name);
+ }
+
+ static void append_frames(sd_result_t& result, const sd_tensor_t& frames, bool video) {
+ const auto& shape = frames.shape();
+ if (frames.type != SD_TENSOR_U8 || (shape.size() != 3 && shape.size() != 4) ||
+ shape[0] < 1 || shape[0] > 4 || shape[1] <= 0 || shape[2] <= 0 || (shape.size() == 4 && shape[3] <= 0))
+ throw dgl::Error(SD_STATUS_TYPE_MISMATCH, "generation pixels must be a packed U8 [C,W,H] image or [C,W,H,T] frames");
+ const auto count = shape.size() == 4 ? shape[3] : 1;
+ if (!video && count != 1)
+ throw dgl::Error(SD_STATUS_TYPE_MISMATCH, "image program returned multiple frames");
+ const auto bytes = frames.bytes / count;
+ const auto* pixels = static_cast<const uint8_t*>(frames.data());
+ for (int64_t i = 0; i < count; ++i) {
+ auto frame = allocate_tensor(SD_TENSOR_U8, {shape[0], shape[1], shape[2]});
+ std::memcpy(frame.get()->mutable_data(), pixels + i * bytes, bytes);
+ result.images.push_back(std::move(frame));
+ }
+ }
+
+ void append_program_image(sd_result_t& result, const dgl::Value& value, const std::string& source) {
+ const auto& image = field(value, "image");
+ const auto first = result.images.size();
+ if (image.type() == dgl::Type{dgl::Kind::List, dgl::Kind::Tensor}) {
+ for (const auto& item : image.items)
+ append_frames(result, *item.as<sd_tensor_t>(), false);
+ } else {
+ append_frames(result, *image.as<sd_tensor_t>(), false);
+ }
+ result.kind = SD_RESULT_IMAGE;
+ for (size_t i = first; i < result.images.size(); ++i)
+ result.generation_infos.push_back(source);
+ }
+
+ void set_program_video(sd_result_t& result, const dgl::Value& value, const std::string& source) {
+ const auto& video = field(value, "video");
+ append_frames(result, *field(video, "frames").as<sd_tensor_t>(), true);
+ const auto numerator = field(video, "fps_num").scalar.get<int64_t>();
+ const auto denominator = field(video, "fps_den").scalar.get<int64_t>();
+ if (numerator <= 0 || denominator <= 0 || numerator % denominator || numerator / denominator > INT32_MAX)
+ throw dgl::Error(SD_STATUS_OUT_OF_RANGE, "C video result requires a positive integral frame rate");
+ result.kind = SD_RESULT_VIDEO;
+ result.fps = int(numerator / denominator);
+ result.generation_infos = {source};
+ const auto audio = value.fields.find("audio");
+ if (audio == value.fields.end() || !audio->second.fields.count("samples"))
+ return;
+ const auto samples = field(audio->second, "samples").as<sd_tensor_t>();
+ if (!samples->bytes)
+ return;
+ const auto rate = field(audio->second, "sample_rate").scalar.get<int64_t>();
+ if (samples->type != SD_TENSOR_F32 || samples->shape().size() != 2 || samples->shape()[0] <= 0 || samples->shape()[0] > 32 || rate < 1000 || rate > 384000)
+ throw dgl::Error(SD_STATUS_TYPE_MISMATCH, "generation audio must contain F32 [channels,samples] and a valid sample rate");
+ result.audio_tensor = TensorRef(samples.get());
+ result.audio = {uint32_t(rate), result.audio_tensor.get()};
+ }
+}
diff --git a/src/api/result.cpp b/src/api/result.cpp
index 388f948..eee5ea0 100644
--- a/src/api/result.cpp
+++ b/src/api/result.cpp
@@ -5,150 +5,6 @@
#include <cstring>
#include <utility>
-namespace sd {
- using nlohmann::json;
-
- char* copy_generation_info(const json& info) {
- const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace);
- char* result = static_cast<char*>(malloc(text.size() + 1));
- if (result)
- memcpy(result, text.c_str(), text.size() + 1);
- return result;
- }
-
- bool make_image_generation_result(sd::Image* images, int count, const std::vector<json>& infos, sd::ImageGenerationResult** result_out) {
- if (!result_out || !images || count <= 0 || infos.size() != static_cast<size_t>(count)) {
- sd::free_images(images, count);
- return false;
- }
- auto* result = static_cast<sd::ImageGenerationResult*>(calloc(1, sizeof(sd::ImageGenerationResult)));
- if (!result) {
- sd::free_images(images, count);
- return false;
- }
- result->items = static_cast<sd::ImageResult*>(calloc(count, sizeof(sd::ImageResult)));
- if (!result->items) {
- free(result);
- sd::free_images(images, count);
- return false;
- }
- result->count = count;
- try {
- for (int i = 0; i < count; ++i) {
- result->items[i].image = images[i];
- images[i] = {};
- result->items[i].generation_info = copy_generation_info(infos[i]);
- if (!result->items[i].generation_info) {
- sd::free_images(images, count);
- sd::free_image_generation_result(result);
- return false;
- }
- }
- } catch (...) {
- sd::free_images(images, count);
- sd::free_image_generation_result(result);
- return false;
- }
- free(images);
- *result_out = result;
- return true;
- }
-}
-
-void sd::free_image_generation_result(sd::ImageGenerationResult* result) {
- if (!result)
- return;
- for (int i = 0; i < result->count; ++i) {
- free(result->items[i].image.data);
- free(result->items[i].generation_info);
- }
- free(result->items);
- free(result);
-}
-
-void sd::free_video_generation_result(sd::VideoGenerationResult* result) {
- if (!result)
- return;
- sd::free_images(result->frames, result->frame_count);
- sd::free_audio(result->audio);
- free(result->generation_info);
- free(result);
-}
-
-static sd::api::TensorRef consume_image(sd::Image& image) {
- std::vector<int64_t> shape{image.channel, image.width, image.height};
- auto* data = image.data;
- image.data = nullptr;
- return sd::api::own_tensor(SD_TENSOR_U8, std::move(shape), data);
-}
-
-bool sd::make_image_result(ImageGenerationResult* images, sd_result_t** result_out) {
- if (result_out)
- *result_out = nullptr;
- std::unique_ptr<ImageGenerationResult, decltype(&free_image_generation_result)> owner(images, free_image_generation_result);
- if (!result_out || !images)
- return false;
- auto result = std::make_unique<sd_result_t>();
- result->kind = SD_RESULT_IMAGE;
- result->images.reserve(images->count);
- result->generation_infos.reserve(images->count);
- for (int i = 0; i < images->count; ++i) {
- result->images.push_back(consume_image(images->items[i].image));
- result->generation_infos.emplace_back(images->items[i].generation_info ? images->items[i].generation_info : "");
- }
- *result_out = result.release();
- return true;
-}
-
-bool sd::make_image_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd_result_t** result_out) {
- if (result_out)
- *result_out = nullptr;
- ImageGenerationResult* raw = nullptr;
- if (!make_image_generation_result(images, count, infos, &raw))
- return false;
- return make_image_result(raw, result_out);
-}
-
-bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_out) {
- if (result_out)
- *result_out = nullptr;
- std::unique_ptr<VideoGenerationResult, decltype(&free_video_generation_result)> owner(video, free_video_generation_result);
- if (!result_out || !video)
- return false;
- auto result = std::make_unique<sd_result_t>();
- result->kind = SD_RESULT_VIDEO;
- result->fps = video->fps;
- result->images.reserve(video->frame_count);
- for (int i = 0; i < video->frame_count; ++i)
- result->images.push_back(consume_image(video->frames[i]));
- result->generation_infos.emplace_back(video->generation_info ? video->generation_info : "");
- if (video->audio) {
- auto& audio = *video->audio;
- std::vector<int64_t> shape{audio.channels, static_cast<int64_t>(audio.sample_count)};
- auto* data = audio.data;
- audio.data = nullptr;
- result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, std::move(shape), data);
- result->audio = {audio.sample_rate, result->audio_tensor.get()};
- }
- *result_out = result.release();
- return true;
-}
-
-void sd::free_images(sd::Image* result_images, int num_images) {
- if (result_images == nullptr) {
- return;
- }
-
- for (int i = 0; i < num_images; ++i) {
- if (result_images[i].data != nullptr) {
- free(result_images[i].data);
- result_images[i].data = nullptr;
- }
- }
-
- free(result_images);
-}
-
void sd::free_audio(sd::Audio* audio) {
if (audio == nullptr) {
return;
@@ -222,10 +78,10 @@ const sd_video_t* sd_result_get_video(const sd_result_t* result) {
if (result->images.empty())
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video has no frames");
const auto& first = *result->images.front().get();
- auto shape = first.shape;
+ auto shape = first.shape();
shape.push_back(static_cast<int64_t>(result->images.size()));
for (const auto& frame : result->images)
- if (frame->type != first.type || frame->shape != first.shape)
+ if (frame->type != first.type || frame->shape() != first.shape())
return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video frames must have identical shapes and types");
auto packed = sd::api::allocate_tensor(first.type, std::move(shape));
auto* destination = static_cast<uint8_t*>(packed.get()->mutable_data());
diff --git a/src/api/result.h b/src/api/result.h
index 452055b..1fe49d3 100644
--- a/src/api/result.h
+++ b/src/api/result.h
@@ -7,39 +7,15 @@
#include <vector>
#include "tensor.h"
-#include "json.hpp"
#include "stable-diffusion.h"
-namespace sd {
- struct ImageResult {
- sd::Image image;
- // Owned UTF-8 JSON using the sdcpp.generation/v1 schema.
- char* generation_info;
- };
-
- struct ImageGenerationResult {
- sd::ImageResult* items;
- int count;
- };
-
- struct VideoGenerationResult {
- sd::Image* frames;
- int frame_count;
- sd::Audio* audio;
- int fps;
- char* generation_info;
- };
-
- void free_image_generation_result(ImageGenerationResult* result);
- void free_video_generation_result(VideoGenerationResult* result);
- void free_images(sd::Image* images, int count);
+namespace sd::dgl {
+ class Value;
+}
- char* copy_generation_info(const nlohmann::json& info);
- // Consumes images even on failure.
- bool make_image_generation_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd::ImageGenerationResult** result_out);
- bool make_image_result(ImageGenerationResult* images, sd_result_t** result_out);
- bool make_image_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd_result_t** result_out);
- bool make_video_result(VideoGenerationResult* video, sd_result_t** result_out);
+namespace sd::api {
+ void append_program_image(sd_result_t& result, const dgl::Value& value, const std::string& source);
+ void set_program_video(sd_result_t& result, const dgl::Value& value, const std::string& source);
}
struct sd_result {
diff --git a/src/api/templates/README.md b/src/api/templates/README.md
new file mode 100644
index 0000000..82f7b4f
--- /dev/null
+++ b/src/api/templates/README.md
@@ -0,0 +1,55 @@
+# C API generation templates
+
+The generation C API selects an entry template for the request, renders a
+complete DGL program and runs it with the input tensors supplied by the host.
+`generation_info` contains the executed source. CMake embeds the templates
+during configuration.
+
+| Template | Flow |
+| --- | --- |
+| [load_components.dgl](load_components.dgl) | Prepare and validate configured model components without generation |
+| [image.dgl](image.dgl) | Text-to-image and ordinary image-to-image |
+| [image_inpaint.dgl](image_inpaint.dgl) | Masked sampling and native inpaint conditioning |
+| [image_edit.dgl](image_edit.dgl) | Native reference-image editing |
+| [image_pixel.dgl](image_pixel.dgl) | Pixel-space diffusion, with optional VAE-encoded references |
+| `image_h3.dgl` | MiniMax-H3 Image and P-H3 latent packing |
+| [image_layered.dgl](image_layered.dgl) | Qwen layered generation and per-layer decoding |
+| [image_sefi.dgl](image_sefi.dgl) | SeFi dual schedules |
+| [video_wan.dgl](video_wan.dgl) | Wan/Hunyuan video, VACE, S2V and dual experts |
+| [video_ltx.dgl](video_ltx.dgl) | LTX video/audio generation and keyframes |
+| `video_h3.dgl` | MiniMax-H3 video/audio generation and references |
+| [upscale.dgl](upscale.dgl) | Standalone image upscaling, including repeats |
+
+## Editing a flow
+
+C++ resolves model families, request defaults, dimensions and host tensor
+bindings. The entry template expresses the resulting workflow as DGL calls.
+Use a template condition for an optional stage and an include for a helper
+shared by several flows. Put the minimum DGL version at the top of each entry.
+When a helper needs a newer version, update the entries that use it.
+See the [DGL version rules](../../../docs/dgl.md#dgl-versions).
+
+For example, the image entry loads the components with these lines.
+
+```text
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+```
+
+Here `model_path` and `text_path` are DGL variables in the rendered program.
+C++ supplies `model_loader` as a function name and `model_options` as an
+already formatted argument list, including its leading comma when nonempty.
+
+The value C++ supplies determines how to insert it into the script.
+
+| Template expression | Input | Rendered DGL |
+| --- | --- | --- |
+| `{{ value \| literal }}` | A raw value, such as a prompt or file path | A DGL literal with strings quoted and escaped |
+| `{{ options \| args }}` | An object such as `{"family": "sd1"}` | `, family = "sd1"` (empty text for an empty object) |
+| `{{model_options}}` | A trusted, preformatted argument list | The supplied text verbatim |
+
+Conditions use boolean fields, for example `{% if flags.has_init %}`, with
+optional `elif`, `else` and `not`. Close them with `{% endif %}`. An include
+such as `{% include "image_helpers" %}` uses the embedded template name.
+Substitutions are evaluated once, so template-like text in a prompt stays text.
diff --git a/src/api/templates/image.dgl b/src/api/templates/image.dgl
new file mode 100644
index 0000000..75e0745
--- /dev/null
+++ b/src/api/templates/image.dgl
@@ -0,0 +1,90 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+ image_base = {{from_vae}}(vae, vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng{{vae_run_options}}))
+{% else %}
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+{% endif %}
+ base = image_base
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+{% if flags.native_control %}
+ control_latent = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng{{vae_run_options}}))
+{% else %}
+ control_model = controlnet::load(control_path{{control_options}})
+{% if flags.z_control %}
+ control_encoded = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng, posterior = "mode"{{vae_run_options}}))
+ control_hint = tensor::pad(control_encoded, [0, 0, 0, 0, 0, 33 - tensor::shape(control_encoded)[2]])
+{% endif %}
+{% endif %}
+{% endif %}
+{% if flags.separate_negative_model %}
+ negative_model = load_dit(negative_model_path{{model_options}})
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.minit2i %}
+ negative_text = zero_out(positive_text, fields = ["vector"])
+{% elif flags.ming %}
+ negative_text = zero_out(positive_text)
+{% elif flags.ideogram %}
+ negative_text = {vector = tensor::vector([1])}
+{% else %}
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% endif %}
+ positive = positive_text
+{% if flags.sdxl_zero %}
+ negative = sdxl_zero_text(negative_text)
+{% else %}
+ negative = negative_text
+{% endif %}
+{% if flags.control_image_guidance %}
+ image_unconditional_concat = tensor::zeros_like(base)
+{% endif %}
+{% include "image_predictors" %}
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+{% else %}
+ sigmas = full_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+{% if flags.hidream_o1 %}
+ start = add_noise(guided, base, tensor::mul(tensor::randn_like(base, rng = rng), eta), sigma = sigmas[0])
+{% else %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+{% endif %}
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ image = vae_decode(vae, {{to_vae}}(vae, {{result_latent}}){{vae_run_options}})
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_edit.dgl b/src/api/templates/image_edit.dgl
new file mode 100644
index 0000000..6cd8c83
--- /dev/null
+++ b/src/api/templates/image_edit.dgl
@@ -0,0 +1,105 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+{% if flags.raw_reference %}
+ image_base = vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng, posterior = "mode"{{vae_run_options}})
+{% else %}
+ image_base = {{from_vae}}(vae, vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng{{vae_run_options}}))
+{% endif %}
+{% else %}
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+{% endif %}
+ base = image_base
+{{references}}
+{% if flags.unet_edit %}
+{% if flags.references %}
+ edit_latent = tensor::interpolate(reference_latent_0, size = tensor::shape(base)[:2])
+{% else %}
+ edit_latent = tensor::zeros_like(base)
+{% endif %}
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::zeros_like(edit_latent)
+{% endif %}
+{% endif %}
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+{% if flags.native_control %}
+ control_latent = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng{{vae_run_options}}))
+{% else %}
+ control_model = controlnet::load(control_path{{control_options}})
+{% if flags.z_control %}
+ control_encoded = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng, posterior = "mode"{{vae_run_options}}))
+ control_hint = tensor::pad(control_encoded, [0, 0, 0, 0, 0, 33 - tensor::shape(control_encoded)[2]])
+{% endif %}
+{% endif %}
+{% endif %}
+{% if flags.separate_negative_model %}
+ negative_model = load_dit(negative_model_path{{model_options}})
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.minit2i %}
+ negative_text = zero_out(positive_text, fields = ["vector"])
+{% elif flags.ming %}
+ negative_text = zero_out(positive_text)
+{% elif flags.ideogram %}
+ negative_text = {vector = tensor::vector([1])}
+{% else %}
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% endif %}
+ positive = positive_text
+{% if flags.sdxl_zero %}
+ negative = sdxl_zero_text(negative_text)
+{% else %}
+ negative = negative_text
+{% endif %}
+{% if flags.control_image_guidance %}
+ image_unconditional_concat = tensor::zeros_like(base)
+{% endif %}
+{% include "image_predictors" %}
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+{% else %}
+ sigmas = full_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+{% if flags.hidream_o1 %}
+ start = add_noise(guided, base, tensor::mul(tensor::randn_like(base, rng = rng), eta), sigma = sigmas[0])
+{% else %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+{% endif %}
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ image = vae_decode(vae, {{to_vae}}(vae, {{result_latent}}){{vae_run_options}})
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_h3.dgl b/src/api/templates/image_h3.dgl
new file mode 100644
index 0000000..55a47cc
--- /dev/null
+++ b/src/api/templates/image_h3.dgl
@@ -0,0 +1,89 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+ image_encoded = {{from_vae}}(vae, vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng{{vae_run_options}}))
+ image_base = tensor::reshape(image_encoded, latent_shape(vae, width = width, height = height))
+{% else %}
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+{% endif %}
+{% if flags.ph3 %}
+ image_shape = tensor::shape(image_base)
+ image_video = tensor::unsqueeze(image_base, axis = 2)
+ audio_channels = math::ceil_div(128, image_shape[0] * image_shape[1])
+ base = tensor::concat([image_video, tensor::zeros([image_shape[0], image_shape[1], 1, audio_channels, 1])], axis = 3)
+{% else %}
+ base = tensor::unsqueeze(image_base, axis = 2)
+{% endif %}
+{{references}}
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+ control_model = controlnet::load(control_path{{control_options}})
+{% endif %}
+{% if flags.mask_stage %}
+{% if flags.has_mask %}
+ mask_pixels = tensor::threshold(image::to_f32(fit_canvas(tensor::grayscale(input_mask, axis = 0), width, height, mode = "nearest_exact")), threshold = 0.5)
+{% else %}
+ mask_pixels = tensor::full([width, height, 1, 1], 1)
+{% endif %}
+ base_shape = tensor::shape(base)
+ mask_scale = [math::floor(width / base_shape[0]), math::floor(height / base_shape[1])]
+ mask_latent = tensor::max_pool2d(mask_pixels, kernel = mask_scale, stride = mask_scale)
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+ positive = positive_text
+ negative = negative_text
+{% if flags.controlnet %}
+ controlled_positive = controlnet::apply(positive, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+ controlled_negative = controlnet::apply(negative, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+{% endif %}
+ pos = predictor(model, {{positive_condition}}{{predictor_options}})
+ neg = predictor({{negative_model}}, {{negative_condition}}{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+{% else %}
+ sigmas = full_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+{% if flags.ph3 %}
+ image = vae_decode(vae, dit_to_vae_latents(vae, {{result_latent}}[:, :, :, :24]){{vae_run_options}})
+{% else %}
+ image = vae_decode(vae, dit_to_vae_latents(vae, {{result_latent}}){{vae_run_options}})
+{% endif %}
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_helpers.dgl b/src/api/templates/image_helpers.dgl
new file mode 100644
index 0000000..bb16609
--- /dev/null
+++ b/src/api/templates/image_helpers.dgl
@@ -0,0 +1,55 @@
+{% if flags.sdxl_zero %}
+fn sdxl_zero_text(cond) {
+ zeroed = zero_out(cond, fields = ["crossattn"])
+ pooled_zeros = tensor::zeros_like(cond.vector[:1280])
+ adm = tensor::replace_slice(cond.vector, pooled_zeros, axis = 0, start = 0)
+ return {crossattn = zeroed.crossattn, vector = adm}
+}
+
+{% endif %}
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn slice_sigmas_by_strength(sigmas, strength) {
+ steps = tensor::numel(sigmas) - 1
+ retained = math::min(math::floor(steps * strength) + 1, steps)
+ return sigmas[steps - retained:]
+}
+
+{% if flags.layered %}
+fn decode_layer(index, args) {
+ latent = args.latent[:, :, index]
+ return vae_decode(args.vae, dit_to_vae_latents(args.vae, latent){{vae_run_options}})
+}
+{% if flags.hires %}
+{% if flags.hires_pixels %}
+fn upscale_layer(index, args) {
+ latent = args.latent[:, :, index]
+ image = vae_decode(args.vae, dit_to_vae_latents(args.vae, latent), dtype = "f32"{{vae_run_options}})
+{% if flags.hires_model %}
+ upscaled = upscale(args.upscaler, image::to_u8(image))
+ pixels = image::to_f32(tensor::interpolate(upscaled, size = args.size, axes = [1, 2], mode = args.mode))
+{% else %}
+ pixels = tensor::interpolate(image, size = args.size, mode = args.mode)
+{% endif %}
+ encoded = vae_to_dit_latents(args.vae, vae_encode(args.vae, pixels, rng = args.rng{{vae_run_options}}))
+ return tensor::unsqueeze(encoded, axis = 2)
+}
+{% endif %}
+{% endif %}
+{% endif %}
+{% if flags.h3_reference_noise %}
+fn h3_image_reference_noise(latent, seed) {
+ shape = tensor::shape(latent)
+ patches = tensor::zeros([2, 2, shape[-2], math::floor(shape[0] / 2), math::floor(shape[1] / 2)])
+ rng = make_rng("cpu", seed = seed)
+ noise = tensor::reshape(tensor::permute(tensor::randn_like(patches, rng = rng), [0, 3, 1, 4, 2]), shape)
+ return tensor::lerp(latent, noise, 0.001)
+}
+{% endif %}
diff --git a/src/api/templates/image_hires.dgl b/src/api/templates/image_hires.dgl
new file mode 100644
index 0000000..14c09a5
--- /dev/null
+++ b/src/api/templates/image_hires.dgl
@@ -0,0 +1,125 @@
+ hires_rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ hires_sampler_rng = hires_rng
+{% else %}
+ hires_sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+{% if flags.ph3 %}
+ hires_source = latent[:, :, :, :24]
+{% else %}
+ hires_source = latent
+{% endif %}
+{% if flags.hires_pixels %}
+{% if flags.layered %}
+{% if flags.hires_model %}
+ hires_upscaler = load_upscaler(hires_model_path{{hires_model_options}})
+ hires_layers = map(tensor::to_list(tensor::arange(0, layers)), process = upscale_layer, args = {vae = vae, latent = latent, rng = hires_rng, upscaler = hires_upscaler, size = [hires_target_width, hires_target_height], mode = hires_mode})
+{% else %}
+ hires_layers = map(tensor::to_list(tensor::arange(0, layers)), process = upscale_layer, args = {vae = vae, latent = latent, rng = hires_rng, size = [hires_target_width, hires_target_height], mode = hires_mode})
+{% endif %}
+ hires_image_base = tensor::concat(hires_layers, axis = 2)
+{% else %}
+{% if flags.hires_pixel_model %}
+ hires_pixels = tensor::clamp(tensor::affine(hires_source, scale = 0.5, offset = 0.5), 0, 1)
+{% elif flags.hires_sefi %}
+ hires_pixels = vae_decode(vae, {{to_vae}}(vae, hires_source[:, :, 16:144]), dtype = "f32"{{vae_run_options}})
+{% else %}
+ hires_pixels = vae_decode(vae, {{to_vae}}(vae, hires_source), dtype = "f32"{{vae_run_options}})
+{% endif %}
+{% if flags.hires_model %}
+ hires_upscaler = load_upscaler(hires_model_path{{hires_model_options}})
+ hires_upscaled = upscale(hires_upscaler, image::to_u8(hires_pixels))
+ hires_resized = image::to_f32(tensor::interpolate(hires_upscaled, size = [hires_target_width, hires_target_height], axes = [1, 2], mode = hires_mode), channels = {{channels}})
+{% else %}
+ hires_resized = tensor::interpolate(hires_pixels, size = [hires_target_width, hires_target_height], mode = hires_mode)
+{% endif %}
+{% if flags.hires_pixel_model %}
+ hires_image_base = tensor::affine(hires_resized, scale = 2, offset = -1)
+{% else %}
+ hires_encoded = {{from_vae}}(vae, vae_encode(vae, hires_resized, rng = hires_rng{{vae_run_options}}))
+{% if flags.hires_h3_image %}
+ hires_image_base = tensor::reshape(hires_encoded, [math::floor(hires_target_width / hires_vae_scale), math::floor(hires_target_height / hires_vae_scale), 1, 24, 1])
+{% elif flags.hires_sefi %}
+ hires_image_base = tensor::concat([tensor::interpolate(latent[:, :, :16], size = [math::floor(hires_target_width / hires_vae_scale), math::floor(hires_target_height / hires_vae_scale)]), hires_encoded], axis = 2)
+{% else %}
+ hires_image_base = hires_encoded
+{% endif %}
+{% endif %}
+{% endif %}
+{% else %}
+ hires_image_base = tensor::interpolate(hires_source, size = [math::floor(hires_target_width / hires_vae_scale), math::floor(hires_target_height / hires_vae_scale)], mode = hires_mode, antialias = hires_antialias)
+{% endif %}
+{% if flags.ph3 %}
+ hires_shape = tensor::shape(hires_image_base)
+ hires_audio = tensor::reshape(latent[:, :, :, 24:], [tensor::numel(latent[:, :, :, 24:])])[:128]
+ hires_audio_channels = math::ceil_div(128, hires_shape[0] * hires_shape[1])
+ hires_base = tensor::concat([hires_image_base, tensor::reshape(tensor::pad(hires_audio, [0, hires_shape[0] * hires_shape[1] * hires_audio_channels - 128]), [hires_shape[0], hires_shape[1], 1, hires_audio_channels, 1])], axis = 3)
+{% else %}
+ hires_base = hires_image_base
+{% endif %}
+{% if flags.mask_stage %}
+ hires_mask = tensor::interpolate({{hires_mask}}, size = tensor::shape(hires_base)[:2], mode = "nearest_max")
+{% elif flags.has_mask %}
+ hires_mask = tensor::interpolate({{hires_mask}}, size = tensor::shape(hires_base)[:2], mode = "nearest_max")
+{% endif %}
+{% if flags.hires_rebuild %}
+{% if flags.inpaint_model %}
+ hires_inpaint_condition = tensor::interpolate(inpaint_condition, size = tensor::shape(hires_base)[:2])
+{% elif flags.unet_edit %}
+ hires_edit_latent = tensor::interpolate(edit_latent, size = tensor::shape(hires_base)[:2])
+{% elif flags.native_control %}
+{% if flags.control_image %}
+ hires_control_pixels = image::to_f32(fit_canvas(input_control, hires_target_width, hires_target_height, mode = "nearest"), channels = 3)
+ hires_control_latent = {{from_vae}}(vae, vae_encode(vae, hires_control_pixels, rng = hires_rng{{vae_run_options}}))
+{% else %}
+ hires_control_latent = tensor::zeros_like(hires_base)
+{% endif %}
+{% endif %}
+{% if flags.z_control %}
+ hires_control_pixels = image::to_f32(fit_canvas(input_control, hires_target_width, hires_target_height, mode = "nearest"), channels = 3)
+ hires_control_encoded = {{from_vae}}(vae, vae_encode(vae, hires_control_pixels, rng = hires_rng, posterior = "mode"{{vae_run_options}}))
+ hires_control_hint = tensor::pad(hires_control_encoded, [0, 0, 0, 0, 0, 33 - tensor::shape(hires_control_encoded)[2]])
+ hires_controlled_positive = controlnet::apply(positive, control_model, hires_control_hint, strength = control_strength, type = {{control_type}})
+ hires_controlled_negative = controlnet::apply(negative, control_model, hires_control_hint, strength = control_strength, type = {{control_type}})
+{% endif %}
+{% if flags.image_guidance %}
+{% if flags.reference_image_guidance %}
+{% if flags.z_control %}
+ hires_controlled_unconditional = controlnet::apply(unconditional_text, control_model, hires_control_hint, strength = control_strength, type = {{control_type}})
+{% endif %}
+{% else %}
+ hires_image_unconditional_concat = tensor::interpolate(image_unconditional_concat, size = tensor::shape(hires_base)[:2])
+{% endif %}
+{% endif %}
+ hires_pos = predictor(model, {{hires_positive_condition}}{{hires_predictor_options}})
+ hires_neg = predictor({{negative_model}}, {{hires_negative_condition}}{{hires_negative_predictor_options}})
+{% if flags.image_guidance %}
+ hires_image_only = predictor(model, {{hires_image_only_condition}}{{hires_image_only_options}})
+{% endif %}
+ hires_guided = {{guidance_function}}(hires_pos, hires_neg, scale = cfg_scale{{hires_guidance_options}})
+{% endif %}
+{% if flags.hires_custom_sigmas %}
+ hires_sigmas = tensor::vector(hires_custom_sigmas, dtype = "f32")
+{% elif flags.hires_sefi %}
+ hires_schedule = sefi::make_sigmas(steps = hires_scheduler_steps{{sefi_schedule_options}})
+ hires_sigmas = hires_schedule.sigmas[-(hires_steps + 1):]
+{% else %}
+ hires_schedule = make_sigmas({{hires_schedule_predictor}}, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler{{schedule_options}})
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+{% endif %}
+{% if flags.hidream_o1 %}
+ hires_start = add_noise({{hires_schedule_predictor}}, hires_base, tensor::mul(tensor::randn_like(hires_base, rng = hires_rng), eta), sigma = hires_sigmas[0])
+{% else %}
+ hires_start = add_noise({{hires_schedule_predictor}}, hires_base, tensor::randn_like(hires_base, rng = hires_rng), sigma = hires_sigmas[0])
+{% endif %}
+{% if flags.hires_sefi %}
+{% if flags.hires_custom_sigmas %}
+ hires_semantic_schedule = sefi::make_sigmas(steps = tensor::numel(hires_sigmas) - 1{{sefi_schedule_options}})
+ hires_semantic_sigmas = hires_semantic_schedule.semantic_sigmas
+{% else %}
+ hires_semantic_sigmas = hires_schedule.semantic_sigmas[-(hires_steps + 1):]
+{% endif %}
+ hires_latent = sefi::sample(hires_start, hires_sigmas, semantic_sigmas = hires_semantic_sigmas, predict = {{hires_schedule_predictor}}, method = method, rng = hires_sampler_rng, eta = eta{{hires_sample_options}})
+{% else %}
+ hires_latent = sample(hires_start, hires_sigmas, predict = {{hires_schedule_predictor}}, method = method, rng = hires_sampler_rng, eta = eta{{hires_sample_options}})
+{% endif %}
diff --git a/src/api/templates/image_inpaint.dgl b/src/api/templates/image_inpaint.dgl
new file mode 100644
index 0000000..536e820
--- /dev/null
+++ b/src/api/templates/image_inpaint.dgl
@@ -0,0 +1,147 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+ image_base = {{from_vae}}(vae, vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng{{vae_run_options}}))
+{% else %}
+ image_base = tensor::zeros(latent_shape(vae, width = width, height = height))
+{% endif %}
+ base = image_base
+{{references}}
+{% if flags.unet_edit %}
+{% if flags.references %}
+ edit_latent = tensor::interpolate(reference_latent_0, size = tensor::shape(base)[:2])
+{% else %}
+ edit_latent = tensor::zeros_like(base)
+{% endif %}
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::zeros_like(edit_latent)
+{% endif %}
+{% endif %}
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+{% if flags.native_control %}
+ control_latent = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng{{vae_run_options}}))
+{% else %}
+ control_model = controlnet::load(control_path{{control_options}})
+{% if flags.z_control %}
+ control_encoded = {{from_vae}}(vae, vae_encode(vae, control_pixels, rng = encode_rng, posterior = "mode"{{vae_run_options}}))
+ control_hint = tensor::pad(control_encoded, [0, 0, 0, 0, 0, 33 - tensor::shape(control_encoded)[2]])
+{% endif %}
+{% endif %}
+{% endif %}
+{% if flags.mask_stage %}
+{% if flags.has_mask %}
+ mask_pixels = tensor::threshold(image::to_f32(fit_canvas(tensor::grayscale(input_mask, axis = 0), width, height, mode = "nearest_exact")), threshold = 0.5)
+{% else %}
+ mask_pixels = tensor::full([width, height, 1, 1], 1)
+{% endif %}
+ base_shape = tensor::shape(base)
+ mask_scale = [math::floor(width / base_shape[0]), math::floor(height / base_shape[1])]
+ mask_latent = tensor::max_pool2d(mask_pixels, kernel = mask_scale, stride = mask_scale)
+{% if flags.inpaint_model %}
+{% if flags.flex %}
+ masked_latent = tensor::mul(base, tensor::affine(mask_latent, scale = -1, offset = 1))
+{% elif flags.has_init %}
+ masked_pixels = tensor::lerp(image::to_f32(source, channels = {{channels}}), 0.5, mask_pixels)
+ masked_latent = {{from_vae}}(vae, vae_encode(vae, masked_pixels, rng = encode_rng{{vae_run_options}}))
+{% else %}
+ masked_latent = tensor::zeros_like(base)
+{% endif %}
+{% if flags.flux_fill %}
+ mask_grid = tensor::reshape(mask_pixels, [mask_scale[0], base_shape[0], mask_scale[1], base_shape[1]])
+ mask_channels = tensor::reshape(tensor::permute(mask_grid, [1, 3, 0, 2]), [base_shape[0], base_shape[1], mask_scale[0] * mask_scale[1], 1])
+ inpaint_condition = tensor::concat([masked_latent, mask_channels], axis = 2)
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::concat([tensor::zeros_like(masked_latent), mask_channels], axis = 2)
+{% endif %}
+{% elif flags.flex %}
+{% if flags.control_image %}
+ inpaint_condition = tensor::concat([masked_latent, mask_latent, control_latent], axis = 2)
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::concat([tensor::zeros_like(masked_latent), mask_latent, tensor::zeros_like(masked_latent)], axis = 2)
+{% endif %}
+{% else %}
+ inpaint_condition = tensor::concat([masked_latent, mask_latent, tensor::zeros_like(base)], axis = 2)
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::concat([tensor::zeros_like(masked_latent), mask_latent, tensor::zeros_like(masked_latent)], axis = 2)
+{% endif %}
+{% endif %}
+{% else %}
+ inpaint_condition = tensor::concat([mask_latent, masked_latent], axis = 2)
+{% if flags.image_guidance %}
+ image_unconditional_concat = tensor::concat([mask_latent, tensor::zeros_like(masked_latent)], axis = 2)
+{% endif %}
+{% endif %}
+ denoise_mask = tensor::max_pool2d(mask_latent, kernel = [3, 3], padding_before = [1, 1], padding_after = [1, 1])
+{% endif %}
+{% endif %}
+{% if flags.separate_negative_model %}
+ negative_model = load_dit(negative_model_path{{model_options}})
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.minit2i %}
+ negative_text = zero_out(positive_text, fields = ["vector"])
+{% elif flags.ming %}
+ negative_text = zero_out(positive_text)
+{% elif flags.ideogram %}
+ negative_text = {vector = tensor::vector([1])}
+{% else %}
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% endif %}
+ positive = positive_text
+{% if flags.sdxl_zero %}
+ negative = sdxl_zero_text(negative_text)
+{% else %}
+ negative = negative_text
+{% endif %}
+{% if flags.control_image_guidance %}
+ image_unconditional_concat = tensor::zeros_like(base)
+{% endif %}
+{% include "image_predictors" %}
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+{% else %}
+ sigmas = full_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+{% if flags.hidream_o1 %}
+ start = add_noise(guided, base, tensor::mul(tensor::randn_like(base, rng = rng), eta), sigma = sigmas[0])
+{% else %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+{% endif %}
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ image = vae_decode(vae, {{to_vae}}(vae, {{result_latent}}){{vae_run_options}})
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_layered.dgl b/src/api/templates/image_layered.dgl
new file mode 100644
index 0000000..32848a6
--- /dev/null
+++ b/src/api/templates/image_layered.dgl
@@ -0,0 +1,52 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+ image_shape = latent_shape(vae, width = width, height = height)
+ base = tensor::zeros([image_shape[0], image_shape[1], layers, image_shape[2], image_shape[3]])
+{{references}}
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+ control_model = controlnet::load(control_path{{control_options}})
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+ positive = positive_text
+ negative = negative_text
+{% include "image_predictors" %}
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+ sigmas = full_sigmas
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ image = map(tensor::to_list(tensor::arange(0, layers)), process = decode_layer, args = {vae = vae, latent = {{result_latent}}})
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_pixel.dgl b/src/api/templates/image_pixel.dgl
new file mode 100644
index 0000000..6814706
--- /dev/null
+++ b/src/api/templates/image_pixel.dgl
@@ -0,0 +1,64 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = image::to_f32(fit_canvas(input_image, width, height, mode = "nearest"), channels = {{channels}})
+ base = tensor::affine(source, scale = 2, offset = -1)
+{% else %}
+ base = tensor::zeros([width, height, {{channels}}, 1])
+{% endif %}
+{{references}}
+{% if flags.has_mask %}
+ mask_pixels = tensor::threshold(image::to_f32(fit_canvas(tensor::grayscale(input_mask, axis = 0), width, height, mode = "nearest_exact")), threshold = 0.5)
+ mask_latent = mask_pixels
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.minit2i %}
+ negative_text = zero_out(positive_text, fields = ["vector"])
+{% else %}
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% endif %}
+ pos = predictor(model, positive_text{{predictor_options}})
+ neg = predictor(model, negative_text{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ full_sigmas = make_sigmas({{schedule_predictor}}, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+{% else %}
+ sigmas = full_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ pixels = tensor::clamp(tensor::affine({{result_latent}}, scale = 0.5, offset = 0.5), 0, 1)
+ return {image = image::to_u8(pixels), latent = {{result_latent}}}
+}
diff --git a/src/api/templates/image_predictors.dgl b/src/api/templates/image_predictors.dgl
new file mode 100644
index 0000000..2ff3cc8
--- /dev/null
+++ b/src/api/templates/image_predictors.dgl
@@ -0,0 +1,16 @@
+{% if flags.reference_image_guidance %}
+ unconditional_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{unconditional_prompt_options}})
+{% endif %}
+{% if flags.controlnet %}
+ controlled_positive = controlnet::apply(positive, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+ controlled_negative = controlnet::apply(negative, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+{% if flags.reference_image_guidance %}
+ controlled_unconditional = controlnet::apply(unconditional_text, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+{% endif %}
+{% endif %}
+ pos = predictor(model, {{positive_condition}}{{predictor_options}})
+ neg = predictor({{negative_model}}, {{negative_condition}}{{negative_predictor_options}})
+{% if flags.image_guidance %}
+ image_only = predictor(model, {{image_only_condition}}{{image_only_options}})
+{% endif %}
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
diff --git a/src/api/templates/image_reference.dgl b/src/api/templates/image_reference.dgl
new file mode 100644
index 0000000..41988f3
--- /dev/null
+++ b/src/api/templates/image_reference.dgl
@@ -0,0 +1,17 @@
+ {{name}}_original = image::to_f32({{name}}, channels = {{channels}})
+{% if center_crop %}
+ {{name}}_resized = tensor::interpolate({{name}}_original, size = [{{resize_width}}, {{resize_height}}], mode = "lanczos")
+ {{name}}_pixels = tensor::crop({{name}}_resized, axes = [0, 1], start = [{{crop_x}}, {{crop_y}}], size = [{{width}}, {{height}}])
+{% else %}
+ {{name}}_pixels = tensor::interpolate({{name}}_original, size = [{{width}}, {{height}}], mode = {{mode}})
+{% endif %}
+{% if encode %}
+{% if flags.raw_reference %}
+ {{latent}} = vae_encode(vae, {{name}}_pixels, rng = encode_rng, posterior = "mode"{{vae_run_options}})
+{% elif augment %}
+ {{latent}}_raw = vae_to_dit_latents(vae, vae_encode(vae, {{name}}_pixels, rng = encode_rng{{vae_run_options}}))
+ {{latent}} = h3_image_reference_noise({{latent}}_raw, seed)
+{% else %}
+ {{latent}} = vae_to_dit_latents(vae, vae_encode(vae, {{name}}_pixels, rng = encode_rng{{vae_run_options}}))
+{% endif %}
+{% endif %}
diff --git a/src/api/templates/image_sefi.dgl b/src/api/templates/image_sefi.dgl
new file mode 100644
index 0000000..f7e799a
--- /dev/null
+++ b/src/api/templates/image_sefi.dgl
@@ -0,0 +1,81 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "image_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = {{model_loader}}(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+{% if flags.has_init %}
+ source = fit_canvas(input_image, width, height, mode = "nearest")
+ image_base = {{from_vae}}(vae, vae_encode(vae, image::to_f32(source, channels = {{channels}}), rng = encode_rng{{vae_run_options}}))
+ base = tensor::pad(image_base, [0, 0, 0, 0, 16, 0])
+{% else %}
+ base = tensor::zeros(latent_shape(vae, width = width, height = height))
+{% endif %}
+{{references}}
+{% if flags.control_image %}
+ control_pixels = image::to_f32(fit_canvas(input_control, width, height, mode = "nearest"), channels = 3)
+ control_model = controlnet::load(control_path{{control_options}})
+{% endif %}
+{% if flags.mask_stage %}
+{% if flags.has_mask %}
+ mask_pixels = tensor::threshold(image::to_f32(fit_canvas(tensor::grayscale(input_mask, axis = 0), width, height, mode = "nearest_exact")), threshold = 0.5)
+{% else %}
+ mask_pixels = tensor::full([width, height, 1, 1], 1)
+{% endif %}
+ base_shape = tensor::shape(base)
+ mask_scale = [math::floor(width / base_shape[0]), math::floor(height / base_shape[1])]
+ mask_latent = tensor::max_pool2d(mask_pixels, kernel = mask_scale, stride = mask_scale)
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, {{negative_prompt_input}}, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+ positive = positive_text
+ negative = negative_text
+{% if flags.controlnet %}
+ controlled_positive = controlnet::apply(positive, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+ controlled_negative = controlnet::apply(negative, control_model, {{control_hint}}, strength = control_strength, type = {{control_type}})
+{% endif %}
+ pos = predictor(model, {{positive_condition}}{{predictor_options}})
+ neg = predictor({{negative_model}}, {{negative_condition}}{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.custom_sigmas %}
+ full_sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+ dual_schedule = sefi::make_sigmas(steps = tensor::numel(full_sigmas) - 1{{sefi_schedule_options}})
+{% else %}
+ dual_schedule = sefi::make_sigmas(steps = steps{{sefi_schedule_options}})
+ full_sigmas = dual_schedule.sigmas
+{% endif %}
+{% if flags.has_init %}
+ sigmas = slice_sigmas_by_strength(full_sigmas, strength)
+ semantic_sigmas = dual_schedule.semantic_sigmas[-tensor::numel(sigmas):]
+{% else %}
+ sigmas = full_sigmas
+ semantic_sigmas = dual_schedule.semantic_sigmas
+{% endif %}
+ rng = make_rng(rng_type, seed = seed)
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sefi::sample(start, sigmas, predict = guided, semantic_sigmas = semantic_sigmas, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "image_hires" %}
+{% endif %}
+ image = vae_decode(vae, dit_to_vae_latents(vae, {{result_latent}}[:, :, 16:144]){{vae_run_options}})
+ return {image = image, latent = {{result_latent}}}
+}
diff --git a/src/api/templates/load_components.dgl b/src/api/templates/load_components.dgl
new file mode 100644
index 0000000..8c41c7f
--- /dev/null
+++ b/src/api/templates/load_components.dgl
@@ -0,0 +1,33 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+fn main() {
+ model = {{model_loader}}(model_path{{model_options}})
+ text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{% if flags.high_model %}
+ load_dit(high_model_path{{high_model_options}})
+{% endif %}
+{% if flags.negative_model %}
+ load_dit(negative_model_path{{negative_model_options}})
+{% endif %}
+{% if flags.control %}
+ controlnet::load(control_path{{control_options}})
+{% endif %}
+{% if flags.vision %}
+ load_vision_encoder(vision_path{{vision_options}})
+{% endif %}
+{% if flags.audio_encoder %}
+ load_audio_encoder(audio_encoder_path{{audio_encoder_options}})
+{% endif %}
+{% if flags.audio_vae %}
+ load_audio_vae(audio_vae_path{{audio_vae_options}})
+{% endif %}
+{% if flags.preview %}
+ load_tae(preview_path, decode_only = true{{preview_options}})
+{% endif %}
+ return {model = model, text = text, vae = vae}
+}
diff --git a/src/api/templates/lora.dgl b/src/api/templates/lora.dgl
new file mode 100644
index 0000000..c12578f
--- /dev/null
+++ b/src/api/templates/lora.dgl
@@ -0,0 +1,7 @@
+ {{name}} = load_lora({{path|literal}})
+{% if diffusion %}
+ {{model_result}} = apply_lora({{model_source}}, {{name}}, weight = {{weight}}, mode = {{mode|literal}})
+{% endif %}
+{% if conditioning %}
+ {{text_result}} = apply_lora({{text_source}}, {{name}}, weight = {{weight}}, mode = {{mode|literal}})
+{% endif %}
diff --git a/src/api/templates/upscale.dgl b/src/api/templates/upscale.dgl
new file mode 100644
index 0000000..daa4649
--- /dev/null
+++ b/src/api/templates/upscale.dgl
@@ -0,0 +1,17 @@
+# version: 0.0.0
+
+var {
+ model_path = {{model_path}}
+ tile_size = {{tile_size}}
+ repeats = {{repeats}}
+}
+
+fn enlarge(image, index, model) {
+ return upscale(model, image)
+}
+
+fn main(input_image) {
+ model = load_upscaler(model_path{{loader_options}}, tile_size = tile_size)
+ image = fold(tensor::to_list(tensor::arange(0, repeats)), input_image, process = enlarge, model = model)
+ return {image = image}
+}
diff --git a/src/api/templates/video_h3.dgl b/src/api/templates/video_h3.dgl
new file mode 100644
index 0000000..db9a360
--- /dev/null
+++ b/src/api/templates/video_h3.dgl
@@ -0,0 +1,75 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "video_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = load_dit(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+{% if flags.first %}
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+{% if flags.last %}
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+{% if flags.decode_audio %}
+ audio_vae = load_audio_vae(audio_vae_path{{audio_vae_options}})
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+{% else %}
+ audio_shape = empty_audio_shape
+{% endif %}
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+{{references}}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% if flags.vision %}
+ positive = {crossattn = positive_text.crossattn, vector = vision_condition}
+ negative = {crossattn = negative_text.crossattn, vector = vision_condition}
+{% else %}
+ positive = positive_text
+ negative = negative_text
+{% endif %}
+ pos = predictor(model, positive{{predictor_options}})
+ neg = predictor(model, negative{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.custom_sigmas %}
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+ rng = encode_rng
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "video_hires" %}
+{% endif %}
+ video_latent = {{result_latent}}[:, :, :{{output_video_frames}}, :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent){{vae_run_options}})
+{% if flags.decode_audio %}
+ sound_latent = unpack_audio({{result_latent}}, channels = tensor::shape(video_base)[3], audio_shape = {{output_audio_shape}}, audio_scale = audio_scale)
+ decoded_audio = audio_vae_decode(audio_vae, sound_latent)
+ max_audio_samples = math::floor(tensor::shape(output_frames)[3] * decoded_audio.sample_rate / fps)
+ audio = {samples = decoded_audio.samples[:, :max_audio_samples], sample_rate = decoded_audio.sample_rate}
+{% else %}
+ audio = {}
+{% endif %}
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = {{result_latent}}, audio = audio}
+}
diff --git a/src/api/templates/video_helpers.dgl b/src/api/templates/video_helpers.dgl
new file mode 100644
index 0000000..6728be7
--- /dev/null
+++ b/src/api/templates/video_helpers.dgl
@@ -0,0 +1,34 @@
+fn fit_canvas(input, width, height, mode) {
+ shape = tensor::shape(input)
+ crop_width = math::min(shape[1], math::floor(shape[2] * width / height))
+ crop_height = math::min(shape[2], math::floor(shape[1] * height / width))
+ start = [math::floor((shape[1] - crop_width) / 2), math::floor((shape[2] - crop_height) / 2)]
+ cropped = tensor::crop(input, axes = [1, 2], start = start, size = [crop_width, crop_height])
+ return tensor::interpolate(cropped, size = [width, height], axes = [1, 2], mode = mode)
+}
+
+fn pack_av(video, audio, audio_scale) {
+ shape = tensor::shape(video)
+ count = tensor::numel(audio)
+ tail_channels = math::ceil_div(count, shape[0] * shape[1] * shape[2])
+ tail_shape = [shape[0], shape[1], shape[2], tail_channels, 1]
+ flat = tensor::reshape(tensor::mul(audio, audio_scale), [count])
+ padded = tensor::pad(flat, [0, math::product(tail_shape) - count])
+ return tensor::concat([video, tensor::reshape(padded, tail_shape)], axis = 3)
+}
+
+fn unpack_audio(packed, channels, audio_shape, audio_scale) {
+ tail = packed[:, :, :, channels:]
+ flat = tensor::reshape(tail, [tensor::numel(tail)])
+ return tensor::div(tensor::reshape(flat[:math::product(audio_shape)], audio_shape), audio_scale)
+}
+
+fn h3_reference_noise(latent, seed) {
+ rng = make_rng("cuda", seed = seed)
+ return tensor::lerp(latent, tensor::randn_like(latent, rng = rng), 0.001)
+}
+
+fn control_frame_pixels(index, args) {
+ pixels = image::to_f32(fit_canvas(args.frames[:, :, :, index], args.width, args.height, mode = "nearest"), channels = 3)
+ return tensor::unsqueeze(pixels, axis = 2)
+}
diff --git a/src/api/templates/video_hires.dgl b/src/api/templates/video_hires.dgl
new file mode 100644
index 0000000..4ca476e
--- /dev/null
+++ b/src/api/templates/video_hires.dgl
@@ -0,0 +1,70 @@
+ hires_source = latent[:, :, :tensor::shape(video_base)[2], :tensor::shape(video_base)[3]]
+{% if flags.hires_model %}
+ hires_upscaler = load_latent_upscaler(hires_model_path{{hires_model_options}})
+{% if flags.hires_ltx %}
+ hires_upscaled = upscale_latent(hires_upscaler, dit_to_vae_latents(vae, hires_source))
+ hires_upscaled_shape = tensor::shape(hires_upscaled)
+ hires_video = vae_to_dit_latents(vae, tensor::reshape(hires_upscaled, [hires_upscaled_shape[0], hires_upscaled_shape[1], hires_upscaled_shape[2], hires_upscaled_shape[3], 1]))
+{% else %}
+ hires_upscaled = upscale_latent(hires_upscaler, hires_source, width = math::floor(hires_target_width / hires_vae_scale), height = math::floor(hires_target_height / hires_vae_scale))
+ hires_upscaled_shape = tensor::shape(hires_upscaled)
+ hires_video = tensor::reshape(hires_upscaled, [hires_upscaled_shape[0], hires_upscaled_shape[1], hires_upscaled_shape[2], hires_upscaled_shape[3], 1])
+{% endif %}
+{% else %}
+ hires_video = tensor::interpolate(hires_source, size = [math::floor(hires_target_width / hires_vae_scale), math::floor(hires_target_height / hires_vae_scale)], mode = hires_mode, antialias = hires_antialias)
+{% endif %}
+ hires_video_shape = tensor::shape(hires_video)
+ hires_original_audio = unpack_audio(latent, channels = tensor::shape(video_base)[3], audio_shape = audio_shape, audio_scale = audio_scale)
+{% if flags.hires_ltx %}
+ hires_width = hires_video_shape[0] * hires_vae_scale
+ hires_height = hires_video_shape[1] * hires_vae_scale
+ hires_frames = (hires_video_shape[2] - 1) * 8 + 1
+{% if flags.decode_audio %}
+ hires_audio_shape = audio_latent_shape(audio_vae, frames = hires_frames, fps = fps)
+{% else %}
+ hires_audio_shape = [16, math::ceil(hires_frames * 25 / fps), 8, 1]
+{% endif %}
+ hires_audio_retained = math::min(audio_shape[1], hires_audio_shape[1])
+ hires_audio = tensor::pad(hires_original_audio[:, :hires_audio_retained], [0, 0, 0, hires_audio_shape[1] - hires_audio_retained])
+{% if flags.conditioned %}
+ hires_video_mask = tensor::full([hires_video_shape[0], hires_video_shape[1], hires_video_shape[2], 1, 1], 1)
+{% if flags.first %}
+ hires_first_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, hires_width, hires_height, mode = "nearest"), channels = 3), axis = 2)
+ hires_first_latent = vae_to_dit_latents(vae, vae_encode(vae, hires_first_pixels, rng = encode_rng{{vae_run_options}}))
+ hires_first = tensor::replace_slice(hires_video, hires_first_latent, axis = 2, start = 0)
+ hires_first_mask = tensor::index_fill(hires_video_mask, axis = 2, indices = [0], value = 0)
+{% else %}
+ hires_first = hires_video
+ hires_first_mask = hires_video_mask
+{% endif %}
+{% if flags.last %}
+ hires_last_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, hires_width, hires_height, mode = "nearest"), channels = 3), axis = 2)
+ hires_last_latent = vae_to_dit_latents(vae, vae_encode(vae, hires_last_pixels, rng = encode_rng{{vae_run_options}}))
+ hires_conditioned = tensor::concat([hires_first, hires_last_latent], axis = 2)
+ hires_condition_mask = tensor::concat([hires_first_mask, tensor::zeros([hires_video_shape[0], hires_video_shape[1], tensor::shape(hires_last_latent)[2], 1, 1])], axis = 2)
+ hires_positions = ltx::video_positions(hires_first, keyframe_latent_frames = tensor::shape(hires_last_latent)[2], keyframe_frame_index = hires_frames - 1, keyframe_pixel_frames = 1, fps = fps)
+{% else %}
+ hires_conditioned = hires_first
+ hires_condition_mask = hires_first_mask
+{% endif %}
+ hires_base = pack_av(hires_conditioned, hires_audio, audio_scale = audio_scale)
+ hires_packed_shape = tensor::shape(hires_base)
+ hires_packed_mask = tensor::concat([tensor::broadcast_to(hires_condition_mask, tensor::shape(hires_conditioned)), tensor::full([hires_packed_shape[0], hires_packed_shape[1], hires_packed_shape[2], hires_packed_shape[3] - hires_video_shape[3], 1], 1)], axis = 3)
+{% else %}
+ hires_base = pack_av(hires_video, hires_audio, audio_scale = audio_scale)
+{% endif %}
+ hires_pos = predictor(model, positive{{predictor_core}}, audio_length = hires_audio_shape[1], fps = fps{{hires_positions_option}})
+ hires_neg = predictor(model, negative{{predictor_core}}, audio_length = hires_audio_shape[1], fps = fps{{hires_positions_option}})
+ hires_guided = {{guidance_function}}(hires_pos, hires_neg, scale = cfg_scale{{guidance_options}})
+{% else %}
+ hires_audio_shape = audio_shape
+ hires_base = pack_av(hires_video, hires_original_audio, audio_scale = audio_scale)
+{% endif %}
+{% if flags.hires_custom_sigmas %}
+ hires_sigmas = tensor::vector(hires_custom_sigmas, dtype = "f32")
+{% else %}
+ hires_schedule = make_sigmas({{hires_schedule_predictor}}, latent = hires_base, steps = hires_scheduler_steps, scheduler = scheduler{{schedule_options}})
+ hires_sigmas = hires_schedule[-(hires_steps + 1):]
+{% endif %}
+ hires_start = add_noise({{hires_schedule_predictor}}, hires_base, tensor::randn_like(hires_base, rng = rng), sigma = hires_sigmas[0])
+ hires_latent = sample(hires_start, hires_sigmas, predict = {{hires_schedule_predictor}}, method = method, rng = sampler_rng, eta = eta{{hires_sample_options}})
diff --git a/src/api/templates/video_ltx.dgl b/src/api/templates/video_ltx.dgl
new file mode 100644
index 0000000..a22ccec
--- /dev/null
+++ b/src/api/templates/video_ltx.dgl
@@ -0,0 +1,101 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "video_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = load_dit(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+{% if flags.first %}
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+{% if flags.last %}
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+ video_shape = tensor::shape(video_base)
+{% if flags.conditioned %}
+ video_mask = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+{% if flags.first %}
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ video_first = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_first = tensor::index_fill(video_mask, axis = 2, indices = [0], value = 1 - strength)
+{% else %}
+ video_first = video_base
+ mask_first = video_mask
+{% endif %}
+{% if flags.last %}
+ last_latent = vae_to_dit_latents(vae, vae_encode(vae, last_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ tail_mask = tensor::full([video_shape[0], video_shape[1], tensor::shape(last_latent)[2], 1, 1], 1 - strength)
+ video_conditioned = tensor::concat([video_first, last_latent], axis = 2)
+ mask_conditioned = tensor::concat([mask_first, tail_mask], axis = 2)
+ positions = ltx::video_positions(video_first, keyframe_latent_frames = tensor::shape(last_latent)[2], keyframe_frame_index = frames - 1, keyframe_pixel_frames = 1, fps = fps)
+{% else %}
+ video_conditioned = video_first
+ mask_conditioned = mask_first
+{% endif %}
+{% endif %}
+{% if flags.decode_audio %}
+ audio_vae = load_audio_vae(audio_vae_path{{audio_vae_options}})
+ audio_shape = audio_latent_shape(audio_vae, frames = frames, fps = fps)
+{% else %}
+ audio_shape = empty_audio_shape
+{% endif %}
+{% if flags.conditioned %}
+ base = pack_av(video_conditioned, tensor::zeros(audio_shape), audio_scale = audio_scale)
+ packed_shape = tensor::shape(base)
+ full_video_mask = tensor::broadcast_to(mask_conditioned, tensor::shape(video_conditioned))
+ packed_mask = tensor::concat([full_video_mask, tensor::full([packed_shape[0], packed_shape[1], packed_shape[2], packed_shape[3] - video_shape[3], 1], 1)], axis = 3)
+{% else %}
+ base = pack_av(video_base, tensor::zeros(audio_shape), audio_scale = audio_scale)
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% if flags.vision %}
+ positive = {crossattn = positive_text.crossattn, vector = vision_condition}
+ negative = {crossattn = negative_text.crossattn, vector = vision_condition}
+{% else %}
+ positive = positive_text
+ negative = negative_text
+{% endif %}
+ pos = predictor(model, positive{{predictor_options}})
+ neg = predictor(model, negative{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.custom_sigmas %}
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+ rng = encode_rng
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% if flags.hires %}
+{% include "video_hires" %}
+{% endif %}
+ video_latent = {{result_latent}}[:, :, :{{output_video_frames}}, :tensor::shape(video_base)[3]]
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, video_latent){{vae_run_options}})
+{% if flags.decode_audio %}
+ sound_latent = unpack_audio({{result_latent}}, channels = tensor::shape(video_base)[3], audio_shape = {{output_audio_shape}}, audio_scale = audio_scale)
+ audio = audio_vae_decode(audio_vae, sound_latent)
+{% else %}
+ audio = {}
+{% endif %}
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = {{result_latent}}, audio = audio}
+}
diff --git a/src/api/templates/video_reference.dgl b/src/api/templates/video_reference.dgl
new file mode 100644
index 0000000..7e295b2
--- /dev/null
+++ b/src/api/templates/video_reference.dgl
@@ -0,0 +1,11 @@
+{% if encode_audio %}
+ {{name}} = audio_vae_encode(audio_vae, {{name}}_samples, sample_rate = {{sample_rate}})
+{% else %}
+{% if resize_video %}
+ {{source}}_pixels = image::to_f32(tensor::interpolate(tensor::index_select({{source}}, axis = 3, indices = {{indices}}), size = [{{width}}, {{height}}], axes = [1, 2]), channels = 3)
+{% elif resize_image %}
+ {{source}}_pixels = image::to_f32(tensor::interpolate({{source}}, size = [{{width}}, {{height}}], axes = [1, 2]), channels = 3)
+{% endif %}
+ {{name}}_raw = vae_to_dit_latents(vae, vae_encode(vae, {{pixels}}, rng = encode_rng{{vae_run_options}}))
+ {{name}} = h3_reference_noise({{name}}_raw, seed)
+{% endif %}
diff --git a/src/api/templates/video_wan.dgl b/src/api/templates/video_wan.dgl
new file mode 100644
index 0000000..e237d28
--- /dev/null
+++ b/src/api/templates/video_wan.dgl
@@ -0,0 +1,206 @@
+# version: 0.0.0
+
+var {
+{{variables}}
+}
+
+{% include "video_helpers" %}
+
+fn main({{inputs}}) {
+ base_model = load_dit(model_path{{model_options}})
+ base_text = load_text_encoder(text_path{{text_options}})
+ vae = {{vae_loader}}(vae_path{{vae_options}})
+{{loras}}
+ model = {{patched_model}}
+ text = {{patched_text}}
+ encode_rng = make_rng(rng_type, seed = seed)
+ video_base = tensor::zeros(latent_shape(vae, width = width, height = height, frames = frames))
+{% if flags.first %}
+ first_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(first_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+{% if flags.last %}
+ last_frame_pixels = tensor::unsqueeze(image::to_f32(fit_canvas(last_frame, width, height, mode = "nearest"), channels = 3), axis = 2)
+{% endif %}
+{% if flags.vace %}
+ control_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+{% if flags.controls %}
+ control_indices = tensor::to_list(tensor::arange(0, control_count))
+ control_parts = map(control_indices, process = control_frame_pixels, args = {frames = control_video, width = width, height = height})
+ control_short = tensor::concat(control_parts, axis = 2)
+ control_pixels = tensor::pad(control_short, [0, 0, 0, 0, 0, frames - tensor::shape(control_short)[2]], value = 0.5)
+{% else %}
+ control_pixels = control_blank
+{% endif %}
+ inactive = vae_to_dit_latents(vae, vae_encode(vae, control_blank, rng = encode_rng{{vae_run_options}}))
+ reactive = vae_to_dit_latents(vae, vae_encode(vae, control_pixels, rng = encode_rng{{vae_run_options}}))
+ vace_shape = tensor::shape(inactive)
+ vace_channels = math::floor(width / vace_shape[0]) * math::floor(height / vace_shape[1])
+ vace_mask = tensor::full([vace_shape[0], vace_shape[1], vace_shape[2], vace_channels, 1], 1)
+ vace_video = tensor::concat([inactive, reactive], axis = 3)
+{% if flags.first %}
+ vace_reference = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ reference_context = tensor::concat([vace_reference, tensor::zeros_like(vace_reference)], axis = 3)
+ vace_pixels = tensor::concat([reference_context, vace_video], axis = 2)
+ vace_masks = tensor::pad(vace_mask, [0, 0, 0, 0, 1, 0])
+ base = tensor::pad(video_base, [0, 0, 0, 0, 1, 0])
+{% else %}
+ vace_pixels = vace_video
+ vace_masks = vace_mask
+ base = video_base
+{% endif %}
+ vace_context = tensor::concat([vace_pixels, vace_masks], axis = 3)
+{% elif flags.image_condition %}
+ condition_blank = tensor::full([width, height, frames, 3, 1], 0.5)
+{% if flags.first %}
+ condition_first = tensor::replace_slice(condition_blank, first_frame_pixels, axis = 2, start = 0)
+{% else %}
+ condition_first = condition_blank
+{% endif %}
+{% if flags.last %}
+ condition_pixels = tensor::replace_slice(condition_first, last_frame_pixels, axis = 2, start = frames - 1)
+{% else %}
+ condition_pixels = condition_first
+{% endif %}
+ condition_latent = vae_to_dit_latents(vae, vae_encode(vae, condition_pixels, rng = encode_rng{{vae_run_options}}))
+ condition_shape = tensor::shape(condition_latent)
+ mask_blank = tensor::zeros([condition_shape[0], condition_shape[1], condition_shape[2], 4, 1])
+{% if flags.first %}
+ first_mask = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+{% else %}
+ first_mask = mask_blank
+{% endif %}
+{% if flags.last %}
+ last_mask = tensor::index_fill(first_mask[:, :, :, 3:4], axis = 2, indices = [condition_shape[2] - 1], value = 1)
+ image_mask = tensor::replace_slice(first_mask, last_mask, axis = 3, start = 3)
+{% else %}
+ image_mask = first_mask
+{% endif %}
+ video_concat = tensor::concat([image_mask, condition_latent], axis = 3)
+ base = video_base
+{% elif flags.frozen %}
+ video_shape = tensor::shape(video_base)
+ mask_blank = tensor::full([video_shape[0], video_shape[1], video_shape[2], 1, 1], 1)
+{% if flags.first %}
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ base_first = tensor::replace_slice(video_base, first_latent, axis = 2, start = 0)
+ mask_first = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 0)
+{% else %}
+ base_first = video_base
+ mask_first = mask_blank
+{% endif %}
+{% if flags.last %}
+ last_latent = vae_to_dit_latents(vae, vae_encode(vae, last_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ base = tensor::replace_slice(base_first, last_latent, axis = 2, start = video_shape[2] - 1)
+ video_mask = tensor::index_fill(mask_first, axis = 2, indices = [video_shape[2] - 1], value = 0)
+{% else %}
+ base = base_first
+ video_mask = mask_first
+{% endif %}
+{% else %}
+ base = video_base
+{% endif %}
+{% if flags.s2v_reference %}
+ reference_video = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ reference_shape = tensor::shape(reference_video)
+ reference_latent = tensor::reshape(reference_video, reference_shape[:4])
+{% endif %}
+{% if flags.driving_audio %}
+ audio_encoder = load_audio_encoder(audio_encoder_path{{audio_encoder_options}})
+ speech = encode_audio(audio_encoder, driving_audio, sample_rate = sample_rate)
+ audio_embedding = wan::s2v_audio_window(speech, latent_frames = tensor::shape(video_base)[2])
+{% endif %}
+{% if flags.hunyuan %}
+ concat_shape = tensor::shape(video_base)
+ concat_blank = tensor::zeros_like(video_base)
+ mask_blank = tensor::zeros([concat_shape[0], concat_shape[1], concat_shape[2], 1, 1])
+{% if flags.first %}
+ first_latent = vae_to_dit_latents(vae, vae_encode(vae, first_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ concat_first = tensor::replace_slice(concat_blank, first_latent[:, :, :1], axis = 2, start = 0)
+ mask_first = tensor::index_fill(mask_blank, axis = 2, indices = [0], value = 1)
+{% else %}
+ concat_first = concat_blank
+ mask_first = mask_blank
+{% endif %}
+{% if flags.last %}
+ last_latent = vae_to_dit_latents(vae, vae_encode(vae, last_frame_pixels, rng = encode_rng{{vae_run_options}}))
+ concat_latent = tensor::replace_slice(concat_first, last_latent[:, :, :1], axis = 2, start = concat_shape[2] - 1)
+ concat_mask = tensor::index_fill(mask_first, axis = 2, indices = [concat_shape[2] - 1], value = 1)
+{% else %}
+ concat_latent = concat_first
+ concat_mask = mask_first
+{% endif %}
+ video_concat = tensor::concat([concat_latent, concat_mask], axis = 3)
+{% endif %}
+{% if flags.vision %}
+ vision = load_vision_encoder(vision_path{{vision_options}})
+{% if flags.first %}
+ vision_first = encode_vision(vision, image::to_u8(first_frame_pixels[:, :, 0]))
+{% else %}
+ vision_first = encode_vision(vision, image::to_u8(tensor::full([width, height, 3, 1], 0.5)), zero_input = true)
+{% endif %}
+{% if flags.end_vision %}
+{% if flags.last %}
+ vision_end = encode_vision(vision, image::to_u8(last_frame_pixels[:, :, 0]))
+{% else %}
+ vision_end = encode_vision(vision, image::to_u8(tensor::full([width, height, 3, 1], 0.5)), zero_input = true)
+{% endif %}
+ vision_condition = tensor::concat([vision_first, vision_end], axis = 1)
+{% else %}
+ vision_condition = vision_first
+{% endif %}
+{% endif %}
+{% if flags.dual %}
+ high_base = load_dit(high_model_path{{model_options}})
+{{high_loras}}
+{% endif %}
+ positive_text = encode_prompt(text, prompt, width = width, height = height{{prompt_options}})
+{% if flags.encode_negative %}
+ negative_text = encode_prompt(text, negative_prompt, width = width, height = height{{negative_prompt_options}})
+{% else %}
+ negative_text = positive_text
+{% endif %}
+{% if flags.vision %}
+ positive = {crossattn = positive_text.crossattn, vector = vision_condition}
+ negative = {crossattn = negative_text.crossattn, vector = vision_condition}
+{% else %}
+ positive = positive_text
+ negative = negative_text
+{% endif %}
+ pos = predictor(model, positive{{predictor_options}})
+ neg = predictor(model, negative{{negative_predictor_options}})
+ guided = {{guidance_function}}(pos, neg, scale = cfg_scale{{guidance_options}})
+{% if flags.dual %}
+ high_pos = predictor({{high_model}}, positive{{predictor_options}})
+ high_neg = predictor({{high_model}}, negative{{negative_predictor_options}})
+ high_guided = {{guidance_function}}(high_pos, high_neg, scale = cfg_scale{{guidance_options}})
+{% endif %}
+{% if flags.custom_sigmas %}
+ sigmas = tensor::vector(custom_sigmas, dtype = "f32")
+{% else %}
+ sigmas = make_sigmas(guided, latent = base, steps = steps, scheduler = scheduler{{schedule_options}})
+{% endif %}
+ rng = encode_rng
+{% if flags.shared_rng %}
+ sampler_rng = rng
+{% else %}
+ sampler_rng = make_rng(sampler_rng_type, seed = seed)
+{% endif %}
+ start = add_noise(guided, base, tensor::randn_like(base, rng = rng), sigma = sigmas[0])
+{% if flags.dual %}
+{% if flags.fixed_high_steps %}
+ split = math::min(high_noise_steps, tensor::numel(sigmas) - 1)
+{% else %}
+ split = math::min(tensor::count_nonzero(tensor::threshold(sigmas, threshold = moe_boundary)), tensor::numel(sigmas) - 1)
+{% endif %}
+ handoff = sample(start, sigmas[:split + 1], predict = high_guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+ latent = sample(handoff, sigmas[split:], predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% else %}
+ latent = sample(start, sigmas, predict = guided, method = method, rng = sampler_rng, eta = eta{{sample_options}})
+{% endif %}
+{% if flags.vace_reference %}
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent[:, :, 1:]){{vae_run_options}})
+{% else %}
+ output_frames = vae_decode(vae, dit_to_vae_latents(vae, latent){{vae_run_options}})
+{% endif %}
+ return {video = {frames = output_frames, fps_num = fps, fps_den = 1}, latent = latent, audio = {{audio}}}
+}
diff --git a/src/api/tensor.cpp b/src/api/tensor.cpp
index 7567535..7f0db51 100644
--- a/src/api/tensor.cpp
+++ b/src/api/tensor.cpp
@@ -4,9 +4,23 @@
#include <climits>
#include <cstdlib>
#include <cstring>
+#include <filesystem>
#include <limits>
#include <stdexcept>
+#include "core/file_access.h"
+#include "model_io/npy_io.h"
+
+bool sd_tensor_save(const sd_tensor_t* tensor, const char* path) {
+ return sd::api::call([&] {
+ if (!tensor || !path || !*path)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor and path are required");
+ sd::require_file_access(path, SD_DGL_FILE_WRITE);
+ sd::npy::write(std::filesystem::u8path(path), tensor->type, tensor->shape(), tensor->data(), tensor->bytes);
+ return SD_STATUS_OK;
+ }) == SD_STATUS_OK;
+}
+
static size_t element_size(sd_tensor_type_t type) {
switch (type) {
case SD_TENSOR_U8:
@@ -61,7 +75,10 @@ static sd_tensor_t::Storage allocate_storage(sd_tensor_type_t type, std::vector<
}
sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape)
- : type(type), shape(std::move(shape)), bytes(sd::api::tensor_byte_size(type, this->shape)), storage_(allocate_storage(type, this->shape, bytes)) {}
+ : shape_(std::move(shape)),
+ type(type),
+ bytes(sd::api::tensor_byte_size(type, shape_)),
+ storage_(allocate_storage(type, shape_, bytes)) {}
static std::vector<int64_t> validated_shape(sd_tensor_type_t type, std::vector<int64_t> shape, const void* data, size_t size) {
if (size != sd::api::tensor_byte_size(type, shape) || (size && !data))
@@ -86,7 +103,7 @@ sd::api::TensorRef sd::api::own_tensor(sd_tensor_type_t type, std::vector<int64_
}
sd::Image sd::api::image_view(const sd_tensor_t& tensor) {
- const auto& s = tensor.shape;
+ const auto& s = tensor.shape();
if (tensor.type != SD_TENSOR_U8 || s.size() != 3 || s[0] < 1 || s[0] > 4 ||
s[1] <= 0 || s[2] <= 0 || s[1] > INT32_MAX || s[2] > INT32_MAX || !tensor.data())
throw std::invalid_argument("image requires U8 [channels,width,height], channels in [1,4], and positive dimensions");
@@ -97,13 +114,14 @@ sd::Audio sd::api::audio_view(const sd_audio_t& audio) {
if (!audio.data || audio.sample_rate == 0)
throw std::invalid_argument("audio requires samples and a positive sample rate");
const auto& t = *audio.data;
- if (t.type != SD_TENSOR_F32 || t.shape.size() != 2 || t.shape[0] <= 0 || t.shape[0] > UINT32_MAX || t.shape[1] <= 0 || !t.data())
+ const auto& s = t.shape();
+ if (t.type != SD_TENSOR_F32 || s.size() != 2 || s[0] <= 0 || s[0] > UINT32_MAX || s[1] <= 0 || !t.data())
throw std::invalid_argument("audio requires F32 [channels,samples] with positive dimensions");
- return {audio.sample_rate, uint32_t(t.shape[0]), uint64_t(t.shape[1]), const_cast<float*>(static_cast<const float*>(t.data()))};
+ return {audio.sample_rate, uint32_t(s[0]), uint64_t(s[1]), const_cast<float*>(static_cast<const float*>(t.data()))};
}
std::vector<sd::Image> sd::api::video_frames(const sd_tensor_t& tensor) {
- const auto& s = tensor.shape;
+ const auto& s = tensor.shape();
if (tensor.type != SD_TENSOR_U8 || s.size() != 4 || s[0] < 1 || s[0] > 4 || s[1] <= 0 || s[2] <= 0 || s[3] <= 0 ||
s[1] > INT32_MAX || s[2] > INT32_MAX || s[3] > INT32_MAX || !tensor.data())
throw std::invalid_argument("video requires U8 [channels,width,height,frames] with positive dimensions");
@@ -172,7 +190,7 @@ sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor) {
return valid_tensor(tensor) ? tensor->type : 0;
}
size_t sd_tensor_rank(const sd_tensor_t* tensor) {
- return valid_tensor(tensor) ? tensor->shape.size() : 0;
+ return valid_tensor(tensor) ? tensor->shape().size() : 0;
}
size_t sd_tensor_byte_size(const sd_tensor_t* tensor) {
return valid_tensor(tensor) ? tensor->bytes : 0;
@@ -183,9 +201,9 @@ const void* sd_tensor_data(const sd_tensor_t* tensor) {
int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis) {
if (!valid_tensor(tensor))
return 0;
- if (axis >= tensor->shape.size()) {
+ if (axis >= tensor->shape().size()) {
sd::api::error(SD_STATUS_OUT_OF_RANGE, nullptr, "invalid tensor axis");
return 0;
}
- return tensor->shape[axis];
+ return tensor->shape()[axis];
}
diff --git a/src/api/tensor.h b/src/api/tensor.h
index 9d57287..a9affa6 100644
--- a/src/api/tensor.h
+++ b/src/api/tensor.h
@@ -16,14 +16,34 @@
namespace sd::api {
size_t tensor_byte_size(sd_tensor_type_t type, const std::vector<int64_t>& shape);
+
+ template <typename T>
+ constexpr sd_tensor_type_t tensor_dtype() {
+ if constexpr (std::is_same_v<T, uint8_t>) {
+ return SD_TENSOR_U8;
+ } else if constexpr (std::is_same_v<T, int32_t>) {
+ return SD_TENSOR_I32;
+ } else if constexpr (std::is_same_v<T, int64_t>) {
+ return SD_TENSOR_I64;
+ } else if constexpr (std::is_same_v<T, float>) {
+ return SD_TENSOR_F32;
+ } else {
+ static_assert(std::is_same_v<T, double>, "unsupported tensor type");
+ return SD_TENSOR_F64;
+ }
+ }
}
struct sd_tensor_t {
using Storage = std::variant<sd::Tensor<uint8_t>, sd::Tensor<int32_t>, sd::Tensor<int64_t>, sd::Tensor<float>, sd::Tensor<double>>;
+
+private:
+ const std::vector<int64_t> shape_;
+
+public:
mutable std::atomic<size_t> references{1};
- sd_tensor_type_t type;
- std::vector<int64_t> shape;
- size_t bytes;
+ const sd_tensor_type_t type;
+ const size_t bytes;
private:
Storage storage_;
@@ -33,19 +53,22 @@ public:
sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape, const void* data, size_t size);
template <typename T>
- explicit sd_tensor_t(sd::Tensor<T> tensor)
- : type(std::is_same_v<T, uint8_t> ? SD_TENSOR_U8 : std::is_same_v<T, int32_t> ? SD_TENSOR_I32
- : std::is_same_v<T, int64_t> ? SD_TENSOR_I64
- : std::is_same_v<T, float> ? SD_TENSOR_F32
- : SD_TENSOR_F64),
- shape(tensor.shape()),
- bytes(sd::api::tensor_byte_size(type, shape)),
+ explicit sd_tensor_t(sd::Tensor<T> tensor, bool scalar = false)
+ : shape_(scalar ? std::vector<int64_t>{} : tensor.shape()),
+ type(sd::api::tensor_dtype<T>()),
+ bytes(sd::api::tensor_byte_size(type, shape_)),
storage_(std::move(tensor)) {
- static_assert(std::is_same_v<T, uint8_t> || std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> || std::is_same_v<T, float> || std::is_same_v<T, double>, "unsupported tensor type");
- if (std::get<sd::Tensor<T>>(storage_).numel() != static_cast<int64_t>(bytes / sizeof(T)))
+ if (as<T>().numel() != static_cast<int64_t>(bytes / sizeof(T)))
throw std::invalid_argument("tensor shape and storage size do not match");
}
+ const std::vector<int64_t>& shape() const { return shape_; }
+
+ template <typename T>
+ const sd::Tensor<T>& as() const {
+ return std::get<sd::Tensor<T>>(storage_);
+ }
+
const void* data() const {
return bytes ? std::visit([](const auto& tensor) -> const void* { return tensor.data(); }, storage_) : nullptr;
}
@@ -81,7 +104,6 @@ namespace sd::api {
explicit operator bool() const { return tensor_ != nullptr; }
};
- size_t tensor_byte_size(sd_tensor_type_t type, const std::vector<int64_t>& shape);
TensorRef own_tensor(sd_tensor_type_t type, std::vector<int64_t> shape, void* data);
TensorRef allocate_tensor(sd_tensor_type_t type, std::vector<int64_t> shape);
sd::Image image_view(const sd_tensor_t& tensor);
diff --git a/src/api/upscaler.cpp b/src/api/upscaler.cpp
new file mode 100644
index 0000000..d2c5478
--- /dev/null
+++ b/src/api/upscaler.cpp
@@ -0,0 +1,121 @@
+#include "api/generation.h"
+
+#include "api/result.h"
+#include "dgl/executor.h"
+#include "dgl/models/upscaler.h"
+#include "dgl/run_plan.h"
+#include "model/upscaler/scale.hpp"
+
+struct sd_upscaler {
+ sd::api::GenerationContext context;
+ std::shared_ptr<sd::dgl::models::ImageUpscaler> model;
+ sd::dgl::Graph program;
+ std::string source;
+
+ explicit sd_upscaler(const sd_opt_t* options)
+ : context(options, sd::api::Upscaler) {}
+};
+
+namespace sd::api {
+ static std::string upscale_source(const GenerationContext& context, int repeats) {
+ const auto& settings = context.config.context;
+ std::string options;
+ for (const auto& [name, spec] : std::map<std::string, const char*>{{"backend", settings.backend}, {"params_backend", settings.params_backend}}) {
+ if (!spec || !*spec)
+ continue;
+ SDBackendAssignment assignment;
+ std::string error;
+ if (!sd_parse_backend_assignment(spec, &assignment, &error))
+ throw dgl::Error(SD_STATUS_INVALID_ARGUMENT, error);
+ options += ", " + name + " = " + nlohmann::json(assignment.get(SDBackendModule::UPSCALER)).dump();
+ }
+ if (context.config.post.conv_direct)
+ options += ", direct = true";
+ return render_template("upscale", {{"model_path", nlohmann::json(settings.model_path).dump()},
+ {"tile_size", std::to_string(context.config.post.tile_size)},
+ {"repeats", std::to_string(repeats)},
+ {"loader_options", options}});
+ }
+}
+
+sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* options, sd_upscaler_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ auto result = std::make_unique<sd_upscaler_t>(options);
+ result->source = sd::api::upscale_source(result->context, 1);
+ result->program = sd::dgl::compile(result->source, sd::dgl::operations(), true);
+ const auto& resource = result->program.resources.at(0);
+ auto arguments = resource.arguments;
+ sd::dgl::RunEvents events({}, false);
+ sd::dgl::RunEventsScope scope(events);
+ sd::dgl::RunState state{result->context.execution, events};
+ sd::dgl::RunPlan plan(result->program, state);
+ result->model = plan.load_resource(*resource.operation, arguments).as<sd::dgl::models::ImageUpscaler>();
+ if (result->model->get_scale() <= 0)
+ throw sd::dgl::Error(SD_STATUS_LOAD_FAILED, "invalid image upscaler scale");
+ *out = result.release();
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
+}
+
+void sd_upscaler_free(sd_upscaler_t* upscaler) {
+ delete upscaler;
+}
+
+int sd_upscaler_get_scale(const sd_upscaler_t* upscaler) {
+ sd::api::clear_error();
+ if (!upscaler || !upscaler->model) {
+ sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
+ return 0;
+ }
+ return upscaler->model->get_scale();
+}
+
+int sd_upscaler_probe_scale(const char* path) {
+ int scale = 0;
+ sd::api::call([&] {
+ if (!path || !*path)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "model_path", "model path is required");
+ scale = sd::upscaling::probe_scale(path);
+ return scale ? SD_STATUS_OK : sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "not a recognized RGB upscaler");
+ },
+ SD_STATUS_LOAD_FAILED);
+ return scale;
+}
+
+sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* options, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out || !upscaler)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler and out are required");
+ sd::api::GenerationRun request(upscaler->context);
+ auto snapshot = sd::api::copy_options(options);
+ sd::OptConfig config;
+ const auto status = sd::api::resolve(snapshot.get(), sd::api::Upscale, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ int64_t width = config.post.image.width, height = config.post.image.height;
+ const auto scale = upscaler->model->get_scale();
+ for (int i = 0; i < config.post.repeats; ++i) {
+ width *= scale;
+ height *= scale;
+ if (width > 16384 || height > 16384 || width * height > 64 * 1024 * 1024)
+ throw sd::dgl::Error(SD_STATUS_OUT_OF_RANGE, "upscaled output exceeds 16384 per axis or 64 megapixels");
+ }
+ const auto* input = sd::api::bound_tensor(snapshot.get(), "image");
+ sd::api::GenerationProgram program;
+ program.source = config.post.repeats == 1 ? upscaler->source : sd::api::upscale_source(upscaler->context, config.post.repeats);
+ program.inputs.emplace("input_image", sd::dgl::Value::object(std::shared_ptr<sd_tensor_t>(sd_tensor_retain(input), sd_tensor_free)));
+ auto value = sd::api::execute_program(upscaler->context, program, config.post.repeats == 1 ? &upscaler->program : nullptr);
+ auto result = std::make_unique<sd_result_t>();
+ sd::api::append_program_image(*result, value, program.source);
+ *out = result.release();
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_GENERATION_FAILED);
+}
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index 808375c..7e2fcf8 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -10,7 +10,10 @@
#include <sstream>
#include <stdexcept>
#include "conditioning/prompt_input.h"
+#include "conditioning/prompt_templates.h"
+#include "conditioning/textual_inversion.h"
#include "core/ggml_tensor_utils.h"
+#include "core/image_geometry.h"
#include "core/tensor_ggml.hpp"
#include "core/util.h"
@@ -18,6 +21,7 @@
#include "model/te/clip.hpp"
#include "model/te/llada_image_te.hpp"
#include "model/te/llm.hpp"
+#include "model/te/ltx_connector.hpp"
#include "model/te/ming_image_te.hpp"
#include "model/te/ph3_image_connector.hpp"
#include "model/te/t5.hpp"
@@ -141,12 +145,17 @@ struct ConditionerParams {
bool zero_out_masked = false;
const std::vector<sd::Tensor<float>>* ref_images = nullptr; // for qwen image edit
const std::vector<MiniMaxH3PresentationItem>* minimax_h3_references = nullptr;
- RefImageParams ref_image_params;
+ std::optional<std::string> system_prompt;
+ bool raw = false;
+ std::shared_ptr<sd::TextualInversion> textual_inversion;
};
struct Conditioner {
virtual int resolve_clip_skip(int) const { return 0; }
virtual bool supports_prompt_images() const { return false; }
+ virtual size_t max_reference_images() const { return 0; }
+ virtual bool supports_multimodal_prompt() const { return false; }
+ virtual bool supports_prompt_formatting() const { return false; }
virtual ~Conditioner() = default;
public:
@@ -310,30 +319,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
ggml_type embd2_type = text_model2 ? text_model2->model.get_token_embed_weight()->type : embd_type;
int64_t hidden_size = text_model->model.hidden_size;
int64_t hidden_size2 = text_model2 ? text_model2->model.hidden_size : 0;
- auto on_load = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) {
- if (tensor_storage.ne[0] == hidden_size) {
- embd = ggml_new_tensor_2d(embd_ctx.get(), embd_type, hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
- if (embd == nullptr) {
- return false;
- }
- *dst_tensor = embd;
- } else if (text_model2) {
- if (tensor_storage.ne[0] == hidden_size2) {
- embd2 = ggml_new_tensor_2d(embd_ctx.get(), embd2_type, hidden_size2, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
- if (embd2 == nullptr) {
- return false;
- }
- *dst_tensor = embd2;
- } else {
- LOG_VERBOSE("embedding skipped, wrong hidden size, got %i, expected %i or %i", tensor_storage.ne[0], hidden_size, hidden_size2);
- }
- } else {
- LOG_VERBOSE("embedding skipped, wrong hidden size, got %i, expected %i", tensor_storage.ne[0], hidden_size);
- }
- return true;
- };
- model_loader.set_n_threads(1);
- if (!model_loader.load_tensors(on_load)) {
+ if (!sd::read_textual_inversion(model_loader, embd_ctx.get(), hidden_size, hidden_size2,
+ embd_type, embd2_type, embd, embd2)) {
LOG_ERROR("embedding '%s' failed", embd_name.c_str());
return false;
}
@@ -397,9 +384,14 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
}
std::pair<std::vector<int>, std::vector<float>> tokenize(std::string text,
- size_t min_length = 0,
- size_t max_length = 0,
- bool allow_overflow_expand = true) {
+ size_t min_length = 0,
+ size_t max_length = 0,
+ bool allow_overflow_expand = true,
+ const sd::PromptInput* input = nullptr,
+ const sd::TextualInversion* inversion = nullptr) {
+ sd::PromptInputTemplate prompt_template(input);
+ if (input)
+ text = prompt_template.text;
auto parsed_attention = parse_prompt_attention(text);
{
@@ -438,7 +430,20 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
}
std::vector<int> curr_tokens;
- if (!tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
+ if (inversion) {
+ for (const auto& part : prompt_template.expand(curr_text)) {
+ if (part.type == sd::PromptPart::Type::TEXT) {
+ std::vector<int> text_tokens;
+ if (!tokenizer->encode(part.content, text_tokens))
+ return {};
+ curr_tokens.insert(curr_tokens.end(), text_tokens.begin(), text_tokens.end());
+ } else if (part.type == sd::PromptPart::Type::EMBEDDING) {
+ const auto& range = inversion->token_ranges.at(part.content);
+ for (int i = range.first; i < range.second; ++i)
+ curr_tokens.push_back(text_model->model.vocab_size + i);
+ }
+ }
+ } else if (!tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
@@ -461,7 +466,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
int clip_skip,
int width,
int height,
- bool zero_out_masked = false) {
+ bool zero_out_masked = false,
+ const sd::TextualInversion* inversion = nullptr) {
int64_t t0 = ggml_time_ms();
sd::Tensor<float> hidden_states; // [n_token, hidden_size] or [n_token, hidden_size + hidden_size2]
sd::Tensor<float> pooled;
@@ -501,8 +507,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
max_token_idx,
false,
clip_skip,
- false);
- GGML_ASSERT(!chunk_hidden_states.empty());
+ false,
+ inversion ? &inversion->first : nullptr);
+ if (chunk_hidden_states.empty()) {
+ return {};
+ }
if (sd_version_is_sdxl(version)) {
auto chunk_hidden_states2 = text_model2->compute(n_threads,
input_ids2,
@@ -511,8 +520,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
max_token_idx,
false,
clip_skip,
- false);
- GGML_ASSERT(!chunk_hidden_states2.empty());
+ false,
+ inversion ? &inversion->second : nullptr);
+ if (chunk_hidden_states2.empty()) {
+ return {};
+ }
chunk_hidden_states = sd::ops::concat(chunk_hidden_states, chunk_hidden_states2, 0);
if (chunk_idx == 0) {
@@ -523,8 +535,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
max_token_idx,
true,
clip_skip,
- false);
- GGML_ASSERT(!pooled.empty());
+ false,
+ inversion ? &inversion->second : nullptr);
+ if (pooled.empty()) {
+ return {};
+ }
}
}
int64_t t1 = ggml_time_ms();
@@ -579,7 +594,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
- auto tokens_and_weights = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true);
+ const auto* inversion = conditioner_params.textual_inversion.get();
+ if (inversion)
+ conditioner_params.textual_inversion->load(text_model->model.hidden_size, text_model2 ? text_model2->model.hidden_size : 0);
+ auto tokens_and_weights = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true,
+ inversion ? conditioner_params.prompt_input : nullptr, inversion);
if (tokens_and_weights.first.empty()) {
return {};
}
@@ -591,7 +610,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
conditioner_params.clip_skip,
conditioner_params.width,
conditioner_params.height,
- conditioner_params.zero_out_masked);
+ conditioner_params.zero_out_masked, inversion);
}
};
@@ -959,7 +978,8 @@ struct SD3CLIPEmbedder : public Conditioner {
false,
clip_skip,
false);
- GGML_ASSERT(!chunk_hidden_states_l.empty());
+ if (chunk_hidden_states_l.empty())
+ return {};
chunk_hidden_states_l = ::apply_token_weights(std::move(chunk_hidden_states_l), chunk_weights);
if (chunk_idx == 0) {
@@ -973,7 +993,8 @@ struct SD3CLIPEmbedder : public Conditioner {
true,
clip_skip,
false);
- GGML_ASSERT(!pooled_l.empty());
+ if (pooled_l.empty())
+ return {};
}
} else {
chunk_hidden_states_l = sd::Tensor<float>::zeros({768, static_cast<int64_t>(chunk_len), 1});
@@ -1002,7 +1023,8 @@ struct SD3CLIPEmbedder : public Conditioner {
false,
clip_skip,
false);
- GGML_ASSERT(!chunk_hidden_states_g.empty());
+ if (chunk_hidden_states_g.empty())
+ return {};
chunk_hidden_states_g = ::apply_token_weights(std::move(chunk_hidden_states_g), chunk_weights);
if (chunk_idx == 0) {
@@ -1016,7 +1038,8 @@ struct SD3CLIPEmbedder : public Conditioner {
true,
clip_skip,
false);
- GGML_ASSERT(!pooled_g.empty());
+ if (pooled_g.empty())
+ return {};
}
} else {
chunk_hidden_states_g = sd::Tensor<float>::zeros({1280, static_cast<int64_t>(chunk_len), 1});
@@ -1039,7 +1062,8 @@ struct SD3CLIPEmbedder : public Conditioner {
input_ids,
sd::Tensor<float>(),
false);
- GGML_ASSERT(!chunk_hidden_states_t5.empty());
+ if (chunk_hidden_states_t5.empty())
+ return {};
chunk_hidden_states_t5 = ::apply_token_weights(std::move(chunk_hidden_states_t5), chunk_weights);
} else {
chunk_hidden_states_t5 = sd::Tensor<float>::zeros({4096, static_cast<int64_t>(chunk_len), 1});
@@ -1063,7 +1087,7 @@ struct SD3CLIPEmbedder : public Conditioner {
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
if (zero_out_masked) {
chunk_hidden_states.fill_(0.0f);
}
@@ -1363,7 +1387,7 @@ struct FluxCLIPEmbedder : public Conditioner {
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
if (!hidden_states.empty()) {
hidden_states = sd::ops::concat(hidden_states, chunk_hidden_states, 1);
} else {
@@ -1597,7 +1621,8 @@ struct T5CLIPEmbedder : public Conditioner {
input_ids,
t5_attn_mask_chunk,
false);
- GGML_ASSERT(!chunk_hidden_states.empty());
+ if (chunk_hidden_states.empty())
+ return {};
chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights);
if (zero_out_masked) {
@@ -1607,7 +1632,7 @@ struct T5CLIPEmbedder : public Conditioner {
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
if (!hidden_states.empty()) {
hidden_states = sd::ops::concat(hidden_states, chunk_hidden_states, 1);
@@ -1732,14 +1757,8 @@ struct MiniT2IConditioner : public Conditioner {
if (!tokenizer.encode(conditioner_params.text, tokens)) {
return {};
}
- if (tokens.size() > prompt_length) {
- tokens.resize(prompt_length);
- }
- std::vector<float> mask(tokens.size(), 1.0f);
- while (tokens.size() < prompt_length) {
- tokens.push_back(tokenizer.PAD_TOKEN_ID);
- mask.push_back(0.0f);
- }
+ std::vector<float> mask;
+ tokenizer.pad_tokens(tokens, nullptr, &mask, prompt_length, prompt_length, false);
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, tokens);
std::vector<float> t5_mask(mask.size(), 0.0f);
@@ -1750,7 +1769,8 @@ struct MiniT2IConditioner : public Conditioner {
input_ids,
sd::Tensor<float>::from_vector(t5_mask),
false);
- GGML_ASSERT(!hidden_states.empty());
+ if (hidden_states.empty())
+ return {};
result.c_crossattn = std::move(hidden_states);
result.c_vector = sd::Tensor<float>::from_vector(mask);
return result;
@@ -1758,6 +1778,8 @@ struct MiniT2IConditioner : public Conditioner {
};
struct SenseNovaU1Conditioner : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+
static constexpr size_t kMaxPromptTokens = 12288;
SenseNovaU1Tokenizer tokenizer;
@@ -1769,37 +1791,9 @@ struct SenseNovaU1Conditioner : public Conditioner {
SD_UNUSED(enabled);
}
- static std::string build_query(const std::string& text, bool is_negative) {
- static const std::string kSystemMessage =
- "You are an image generation and editing assistant that accurately understands and executes user intent.\n\n"
- "You support two modes:\n\n1. Think Mode:\nIf the task requires reasoning, you MUST start with a "
- "<think></think> block. Put all reasoning inside the block using plain text. DO NOT include any image tags. "
- "Keep it reasonable and directly useful for producing the final image.\n\n2. Non-Think Mode:\nIf no reasoning "
- "is needed, directly produce the final image.\n\nTask Types:\n\nA. Text-to-Image Generation:\n- Generate a "
- "high-quality image based on the user's description.\n- Ensure visual clarity, semantic consistency, and "
- "completeness.\n- DO NOT introduce elements that contradict or override the user's intent.\n\nB. Image Editing:\n"
- "- Use the provided image(s) as input or reference for modification or transformation.\n- The result can be an "
- "edited image or a new image based on the reference(s).\n- Preserve all unspecified attributes unless explicitly "
- "changed.\n\nGeneral Rules:\n- For any visible text in the image, follow the language specified for the rendered "
- "text in the user's description, not the language of the prompt. If no language is specified, use the user's input "
- "language.";
-
- std::string query;
- if (!is_negative) {
- query += "<|im_start|>system\n";
- query += kSystemMessage;
- query += "<|im_end|>\n";
- }
- query += "<|im_start|>user\n";
- query += text;
- query += "<|im_end|>\n<|im_start|>assistant\n";
- query += is_negative ? "<img>" : "<think>\n\n</think>\n\n<img>";
- return query;
- }
-
- SDCondition tokenize_condition(const std::string& text, bool is_negative) {
+ SDCondition tokenize_condition(const std::string& text) {
std::vector<int> tokens;
- if (!tokenizer.encode(build_query(text, is_negative), tokens)) {
+ if (!tokenizer.encode(text, tokens)) {
return {};
}
if (tokens.empty() || tokens.size() > kMaxPromptTokens) {
@@ -1817,15 +1811,14 @@ struct SenseNovaU1Conditioner : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
SD_UNUSED(n_threads);
- return tokenize_condition(conditioner_params.text, false);
- }
-
- SDCondition get_unconditional_condition(const std::string& text) {
- return tokenize_condition(text, true);
+ return tokenize_condition(conditioner_params.raw ? conditioner_params.text
+ : sd::prompt_templates::sensenova_u1(conditioner_params.text, conditioner_params.system_prompt));
}
};
struct AnimaConditioner : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+
std::shared_ptr<Tokenizer> qwen_tokenizer;
T5UniGramTokenizer t5_tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
@@ -1890,8 +1883,8 @@ struct AnimaConditioner : public Conditioner {
llm->runner_end();
}
- std::tuple<std::vector<int>, std::vector<float>, std::vector<int>, std::vector<float>> tokenize(std::string text) {
- auto parsed_attention = parse_prompt_attention(text);
+ std::tuple<std::vector<int>, std::vector<float>, std::vector<int>, std::vector<float>> tokenize(std::string text, bool raw = false, const std::optional<std::string>& system = std::nullopt) {
+ auto parsed_attention = raw ? std::vector<std::pair<std::string, float>>{{text, 1.f}} : parse_prompt_attention(text);
{
std::stringstream ss;
@@ -1908,6 +1901,11 @@ struct AnimaConditioner : public Conditioner {
std::vector<int> t5_tokens;
std::vector<float> t5_weights;
+ if (!raw && system && !system->empty()) {
+ if (!qwen_tokenizer->encode(sd::prompt_templates::with_system("", system), qwen_tokens))
+ return {};
+ qwen_weights.assign(qwen_tokens.size(), 1.f);
+ }
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
std::vector<int> curr_tokens;
@@ -1942,7 +1940,7 @@ struct AnimaConditioner : public Conditioner {
const ConditionerParams& conditioner_params) override {
int64_t t0 = ggml_time_ms();
- auto tokenized = tokenize(conditioner_params.text);
+ auto tokenized = tokenize(conditioner_params.text, conditioner_params.raw, conditioner_params.system_prompt);
auto& qwen_tokens = std::get<0>(tokenized);
auto& qwen_weights = std::get<1>(tokenized);
auto& t5_tokens = std::get<2>(tokenized);
@@ -1960,13 +1958,14 @@ struct AnimaConditioner : public Conditioner {
{},
false,
false);
- GGML_ASSERT(!hidden_states.empty());
+ if (hidden_states.empty())
+ return {};
hidden_states = apply_token_weights(std::move(hidden_states), qwen_weights);
auto t5_ids_tensor = sd::Tensor<int32_t>::from_vector(t5_tokens);
auto t5_weight_tensor = sd::Tensor<float>::from_vector(t5_weights);
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
SDCondition result;
result.c_crossattn = std::move(hidden_states);
@@ -1977,6 +1976,8 @@ struct AnimaConditioner : public Conditioner {
};
struct LLMEmbedder : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+
SDVersion version;
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
@@ -1987,6 +1988,21 @@ struct LLMEmbedder : public Conditioner {
return llm->supports_prompt_images();
}
+ size_t max_reference_images() const override {
+ if (!llm->enable_vision)
+ return 0;
+ return sd_version_is_minimax_h3(version) || sd_version_is_lingbot_video(version) ||
+ sd_version_is_qwen_image(version) || sd_version_is_mage_flow(version) ||
+ sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_longcat(version)
+ ? std::numeric_limits<size_t>::max()
+ : 0;
+ }
+
+ bool supports_multimodal_prompt() const override {
+ return llm->enable_vision && sd_version_is_minimax_h3(version) &&
+ version != VERSION_MINIMAX_H3_IMAGE && version != VERSION_P_H3_IMAGE;
+ }
+
LLMEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
SDVersion version = VERSION_QWEN_IMAGE,
@@ -2394,6 +2410,10 @@ struct LLMEmbedder : public Conditioner {
if (tokens.empty()) {
return {};
}
+ if (prompt_template_encode_start_idx < 0 || static_cast<size_t>(prompt_template_encode_start_idx) >= weights.size()) {
+ LOG_ERROR("Prompt template exceeds the text encoder context; shorten system or prompt");
+ return {};
+ }
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, tokens);
sd::Tensor<float> attention_mask;
@@ -2480,51 +2500,16 @@ struct LLMEmbedder : public Conditioner {
return new_hidden_states;
}
- void resize_image_dims(int height, int width, int& h_bar, int& w_bar, int factor, int min_size, int max_size, RefImageResizeMode mode) {
- if (min_size > 0 && min_size == max_size) {
- if (mode == RefImageResizeMode::AREA) {
- double beta = std::sqrt(static_cast<double>(min_size) / (static_cast<double>(height) * width));
- h_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::round(height * beta / factor)) * static_cast<int>(factor));
- w_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::round(width * beta / factor)) * static_cast<int>(factor));
- } else if (mode == RefImageResizeMode::LONGEST_SIDE) {
- int current_max_side = std::max(height, width);
- double beta = static_cast<double>(min_size) / current_max_side;
- h_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::round(height * beta / factor)) * static_cast<int>(factor));
- w_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::round(width * beta / factor)) * static_cast<int>(factor));
- }
- return;
- }
-
- if (mode == RefImageResizeMode::AREA) {
- double current_area = static_cast<double>(h_bar) * w_bar;
- if (max_size > 0 && current_area > max_size) {
- double beta = std::sqrt((static_cast<double>(height) * width) / static_cast<double>(max_size));
- h_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::floor(height / beta / factor)) * static_cast<int>(factor));
- w_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::floor(width / beta / factor)) * static_cast<int>(factor));
- } else if (min_size > 0 && current_area < min_size) {
- double beta = std::sqrt(static_cast<double>(min_size) / (static_cast<double>(height) * width));
- h_bar = static_cast<int>(std::ceil(height * beta / factor)) * static_cast<int>(factor);
- w_bar = static_cast<int>(std::ceil(width * beta / factor)) * static_cast<int>(factor);
- }
- } else if (mode == RefImageResizeMode::LONGEST_SIDE) {
- int current_max_side = std::max(height, width);
- if (max_size > 0 && current_max_side > max_size) {
- double beta = static_cast<double>(max_size) / current_max_side;
- h_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::floor(height * beta / factor)) * static_cast<int>(factor));
- w_bar = std::max(static_cast<int>(factor),
- static_cast<int>(std::floor(width * beta / factor)) * static_cast<int>(factor));
- } else if (min_size > 0 && current_max_side < min_size) {
- double beta = static_cast<double>(min_size) / current_max_side;
- h_bar = static_cast<int>(std::ceil(height * beta / factor)) * static_cast<int>(factor);
- w_bar = static_cast<int>(std::ceil(width * beta / factor)) * static_cast<int>(factor);
- }
+ static void fit_h3_vision_dims(int height, int width, int& h_bar, int& w_bar, int factor) {
+ const double area = static_cast<double>(h_bar) * w_bar;
+ if (area > 12845056) {
+ const double beta = std::sqrt(static_cast<double>(height) * width / 12845056);
+ h_bar = std::max(factor, static_cast<int>(std::floor(height / beta / factor)) * factor);
+ w_bar = std::max(factor, static_cast<int>(std::floor(width / beta / factor)) * factor);
+ } else if (area < 3136) {
+ const double beta = std::sqrt(3136.0 / (static_cast<double>(height) * width));
+ h_bar = static_cast<int>(std::ceil(height * beta / factor)) * factor;
+ w_bar = static_cast<int>(std::ceil(width * beta / factor)) * factor;
}
}
@@ -2599,10 +2584,29 @@ struct LLMEmbedder : public Conditioner {
std::string prompt_suffix;
sd::Tensor<float> text_mask;
- int64_t t0 = ggml_time_ms();
- RefImageResizeMode resize_mode = conditioner_params.ref_image_params.vlm_resize_mode;
+ int template_token_delta = 0;
+ bool dynamic_template_crop = false;
+ bool max_length_includes_template = false;
+ auto system_prefix = [&](const std::string& prefix) {
+ auto format = version == VERSION_FLUX2 ? sd::prompt_templates::RoleFormat::Mistral
+ : sd_version_is_lens(version) ? sd::prompt_templates::RoleFormat::GPTOSS
+ : sd_version_is_pid(version) ? sd::prompt_templates::RoleFormat::Gemma
+ : sd::prompt_templates::RoleFormat::ChatML;
+ auto result = sd::prompt_templates::with_system(prefix, conditioner_params.system_prompt, format);
+ if (result != prefix && !dynamic_template_crop) {
+ std::vector<int> before, after;
+ if ((!prefix.empty() && !tokenizer->encode(prefix, before)) ||
+ (!result.empty() && !tokenizer->encode(result, after)))
+ throw std::runtime_error("cannot tokenize prompt template");
+ template_token_delta += static_cast<int>(after.size()) - static_cast<int>(before.size());
+ }
+ return result;
+ };
+
+ int64_t t0 = ggml_time_ms();
if (sd_version_is_minimax_h3(version)) {
+ prompt = system_prefix("");
prompt_template_encode_start_idx = 0;
// The final tap is normalized; LLMRunner exposes it at num_layers + 1.
out_layers = ph3_connector ? std::set<int>{13, 19, 25, 31, 37} : std::set<int>{50};
@@ -2617,14 +2621,7 @@ struct LLMEmbedder : public Conditioner {
int width = static_cast<int>(image.shape()[0]);
int h_bar = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor)) * factor);
int w_bar = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
- resize_image_dims(height,
- width,
- h_bar,
- w_bar,
- factor,
- 3136,
- 12845056,
- RefImageResizeMode::AREA);
+ fit_h3_vision_dims(height, width, h_bar, w_bar, factor);
auto resized = sd::ops::interpolate(
image,
std::vector<int64_t>{w_bar, h_bar, image.shape()[2], image.shape()[3]});
@@ -2730,34 +2727,37 @@ struct LLMEmbedder : public Conditioner {
out_layers = {26};
prompt =
- "<|im_start|>system\nYou are a helpful assistant. Describe the video by detailing the following aspects:\n"
- "1. The main content and theme of the video.\n"
- "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n"
- "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.\n"
- "4. background environment, light, style and atmosphere.\n"
- "5. camera angles, movements, and transitions used in the video.<|im_end|>\n"
- "<|im_start|>user\n";
+ system_prefix(
+ "<|im_start|>system\nYou are a helpful assistant. Describe the video by detailing the following aspects:\n"
+ "1. The main content and theme of the video.\n"
+ "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n"
+ "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.\n"
+ "4. background environment, light, style and atmosphere.\n"
+ "5. camera angles, movements, and transitions used in the video.<|im_end|>\n"
+ "<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
prompt += "<|im_end|>\n<|im_start|>assistant\n";
} else if (sd_version_is_lingbot_video(version)) {
- const int pad_token = 151643;
+ dynamic_template_crop = true;
+ const int pad_token = 151643;
const std::string prompt_prefix =
- "<|im_start|>system\nGiven a user input that may include a text prompt alone, "
- "a text prompt with an image reference, or a text prompt with a video reference "
- "or a video reference alone, generate an \"Enhanced prompt\" that provides detailed "
- "visual descriptions suitable for video generation. Evaluate the level of detail "
- "in the user's input: if it is simple, enrich it by adding specifics about colors, "
- "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal "
- "progression, and spatial relationships to create vivid, concrete, and temporally "
- "coherent scenes to create vivid and concrete scenes. Please generate only the "
- "enhanced description for the prompt below and avoid including any additional "
- "commentary or evaluations:<|im_end|>\n<|im_start|>user\n";
+ system_prefix(
+ "<|im_start|>system\nGiven a user input that may include a text prompt alone, "
+ "a text prompt with an image reference, or a text prompt with a video reference "
+ "or a video reference alone, generate an \"Enhanced prompt\" that provides detailed "
+ "visual descriptions suitable for video generation. Evaluate the level of detail "
+ "in the user's input: if it is simple, enrich it by adding specifics about colors, "
+ "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal "
+ "progression, and spatial relationships to create vivid, concrete, and temporally "
+ "coherent scenes to create vivid and concrete scenes. Please generate only the "
+ "enhanced description for the prompt below and avoid including any additional "
+ "commentary or evaluations:<|im_end|>\n<|im_start|>user\n");
std::vector<int> prefix_tokens;
- if (!tokenizer->encode(prompt_prefix, prefix_tokens, nullptr)) {
+ if (!conditioner_params.raw && !tokenizer->encode(prompt_prefix, prefix_tokens, nullptr)) {
return {};
}
prompt_template_encode_start_idx = 0;
@@ -2780,23 +2780,6 @@ struct LLMEmbedder : public Conditioner {
int height = static_cast<int>(image.shape()[1]);
int width = static_cast<int>(image.shape()[0]);
- int min_pixels = conditioner_params.ref_image_params.vlm_min_size;
- if (min_pixels <= 0) {
- if (resize_mode == RefImageResizeMode::AREA) {
- min_pixels = static_cast<int>(4 * factor * factor);
- } else {
- min_pixels = static_cast<int>(2 * factor);
- }
- }
- int max_pixels = conditioner_params.ref_image_params.vlm_max_size;
- if (max_pixels <= 0) {
- if (resize_mode == RefImageResizeMode::AREA) {
- max_pixels = static_cast<int>(16384 * factor * factor);
- } else {
- max_pixels = static_cast<int>(128 * factor);
- }
- }
-
int h_bar = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor) * factor));
int w_bar = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor) * factor));
@@ -2804,12 +2787,11 @@ struct LLMEmbedder : public Conditioner {
LOG_WARN("LingBotVideo image aspect ratio is very large: %dx%d", width, height);
}
- resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
-
LOG_VERBOSE("resize LingBotVideo ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
auto resized_image = clip_preprocess(image, w_bar, h_bar);
auto image_embed = llm->encode_image(n_threads, resized_image, false);
- GGML_ASSERT(!image_embed.empty());
+ if (image_embed.empty())
+ return {};
std::string image_prefix = prompt + img_prompt + "<|vision_start|>";
std::vector<int> prefix_tokens;
@@ -2837,9 +2819,9 @@ struct LLMEmbedder : public Conditioner {
LOG_ERROR("Qwen Image 2.1 editing requires Qwen3-VL vision weights; provide --llm_vision or a combined encoder");
return {};
}
- prompt = "<|im_start|>system\nComprehend and analyze the provided prompt.<|im_end|>\n";
+ prompt = sd::prompt_templates::system_message(conditioner_params.system_prompt, "Comprehend and analyze the provided prompt.");
std::vector<int> system_tokens;
- if (!tokenizer->encode(prompt, system_tokens, nullptr)) {
+ if (!conditioner_params.raw && !prompt.empty() && !tokenizer->encode(prompt, system_tokens, nullptr)) {
return {};
}
prompt_template_encode_start_idx = static_cast<int>(system_tokens.size());
@@ -2847,15 +2829,13 @@ struct LLMEmbedder : public Conditioner {
prompt += "<|im_start|>user\n";
if (llm->enable_vision && conditioner_params.ref_images != nullptr) {
for (size_t i = 0; i < conditioner_params.ref_images->size(); ++i) {
- const auto& image = (*conditioner_params.ref_images)[i];
- int64_t width = image.shape()[0];
- int64_t height = image.shape()[1];
- int64_t pixels = width * height;
- if (width % 32 != 0 || height % 32 != 0) {
- LOG_ERROR("Qwen Image 2.1 reference dimensions must be multiples of 32");
- return {};
- }
- auto rgb = sd::Tensor<float>({width, height, 3, 1});
+ const auto& source = (*conditioner_params.ref_images)[i];
+ const auto resized = sd::fit_image_tensor(source, 32, "Qwen Image 2.1 reference encoding");
+ const auto& image = resized.empty() ? source : resized;
+ int64_t width = image.shape()[0];
+ int64_t height = image.shape()[1];
+ int64_t pixels = width * height;
+ auto rgb = sd::Tensor<float>({width, height, 3, 1});
for (int64_t p = 0; p < pixels; ++p) {
float alpha = image.shape()[2] == 4 ? image[p + 3 * pixels] : 1.f;
for (int c = 0; c < 3; ++c) {
@@ -2886,22 +2866,7 @@ struct LLMEmbedder : public Conditioner {
LOG_INFO("%s", sd_version_is_mage_flow(version) ? "MageFlowEditPipeline" : "QwenImageEditPlusPipeline");
prompt_template_encode_start_idx = 64;
- int min_pixels = conditioner_params.ref_image_params.vlm_min_size;
- if (min_pixels <= 0) {
- min_pixels = sd_version_is_mage_flow(version) ? -1 : 384;
- if (min_pixels > 0 && resize_mode == RefImageResizeMode::AREA) {
- min_pixels *= min_pixels;
- }
- }
- int max_pixels = conditioner_params.ref_image_params.vlm_max_size;
- if (max_pixels <= 0) {
- max_pixels = sd_version_is_mage_flow(version) ? 384 : 560;
- if (resize_mode == RefImageResizeMode::AREA) {
- max_pixels *= max_pixels;
- }
- }
-
- prompt = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n");
std::string placeholder = "<|image_pad|>";
std::string img_prompt;
@@ -2913,14 +2878,13 @@ struct LLMEmbedder : public Conditioner {
int h_bar = static_cast<int>(std::round(static_cast<double>(height) / factor) * factor);
int w_bar = static_cast<int>(std::round(static_cast<double>(width) / factor) * factor);
- resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
-
LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
auto resized_image = clip_preprocess(image, w_bar, h_bar);
auto image_embed = llm->encode_image(n_threads, resized_image, false);
- GGML_ASSERT(!image_embed.empty());
+ if (image_embed.empty())
+ return {};
std::string image_prefix = prompt + img_prompt + (sd_version_is_mage_flow(version) ? "Image " : "Picture ") + std::to_string(i + 1) + ": <|vision_start|>";
std::vector<int> prefix_tokens;
if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
@@ -2947,7 +2911,7 @@ struct LLMEmbedder : public Conditioner {
} else {
prompt_template_encode_start_idx = 34;
- prompt = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -2956,7 +2920,8 @@ struct LLMEmbedder : public Conditioner {
prompt += "<|im_end|>\n<|im_start|>assistant\n";
}
if (sd_version_is_mage_flow(version)) {
- max_length = 2048 + prompt_template_encode_start_idx;
+ max_length = 2048 + prompt_template_encode_start_idx;
+ max_length_includes_template = true;
}
} else if (sd_version_is_boogu_image(version)) {
prompt_template_encode_start_idx = 0;
@@ -2970,25 +2935,10 @@ struct LLMEmbedder : public Conditioner {
if (has_ref_images) {
LOG_INFO("BooguImageEditPipeline");
- const std::string prompt_prefix = "<|im_start|>system\n" + edit_system_prompt + "<|im_end|>\n<|im_start|>user\n";
+ const std::string prompt_prefix = system_prefix("<|im_start|>system\n" + edit_system_prompt + "<|im_end|>\n<|im_start|>user\n");
std::string img_prompt;
const std::string placeholder = "<|image_pad|>";
- int min_pixels = conditioner_params.ref_image_params.vlm_min_size;
- if (min_pixels <= 0) {
- min_pixels = 384;
- if (resize_mode == RefImageResizeMode::AREA) {
- min_pixels *= min_pixels;
- }
- }
- int max_pixels = conditioner_params.ref_image_params.vlm_max_size;
- if (max_pixels <= 0) {
- max_pixels = 384;
- if (resize_mode == RefImageResizeMode::AREA) {
- max_pixels *= max_pixels;
- }
- }
-
for (int i = 0; i < conditioner_params.ref_images->size(); i++) {
const auto& image = (*conditioner_params.ref_images)[i];
const int factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size;
@@ -3000,13 +2950,12 @@ struct LLMEmbedder : public Conditioner {
int w_bar = std::max(factor,
static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
- resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
-
LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
auto resized_image = clip_preprocess(image, w_bar, h_bar);
auto image_embed = llm->encode_image(n_threads, resized_image, false);
- GGML_ASSERT(!image_embed.empty());
+ if (image_embed.empty())
+ return {};
std::string image_prefix = prompt_prefix + img_prompt + "<|vision_start|>";
std::vector<int> prefix_tokens;
@@ -3031,7 +2980,7 @@ struct LLMEmbedder : public Conditioner {
prompt += "<|im_end|>\n";
} else {
const std::string& system_prompt = text_empty ? edit_system_prompt : t2i_system_prompt;
- prompt = "<|im_start|>system\n" + system_prompt + "<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\n" + system_prompt + "<|im_end|>\n<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
@@ -3041,25 +2990,10 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx = 34;
out_layers = {2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35};
- prompt = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n");
if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) {
std::string img_prompt = "";
const std::string placeholder = "<|image_pad|>";
- int min_pixels = conditioner_params.ref_image_params.vlm_min_size;
- if (min_pixels <= 0) {
- min_pixels = 384;
- if (resize_mode == RefImageResizeMode::AREA) {
- min_pixels *= min_pixels;
- }
- }
- int max_pixels = conditioner_params.ref_image_params.vlm_max_size;
- if (max_pixels <= 0) {
- max_pixels = 1024;
- if (resize_mode == RefImageResizeMode::AREA) {
- max_pixels *= max_pixels;
- }
- }
-
for (int i = 0; i < conditioner_params.ref_images->size(); i++) {
const auto& image = (*conditioner_params.ref_images)[i];
const int factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size;
@@ -3071,13 +3005,12 @@ struct LLMEmbedder : public Conditioner {
int w_bar = std::max(factor,
static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
- resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
-
LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
auto resized_image = clip_preprocess(image, w_bar, h_bar);
auto image_embed = llm->encode_image(n_threads, resized_image, false);
- GGML_ASSERT(!image_embed.empty());
+ if (image_embed.empty())
+ return {};
std::string image_prefix = prompt + img_prompt + "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
std::vector<int> prefix_tokens;
@@ -3110,22 +3043,7 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx = 67;
min_length = 512 + prompt_template_encode_start_idx;
- int min_pixels = conditioner_params.ref_image_params.vlm_min_size;
- if (min_pixels <= 0) {
- min_pixels = 384;
- if (resize_mode == RefImageResizeMode::AREA) {
- min_pixels *= min_pixels;
- }
- }
- int max_pixels = conditioner_params.ref_image_params.vlm_max_size;
- if (max_pixels <= 0) {
- max_pixels = 560;
- if (resize_mode == RefImageResizeMode::AREA) {
- max_pixels *= max_pixels;
- }
- }
-
- prompt = "<|im_start|>system\nAs an image editing expert, first analyze the content and attributes of the input image(s). Then, based on the user's editing instructions, clearly and precisely determine how to modify the given image(s), ensuring that only the specified parts are altered and all other aspects remain consistent with the original(s).<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\nAs an image editing expert, first analyze the content and attributes of the input image(s). Then, based on the user's editing instructions, clearly and precisely determine how to modify the given image(s), ensuring that only the specified parts are altered and all other aspects remain consistent with the original(s).<|im_end|>\n<|im_start|>user\n");
std::string placeholder = "<|image_pad|>";
std::string img_prompt;
@@ -3137,13 +3055,12 @@ struct LLMEmbedder : public Conditioner {
int h_bar = static_cast<int>(std::round(static_cast<double>(height) / factor) * factor);
int w_bar = static_cast<int>(std::round(static_cast<double>(width) / factor) * factor);
- resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
-
LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
auto resized_image = clip_preprocess(image, w_bar, h_bar);
auto image_embed = llm->encode_image(n_threads, resized_image, false);
- GGML_ASSERT(!image_embed.empty());
+ if (image_embed.empty())
+ return {};
std::string image_prefix = prompt + img_prompt + "<|vision_start|>";
std::vector<int> prefix_tokens;
if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
@@ -3165,7 +3082,7 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx = 36;
min_length = 512 + prompt_template_encode_start_idx;
- prompt = "<|im_start|>system\nAs an image captioning expert, generate a descriptive text prompt based on an image content, suitable for input to a text-to-image model.<|im_end|>\n<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>system\nAs an image captioning expert, generate a descriptive text prompt based on an image content, suitable for input to a text-to-image model.<|im_end|>\n<|im_start|>user\n");
}
prompt_attn_range.first = static_cast<int>(prompt.size());
@@ -3178,7 +3095,7 @@ struct LLMEmbedder : public Conditioner {
hidden_states_min_length = 512;
out_layers = {10, 20, 30};
- prompt = "[SYSTEM_PROMPT]You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation.[/SYSTEM_PROMPT][INST]";
+ prompt = system_prefix("[SYSTEM_PROMPT]You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation.[/SYSTEM_PROMPT][INST]");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -3189,15 +3106,16 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx = 0;
out_layers = {1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 36};
- prompt = "<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>user\n");
prompt += conditioner_params.text;
prompt += "<|im_end|>\n<|im_start|>assistant\n";
prompt_attn_range = {0, 0};
} else if (sd_version_is_ernie_image(version)) {
+ prompt = system_prefix("");
prompt_template_encode_start_idx = 0;
out_layers = {25}; // -2
- prompt_attn_range.first = 0;
+ prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
} else if (sd_version_is_lens(version)) {
@@ -3207,17 +3125,18 @@ struct LLMEmbedder : public Conditioner {
out_layers = {6, 12, 18, 24};
prompt =
- "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n"
- "Knowledge cutoff: 2024-06\n"
- "Current date: 2026-05-26\n" // fix for current date
- "\n"
- "Reasoning: medium\n"
- "\n"
- "# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions\n"
- "\n"
- "Describe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background.\n"
- "\n"
- "<|end|><|start|>user<|message|>";
+ system_prefix(
+ "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n"
+ "Knowledge cutoff: 2024-06\n"
+ "Current date: 2026-05-26\n" // fix for current date
+ "\n"
+ "Reasoning: medium\n"
+ "\n"
+ "# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions\n"
+ "\n"
+ "Describe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background.\n"
+ "\n"
+ "<|end|><|start|>user<|message|>");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -3233,14 +3152,14 @@ struct LLMEmbedder : public Conditioner {
}
if (conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) {
LOG_INFO("ZImageOmniPipeline");
- prompt = "<|im_start|>user\n<|vision_start|>";
+ prompt = system_prefix("<|im_start|>user\n<|vision_start|>");
for (int i = 0; i < conditioner_params.ref_images->size() - 1; i++) {
extra_prompts.push_back("<|vision_end|><|vision_start|>");
}
extra_prompts.push_back("<|vision_end|>" + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<|vision_start|>");
extra_prompts.push_back("<|vision_end|><|im_end|>");
} else {
- prompt = "<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -3251,13 +3170,13 @@ struct LLMEmbedder : public Conditioner {
} else if (version == VERSION_MOSS_IMAGE) {
prompt_template_encode_start_idx = 0;
max_length = 4096;
- prompt = "<|im_start|>user\n" + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<think>\n";
+ prompt = system_prefix("<|im_start|>user\n") + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<think>\n";
prompt_attn_range = {0, 0};
} else if (sd_version_is_flux3(version)) {
prompt_template_encode_start_idx = 0;
max_length = 4096;
out_layers = {4, 8, 12, 16, 20, 24, 28, 32};
- prompt = "<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
@@ -3267,7 +3186,7 @@ struct LLMEmbedder : public Conditioner {
min_length = 512;
out_layers = {9, 18, 27};
- prompt = "<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -3279,7 +3198,7 @@ struct LLMEmbedder : public Conditioner {
min_length = 1024;
out_layers = {9, 18, 27};
- prompt = "<|im_start|>user\n";
+ prompt = system_prefix("<|im_start|>user\n");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
@@ -3290,7 +3209,7 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx = 28;
min_length = prompt_template_encode_start_idx + 256;
- prompt = "<|im_start|>user\nDescribe the image by detailing the color, quantity, text, shape, size, texture, spatial relationships of the objects and background:";
+ prompt = system_prefix("<|im_start|>user\nDescribe the image by detailing the color, quantity, text, shape, size, texture, spatial relationships of the objects and background:");
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += " " + conditioner_params.text;
@@ -3298,6 +3217,7 @@ struct LLMEmbedder : public Conditioner {
prompt += "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n";
} else if (sd_version_is_pid(version)) {
+ prompt = system_prefix("");
constexpr int pixeldit_max_length = 300;
const std::string chi_prompt =
"Given a user prompt, generate an \"Enhanced prompt\" that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt:\n"
@@ -3308,17 +3228,21 @@ struct LLMEmbedder : public Conditioner {
"- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street lamps, a diverse crowd of people in colorful clothing, and a double-decker bus passing by towering glass skyscrapers.\n"
"Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:\n"
"User Prompt: ";
- auto chi_tokens = std::get<0>(tokenize(chi_prompt, {0, 0}));
- if (chi_tokens.empty()) {
+ auto chi_tokens = conditioner_params.raw ? std::vector<int>{} : std::get<0>(tokenize(chi_prompt, {0, 0}));
+ if (!conditioner_params.raw && chi_tokens.empty()) {
return {};
}
size_t num_chi_tokens = chi_tokens.size();
- max_length = (int)num_chi_tokens + pixeldit_max_length - 2;
+ max_length = conditioner_params.raw ? pixeldit_max_length : (int)num_chi_tokens + pixeldit_max_length - 2;
min_length = max_length;
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += " " + conditioner_params.text;
+ if (conditioner_params.raw)
+ prompt = conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
+ if (conditioner_params.raw)
+ prompt_attn_range = {0, 0};
auto hidden_states = encode_prompt(n_threads,
prompt,
@@ -3344,18 +3268,20 @@ struct LLMEmbedder : public Conditioner {
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
SDCondition result;
result.c_crossattn = std::move(hidden_states);
return result;
} else if (version == VERSION_IRIS) {
+ dynamic_template_crop = true;
prompt =
- "<|im_start|>system\n"
- "Describe the image by detailing the color, shape, size, texture, quantity, text, spatial "
- "relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
- auto prefix_tokens = std::get<0>(tokenize(prompt, {0, 0}));
- if (prefix_tokens.empty()) {
+ system_prefix(
+ "<|im_start|>system\n"
+ "Describe the image by detailing the color, shape, size, texture, quantity, text, spatial "
+ "relationships of the objects and background:<|im_end|>\n<|im_start|>user\n");
+ auto prefix_tokens = conditioner_params.raw ? std::vector<int>{} : std::get<0>(tokenize(prompt, {0, 0}));
+ if (!conditioner_params.raw && prefix_tokens.empty()) {
return {};
}
prompt_template_encode_start_idx = static_cast<int>(prefix_tokens.size());
@@ -3370,6 +3296,24 @@ struct LLMEmbedder : public Conditioner {
GGML_ABORT("unknown version %d", version);
}
+ if (conditioner_params.raw) {
+ prompt = conditioner_params.text;
+ prompt_suffix.clear();
+ prompt_attn_range = {0, 0};
+ spell_quotes = false;
+ if (min_length > 0)
+ min_length = std::max(0, min_length - prompt_template_encode_start_idx);
+ if (max_length_includes_template)
+ max_length -= prompt_template_encode_start_idx;
+ prompt_template_encode_start_idx = 0;
+ } else if (conditioner_params.system_prompt && !dynamic_template_crop && prompt_template_encode_start_idx > 0) {
+ prompt_template_encode_start_idx += template_token_delta;
+ if (min_length > 0)
+ min_length += template_token_delta;
+ if (max_length_includes_template)
+ max_length += template_token_delta;
+ }
+
LLM::PromptTokens prompt_tokens;
auto hidden_states = encode_prompt(n_threads,
prompt,
@@ -3425,7 +3369,8 @@ struct LLMEmbedder : public Conditioner {
input_ids,
sd::Tensor<float>(),
false);
- GGML_ASSERT(!byt5_hidden_states.empty());
+ if (byt5_hidden_states.empty())
+ return {};
extra_hidden_states_vec.push_back(std::move(byt5_hidden_states));
}
}
@@ -3447,7 +3392,7 @@ struct LLMEmbedder : public Conditioner {
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+ LOG_INFO("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
SDCondition result;
result.c_crossattn = std::move(hidden_states);
result.extra_c_crossattns = std::move(extra_hidden_states_vec);
@@ -3455,7 +3400,7 @@ struct LLMEmbedder : public Conditioner {
if (version == VERSION_QWEN_IMAGE_2_1) {
auto slots = sd::Tensor<int32_t>::zeros({result.c_crossattn.shape()[1]});
const auto& references = params.prompt_input == nullptr ? image_embeds : prompt_tokens.image_embeds;
- for (size_t i = 0; i < image_embeds.size(); ++i) {
+ for (size_t i = 0; i < references.size(); ++i) {
int64_t begin = references[i].first - prompt_template_encode_start_idx;
int64_t end = begin + references[i].second.shape()[1];
if (begin < 0 || end > slots.numel()) {
@@ -3486,9 +3431,11 @@ struct LLMEmbedder : public Conditioner {
};
struct MingImageEmbedder : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<MingImageTE::MingImageTextRunner> text_model;
- const std::string prefix = "text_encoders.llm";
+ const std::string prefix = "text_encoders";
MingImageEmbedder(ggml_backend_t backend, const String2TensorStorage& tensors, std::shared_ptr<RunnerWeightManager> weight_manager, const TokenizerConfig& tokenizers) {
if (!tokenizers.has(TokenizerConfig::MAIN)) {
@@ -3521,12 +3468,9 @@ struct MingImageEmbedder : public Conditioner {
LOG_ERROR("Ming-Image currently supports text-to-image only");
return {};
}
- std::string prompt =
- "<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|>"
- "<role>HUMAN</role>" +
- input.text + "<|role_end|><role>ASSISTANT</role>";
+ std::string prompt = sd::prompt_templates::ming_image(input.text, input.system_prompt, input.raw);
std::vector<int> tokens;
- if (!tokenizer->encode(prompt, tokens, nullptr)) {
+ if (!tokenizer->encode(prompt, tokens, nullptr) || tokens.empty()) {
return {};
}
if (tokens.size() + 258 > 32768) {
@@ -3619,6 +3563,9 @@ struct LTXAVTextProjectionRunner : public GGMLRunner {
// input, and the backbone's final hidden states are projected to the denoiser's caption dim.
// Ref: LLaDAImagePipeline._encode_text.
struct LLaDAImageEmbedder : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+ size_t max_reference_images() const override { return sigvq ? 1 : 0; }
+
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
std::shared_ptr<LLaDAImageTE::QueryFormerRunner> query_former;
@@ -3633,9 +3580,9 @@ struct LLaDAImageEmbedder : public Conditioner {
LLaDAImageEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
const std::string& llm_prefix = "text_encoders.llm",
- const std::string& query_former_prefix = "queryformer",
- const std::string& text_projection_prefix = "text_projection",
- const std::string& sigvq_prefix = "sigvq",
+ const std::string& query_former_prefix = "text_encoders.connector.queryformer",
+ const std::string& text_projection_prefix = "text_encoders.connector.text_projection",
+ const std::string& sigvq_prefix = "text_encoders.connector.sigvq",
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: llm_prefix(llm_prefix),
@@ -3748,19 +3695,10 @@ struct LLaDAImageEmbedder : public Conditioner {
return {};
}
- std::string text = conditioner_params.text;
- while (!text.empty() && std::isspace(static_cast<unsigned char>(text.front()))) {
- text.erase(text.begin());
- }
- while (!text.empty() && std::isspace(static_cast<unsigned char>(text.back()))) {
- text.pop_back();
- }
- std::string prompt = text.empty()
- ? "<role>HUMAN</role> Generate an image.\n<role>ASSISTANT</role>\n<IMAGE1>"
- : "<role>HUMAN</role> Generate an image: " + text + "\n<role>ASSISTANT</role>\n<IMAGE1>";
+ std::string prompt = sd::prompt_templates::llada_image(conditioner_params.text, conditioner_params.system_prompt, conditioner_params.raw);
std::vector<int> tokens;
- if (!tokenizer->encode(prompt, tokens, nullptr)) {
+ if (!tokenizer->encode(prompt, tokens, nullptr) || tokens.empty()) {
return {};
}
int64_t n_text = static_cast<int64_t>(tokens.size());
@@ -3821,6 +3759,8 @@ struct LLaDAImageEmbedder : public Conditioner {
};
struct LTXAVEmbedder : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+
static constexpr int64_t kHiddenSize = 3840;
static constexpr int64_t kNumStates = 49;
static constexpr int64_t kMinLength = 1024;
@@ -3828,6 +3768,7 @@ struct LTXAVEmbedder : public Conditioner {
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
std::shared_ptr<LTXAVTextProjectionRunner> projector;
+ std::shared_ptr<LTXV::LTXConditioningRunner> connectors;
std::string projector_prefix;
bool dual_projection = false;
@@ -3851,7 +3792,7 @@ struct LTXAVEmbedder : public Conditioner {
LTXAVEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
const std::string& llm_prefix = "text_encoders.llm",
- const std::string& projector_prefix = "text_embedding_projection",
+ const std::string& projector_prefix = "text_encoders.connector.text_embedding_projection",
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: projector_prefix(projector_prefix) {
@@ -3872,56 +3813,71 @@ struct LTXAVEmbedder : public Conditioner {
tensor_storage_map,
projector_prefix,
weight_manager);
+ connectors = std::make_shared<LTXV::LTXConditioningRunner>(backend, tensor_storage_map, weight_manager, projector_prefix);
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
llm->get_param_tensors(tensors, "text_encoders.llm");
projector->get_param_tensors(tensors, projector_prefix);
+ connectors->model.get_param_tensors(tensors, LTXV::LTXConditioningRunner::prefix);
}
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
llm->get_param_tensor_ops(tensor_ops);
+ projector->model.get_param_tensor_ops(tensor_ops);
+ connectors->model.get_param_tensor_ops(tensor_ops);
}
void set_flash_attention_enabled(bool enabled) override {
llm->set_flash_attention_enabled(enabled);
projector->set_flash_attention_enabled(enabled);
+ connectors->set_flash_attention_enabled(enabled);
}
void set_scale_overrides(float linear_scale, float attn_scale) override {
llm->set_scale_overrides(linear_scale, attn_scale);
projector->set_scale_overrides(linear_scale, attn_scale);
+ connectors->set_scale_overrides(linear_scale, attn_scale);
}
void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
llm->set_max_graph_vram_bytes(max_vram_bytes);
projector->set_max_graph_vram_bytes(max_vram_bytes);
+ connectors->set_max_graph_vram_bytes(max_vram_bytes);
}
void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
llm->set_runtime_backends(backends);
+ projector->set_runtime_backends(backends);
+ connectors->set_runtime_backends(backends);
}
void set_graph_cut_layer_split_enabled(bool enabled) override {
llm->set_graph_cut_layer_split_enabled(enabled);
+ projector->set_graph_cut_layer_split_enabled(enabled);
+ connectors->set_graph_cut_layer_split_enabled(enabled);
}
void set_graph_cut_layer_split_backend_vram_limits(const std::vector<size_t>& limits) override {
llm->set_graph_cut_layer_split_backend_vram_limits(limits);
+ projector->set_graph_cut_layer_split_backend_vram_limits(limits);
+ connectors->set_graph_cut_layer_split_backend_vram_limits(limits);
}
void get_layer_split_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
- llm->get_param_tensors(tensors, "text_encoders.llm");
+ get_param_tensors(tensors);
}
void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
llm->set_weight_adapter(adapter);
projector->set_weight_adapter(adapter);
+ connectors->set_weight_adapter(adapter);
}
void runner_end() override {
llm->runner_end();
projector->runner_end();
+ connectors->runner_end();
}
std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> tokenize(std::string text,
@@ -3975,13 +3931,14 @@ struct LTXAVEmbedder : public Conditioner {
if (!mask.empty()) {
const float mask_min = std::numeric_limits<float>::lowest() / 4.0f;
attention_mask = sd::Tensor<float>({static_cast<int64_t>(mask.size()), static_cast<int64_t>(mask.size())});
+ // Fully masked padding queries become all -inf after Flash Attention's F16 cast.
for (size_t i1 = 0; i1 < mask.size(); ++i1) {
for (size_t i0 = 0; i0 < mask.size(); ++i0) {
float value = 0.0f;
- if (mask[i0] == 0.0f) {
+ if (mask[i1] != 0.0f && mask[i0] == 0.0f) {
value += mask_min;
}
- if (i0 > i1) {
+ if (mask[i1] != 0.0f && i0 > i1) {
value += mask_min;
}
attention_mask[static_cast<int64_t>(i0 + mask.size() * i1)] = value;
@@ -3996,7 +3953,8 @@ struct LTXAVEmbedder : public Conditioner {
{},
true,
false);
- GGML_ASSERT(!hidden_states.empty());
+ if (hidden_states.empty())
+ return {};
hidden_states = apply_token_weights(std::move(hidden_states), weights);
int64_t valid_tokens = 0;
@@ -4061,22 +4019,29 @@ struct LTXAVEmbedder : public Conditioner {
const ConditionerParams& conditioner_params) override {
int64_t t0 = ggml_time_ms();
- std::string prompt;
+ std::string prompt = conditioner_params.raw ? std::string{} : sd::prompt_templates::with_system("", conditioner_params.system_prompt, sd::prompt_templates::RoleFormat::Gemma);
std::pair<int, int> prompt_attn_range;
prompt_attn_range.first = static_cast<int>(prompt.size());
prompt += conditioner_params.text;
prompt_attn_range.second = static_cast<int>(prompt.size());
+ if (conditioner_params.raw)
+ prompt_attn_range = {0, 0};
auto hidden_states = encode_prompt(n_threads, prompt, prompt_attn_range);
if (hidden_states.empty()) {
return {};
}
+ auto prepared = connectors->compute(n_threads, hidden_states);
+ if (prepared.first.empty() || prepared.second.empty())
+ return {};
+
int64_t t1 = ggml_time_ms();
LOG_VERBOSE("computing LTXAV condition graph completed, taking %" PRId64 " ms", t1 - t0);
SDCondition result;
- result.c_crossattn = std::move(hidden_states);
+ result.c_crossattn = std::move(prepared.first);
+ result.extra_c_crossattns.push_back(std::move(prepared.second));
return result;
}
};
diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h
index 58f539a..b07e768 100644
--- a/src/conditioning/conditioning_cache.h
+++ b/src/conditioning/conditioning_cache.h
@@ -47,16 +47,22 @@ class ConditioningCache {
});
}
+ static bool same_textual_inversion(const std::shared_ptr<sd::TextualInversion>& a,
+ const std::shared_ptr<sd::TextualInversion>& b) {
+ if (!a || !b)
+ return a == b;
+ return std::equal(a->sources.begin(), a->sources.end(), b->sources.begin(), b->sources.end(),
+ [](const ModelLoader::FileStamp& x, const ModelLoader::FileStamp& y) {
+ return std::tie(x.path, x.size, x.modified, x.exists) == std::tie(y.path, y.size, y.modified, y.exists);
+ });
+ }
+
static bool same_params(const ConditionerParams& a, const ConditionerParams& b) {
const auto fields = [](const ConditionerParams& p) {
- const auto& r = p.ref_image_params;
- return std::tie(p.text, p.clip_skip, p.width, p.height, p.zero_out_masked,
- r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode,
- r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels,
- r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target,
- r.ref_image_size);
+ return std::tie(p.text, p.system_prompt, p.raw, p.clip_skip, p.width, p.height, p.zero_out_masked);
};
if (fields(a) != fields(b) ||
+ !same_textual_inversion(a.textual_inversion, b.textual_inversion) ||
(a.prompt_input == nullptr) != (b.prompt_input == nullptr) ||
(a.ref_images == nullptr) != (b.ref_images == nullptr) ||
(a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) {
@@ -104,6 +110,11 @@ public:
}
for (auto it = entries_.begin(); it != entries_.end(); ++it) {
if (same_params(it->params, params)) {
+ if (it->params.textual_inversion) {
+ bool changed = false;
+ if (!ModelLoader::dependencies_changed(it->params.textual_inversion->dependencies, changed) || changed)
+ continue;
+ }
entries_.splice(entries_.begin(), entries_, it);
LOG_INFO("conditioning cache hit");
return entries_.front().condition;
diff --git a/src/conditioning/fibo2.hpp b/src/conditioning/fibo2.hpp
index db124ea..1c73008 100644
--- a/src/conditioning/fibo2.hpp
+++ b/src/conditioning/fibo2.hpp
@@ -4,6 +4,7 @@
#include "conditioning/conditioner.hpp"
struct Fibo2Conditioner : public LLMEmbedder {
+ size_t max_reference_images() const override { return std::numeric_limits<size_t>::max(); }
using JSON = nlohmann::ordered_json;
Fibo2Conditioner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<RunnerWeightManager> weight_manager, const TokenizerConfig& tokenizers)
@@ -143,7 +144,7 @@ struct Fibo2Conditioner : public LLMEmbedder {
LOG_ERROR("Fibo 2.0 editing requires vision weights");
return {};
}
- std::string prompt = "<|im_start|>user\n" + format_prompt(p.text, image_count) + "<|im_end|>\n<|im_start|>assistant\n";
+ std::string prompt = p.raw ? p.text : sd::prompt_templates::with_system("<|im_start|>user\n", p.system_prompt) + format_prompt(p.text, image_count) + "<|im_end|>\n<|im_start|>assistant\n";
std::vector<std::pair<int, sd::Tensor<float>>> images;
LLM::DeepStackImageEmbeds deepstack;
std::vector<LLM::ImageGrid> grids;
@@ -152,12 +153,8 @@ struct Fibo2Conditioner : public LLMEmbedder {
const auto& image = (*p.ref_images)[i];
int factor = llm->config.vision.patch_size * llm->config.vision.spatial_merge_size;
int height = static_cast<int>(image.shape()[1]), width = static_cast<int>(image.shape()[0]);
- int h = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor)) * factor);
- int w = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
- resize_image_dims(height, width, h, w, factor,
- p.ref_image_params.vlm_min_size,
- p.ref_image_params.vlm_max_size,
- p.ref_image_params.vlm_resize_mode);
+ int h = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor)) * factor);
+ int w = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
auto pixels = sd::ops::interpolate(image, {w, h, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Bicubic, false, true);
auto rgb = sd::Tensor<float>({w, h, 3, 1});
for (int64_t pixel = 0; pixel < int64_t(w) * h; ++pixel) {
diff --git a/src/conditioning/photomaker_prompt.h b/src/conditioning/photomaker_prompt.h
new file mode 100644
index 0000000..2cd49dc
--- /dev/null
+++ b/src/conditioning/photomaker_prompt.h
@@ -0,0 +1,128 @@
+#ifndef __SD_PHOTOMAKER_PROMPT_H__
+#define __SD_PHOTOMAKER_PROMPT_H__
+
+#include <algorithm>
+#include <tuple>
+#include <utility>
+
+#include "conditioning/conditioner.hpp"
+
+namespace sd::photomaker {
+ inline std::tuple<std::vector<int>, std::vector<float>, std::vector<bool>>
+ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
+ const std::string& text,
+ int trigger_token_count,
+ int32_t image_token) {
+ auto tokens_and_weights = clip_conditioner.tokenize(text);
+ std::vector<int> source_tokens = std::move(tokens_and_weights.first);
+ std::vector<float> source_weights = std::move(tokens_and_weights.second);
+ if (source_tokens.empty()) {
+ return {};
+ }
+
+ if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer->BOS_TOKEN_ID) {
+ source_tokens.erase(source_tokens.begin());
+ source_weights.erase(source_weights.begin());
+ }
+ if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer->EOS_TOKEN_ID) {
+ source_tokens.pop_back();
+ source_weights.pop_back();
+ }
+
+ std::vector<int> tokens;
+ std::vector<float> weights;
+ int32_t class_idx = -1;
+ for (size_t i = 0; i < source_tokens.size(); i++) {
+ int token = source_tokens[i];
+ if (token == image_token) {
+ if (!tokens.empty()) {
+ class_idx = static_cast<int32_t>(tokens.size()) - 1;
+ int class_token = tokens.back();
+ float class_weight = weights.back();
+ for (int j = 1; j < trigger_token_count; j++) {
+ tokens.push_back(class_token);
+ weights.push_back(class_weight);
+ }
+ }
+ continue;
+ }
+ tokens.push_back(token);
+ weights.push_back(source_weights[i]);
+ }
+
+ clip_conditioner.tokenizer->pad_tokens(tokens,
+ &weights,
+ nullptr,
+ clip_conditioner.text_model->model.n_token,
+ clip_conditioner.text_model->model.n_token,
+ true);
+ std::vector<bool> class_token_mask;
+ for (int i = 0; i < tokens.size(); i++) {
+ class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
+ }
+
+ return std::make_tuple(tokens, weights, class_token_mask);
+ }
+
+ inline std::tuple<SDCondition, std::vector<bool>>
+ encode_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
+ int n_threads,
+ const ConditionerParams& conditioner_params,
+ const std::string& trigger_word,
+ int trigger_token_count) {
+ std::vector<int> image_tokens;
+ if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
+ return {};
+ }
+ if (image_tokens.size() != 1) {
+ LOG_ERROR("PhotoMaker trigger word must encode to one token");
+ return {};
+ }
+ auto tokens_and_weights = tokenize_photomaker_trigger(clip_conditioner,
+ conditioner_params.text,
+ trigger_token_count,
+ image_tokens[0]);
+ std::vector<int>& tokens = std::get<0>(tokens_and_weights);
+ std::vector<float>& weights = std::get<1>(tokens_and_weights);
+ std::vector<bool>& trigger_mask = std::get<2>(tokens_and_weights);
+ if (tokens.empty()) {
+ return {};
+ }
+ auto cond = clip_conditioner.get_learned_condition_common(n_threads,
+ tokens,
+ weights,
+ conditioner_params.clip_skip,
+ conditioner_params.width,
+ conditioner_params.height,
+ conditioner_params.zero_out_masked);
+ return std::make_tuple(std::move(cond), trigger_mask);
+ }
+
+ inline bool remove_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
+ const std::string& prompt,
+ const std::string& trigger_word,
+ std::string& result) {
+ std::vector<int> image_tokens;
+ if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
+ return false;
+ }
+ if (image_tokens.size() != 1) {
+ LOG_ERROR("PhotoMaker trigger word must encode to one token");
+ return false;
+ }
+ auto tokens_and_weights = clip_conditioner.tokenize(prompt);
+ std::vector<int>& tokens = tokens_and_weights.first;
+ if (tokens.empty()) {
+ return false;
+ }
+ auto it = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
+ if (it == tokens.end()) {
+ LOG_ERROR("PhotoMaker trigger word was not found in tokenized prompt");
+ return false;
+ }
+ tokens.erase(it);
+ return clip_conditioner.decode(tokens, result);
+ }
+} // namespace sd::photomaker
+
+#endif // __SD_PHOTOMAKER_PROMPT_H__
diff --git a/src/conditioning/prompt_input.cpp b/src/conditioning/prompt_input.cpp
index 0748e9c..cb323bb 100644
--- a/src/conditioning/prompt_input.cpp
+++ b/src/conditioning/prompt_input.cpp
@@ -24,7 +24,7 @@ namespace sd {
std::string marker;
do {
if (codepoint > 0xffffd) {
- throw std::invalid_argument("Too many prompt image markers");
+ throw std::invalid_argument("Too many prompt markers");
}
marker = {static_cast<char>(0xf0 | (codepoint >> 18)),
static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f)),
@@ -34,7 +34,7 @@ namespace sd {
} while (std::any_of(input->begin(), input->end(), [&](const PromptPart& source) {
return source.content.find(marker) != std::string::npos;
}));
- image_markers_.emplace(marker, i);
+ markers_.emplace(marker, i);
text += marker;
}
}
@@ -48,9 +48,9 @@ namespace sd {
if (begin == std::string::npos) {
break;
}
- const auto marker = image_markers_.find(formatted_text.substr(begin, 4));
+ const auto marker = markers_.find(formatted_text.substr(begin, 4));
search = begin + 1;
- if (marker == image_markers_.end()) {
+ if (marker == markers_.end()) {
continue;
}
if (begin > cursor) {
@@ -66,12 +66,17 @@ namespace sd {
return result;
}
- PromptInput parse_prompt_input(const std::string& text) {
+ PromptInput parse_prompt_input(const std::string& text, bool allow_embeddings) {
PromptInput parts;
std::string plain_text;
size_t cursor = 0;
while (cursor < text.size()) {
- const size_t begin = text.find("<image:", cursor);
+ const size_t image = text.find("<image:", cursor);
+ const size_t embedding = allow_embeddings ? text.find("<embedding:", cursor) : std::string::npos;
+ const size_t begin = std::min(image, embedding);
+ const bool is_embedding = embedding != std::string::npos && begin == embedding;
+ const std::string marker = is_embedding ? "<embedding:" : "<image:";
+ const size_t path_begin = begin + marker.size();
if (begin == std::string::npos) {
plain_text += text.substr(cursor);
break;
@@ -82,20 +87,20 @@ namespace sd {
}
if (slashes % 2 != 0) {
plain_text += text.substr(cursor, begin - cursor - 1);
- plain_text += "<image:";
- cursor = begin + 7;
+ plain_text += marker;
+ cursor = path_begin;
continue;
}
- const size_t end = text.find('>', begin + 7);
- if (end == std::string::npos || end == begin + 7) {
- throw std::invalid_argument("Invalid <image:...> prompt marker at byte " + std::to_string(begin));
+ const size_t end = text.find('>', path_begin);
+ if (end == std::string::npos || end == path_begin) {
+ throw std::invalid_argument("Invalid " + marker + "...> prompt marker at byte " + std::to_string(begin));
}
plain_text += text.substr(cursor, begin - cursor);
if (!plain_text.empty()) {
parts.push_back({PromptPart::Type::TEXT, std::move(plain_text), {}});
plain_text.clear();
}
- parts.push_back({PromptPart::Type::IMAGE, text.substr(begin + 7, end - begin - 7), {}});
+ parts.push_back({is_embedding ? PromptPart::Type::EMBEDDING : PromptPart::Type::IMAGE, text.substr(path_begin, end - path_begin), {}});
cursor = end + 1;
}
if (!plain_text.empty() || parts.empty()) {
diff --git a/src/conditioning/prompt_input.h b/src/conditioning/prompt_input.h
index ed5fa2d..fdf48d9 100644
--- a/src/conditioning/prompt_input.h
+++ b/src/conditioning/prompt_input.h
@@ -13,7 +13,8 @@ namespace sd {
struct PromptPart {
enum class Type { TEXT,
- IMAGE };
+ IMAGE,
+ EMBEDDING };
Type type = Type::TEXT;
std::string content;
@@ -26,7 +27,7 @@ namespace sd {
class PromptInputTemplate {
const PromptInput* input_;
- std::map<std::string, size_t> image_markers_;
+ std::map<std::string, size_t> markers_;
public:
std::string text;
@@ -35,7 +36,7 @@ namespace sd {
PromptInput expand(const std::string& formatted_text, float weight = 1.f) const;
};
- PromptInput parse_prompt_input(const std::string& text);
+ PromptInput parse_prompt_input(const std::string& text, bool allow_embeddings = false);
bool prompt_has_images(const PromptInput& input);
std::string prompt_input_text(const PromptInput& input);
bool resolve_prompt_images(PromptInput& input, const PromptImageLoader& loader);
diff --git a/src/conditioning/prompt_templates.h b/src/conditioning/prompt_templates.h
new file mode 100644
index 0000000..7de8b34
--- /dev/null
+++ b/src/conditioning/prompt_templates.h
@@ -0,0 +1,91 @@
+#ifndef __SD_CONDITIONING_PROMPT_TEMPLATES_H__
+#define __SD_CONDITIONING_PROMPT_TEMPLATES_H__
+
+#include <optional>
+#include <string>
+
+namespace sd::prompt_templates {
+ enum class RoleFormat { ChatML,
+ Mistral,
+ GPTOSS,
+ Role,
+ Gemma };
+
+ inline std::string with_system(const std::string& prefix, const std::optional<std::string>& system, RoleFormat format = RoleFormat::ChatML) {
+ if (!system)
+ return prefix;
+ const char* open = "<|im_start|>system\n";
+ const char* close = "<|im_end|>\n";
+ switch (format) {
+ case RoleFormat::Mistral:
+ open = "[SYSTEM_PROMPT]";
+ close = "[/SYSTEM_PROMPT]";
+ break;
+ case RoleFormat::GPTOSS:
+ open = "<|start|>system<|message|>";
+ close = "<|end|>";
+ break;
+ case RoleFormat::Role:
+ open = "<role>SYSTEM</role>";
+ close = "<|role_end|>";
+ break;
+ case RoleFormat::Gemma:
+ open = "<start_of_turn>system\n";
+ close = "<end_of_turn>\n";
+ break;
+ case RoleFormat::ChatML:
+ break;
+ }
+ const std::string begin(open), end(close);
+ const auto start = prefix.find(begin);
+ const auto finish = start == std::string::npos ? std::string::npos : prefix.find(end, start + begin.size());
+ const auto segment = system->empty() ? std::string{} : begin + *system + end;
+ if (finish == std::string::npos)
+ return segment + prefix;
+ return prefix.substr(0, start) + segment + prefix.substr(finish + end.size());
+ }
+
+ inline std::string ming_image(const std::string& text, const std::optional<std::string>& system, bool raw) {
+ if (raw)
+ return text;
+ return with_system("<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|><role>HUMAN</role>", system, RoleFormat::Role) +
+ text + "<|role_end|><role>ASSISTANT</role>";
+ }
+
+ inline std::string llada_image(std::string text, const std::optional<std::string>& system, bool raw) {
+ if (raw)
+ return text;
+ const auto first = text.find_first_not_of(" \t\n\r\f\v");
+ const auto last = text.find_last_not_of(" \t\n\r\f\v");
+ text = first == std::string::npos ? std::string{} : text.substr(first, last - first + 1);
+ auto prompt = text.empty() ? "<role>HUMAN</role> Generate an image.\n<role>ASSISTANT</role>\n<IMAGE1>"
+ : "<role>HUMAN</role> Generate an image: " + text + "\n<role>ASSISTANT</role>\n<IMAGE1>";
+ return with_system("", system, RoleFormat::Role) + prompt;
+ }
+
+ inline std::string system_message(const std::optional<std::string>& system, const std::string& fallback) {
+ const auto& text = system ? *system : fallback;
+ return text.empty() ? std::string{} : "<|im_start|>system\n" + text + "<|im_end|>\n";
+ }
+
+ inline std::string sensenova_u1(const std::string& text, const std::optional<std::string>& system) {
+ static const std::string kSystemMessage =
+ "You are an image generation and editing assistant that accurately understands and executes user intent.\n\n"
+ "You support two modes:\n\n1. Think Mode:\nIf the task requires reasoning, you MUST start with a "
+ "<think></think> block. Put all reasoning inside the block using plain text. DO NOT include any image tags. "
+ "Keep it reasonable and directly useful for producing the final image.\n\n2. Non-Think Mode:\nIf no reasoning "
+ "is needed, directly produce the final image.\n\nTask Types:\n\nA. Text-to-Image Generation:\n- Generate a "
+ "high-quality image based on the user's description.\n- Ensure visual clarity, semantic consistency, and "
+ "completeness.\n- DO NOT introduce elements that contradict or override the user's intent.\n\nB. Image Editing:\n"
+ "- Use the provided image(s) as input or reference for modification or transformation.\n- The result can be an "
+ "edited image or a new image based on the reference(s).\n- Preserve all unspecified attributes unless explicitly "
+ "changed.\n\nGeneral Rules:\n- For any visible text in the image, follow the language specified for the rendered "
+ "text in the user's description, not the language of the prompt. If no language is specified, use the user's input "
+ "language.";
+
+ return system_message(system, kSystemMessage) + "<|im_start|>user\n" + text +
+ "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n<img>";
+ }
+}
+
+#endif // __SD_CONDITIONING_PROMPT_TEMPLATES_H__
diff --git a/src/conditioning/textual_inversion.cpp b/src/conditioning/textual_inversion.cpp
new file mode 100644
index 0000000..77c5429
--- /dev/null
+++ b/src/conditioning/textual_inversion.cpp
@@ -0,0 +1,80 @@
+#include "conditioning/textual_inversion.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstring>
+#include <memory>
+#include <stdexcept>
+
+#include "core/util.h"
+
+namespace sd {
+ bool read_textual_inversion(ModelLoader& loader, ggml_context* ctx, int64_t hidden_size, int64_t hidden_size2, ggml_type embd_type, ggml_type embd2_type, ggml_tensor*& embd, ggml_tensor*& embd2) {
+ auto on_load = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) {
+ if (tensor_storage.ne[0] == hidden_size) {
+ embd = ggml_new_tensor_2d(ctx, embd_type, hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
+ if (embd == nullptr) {
+ return false;
+ }
+ *dst_tensor = embd;
+ } else if (hidden_size2 > 0) {
+ if (tensor_storage.ne[0] == hidden_size2) {
+ embd2 = ggml_new_tensor_2d(ctx, embd2_type, hidden_size2, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
+ if (embd2 == nullptr) {
+ return false;
+ }
+ *dst_tensor = embd2;
+ } else {
+ LOG_VERBOSE("embedding skipped, wrong hidden size, got %" PRId64 ", expected %" PRId64 " or %" PRId64, tensor_storage.ne[0], hidden_size, hidden_size2);
+ }
+ } else {
+ LOG_VERBOSE("embedding skipped, wrong hidden size, got %" PRId64 ", expected %" PRId64, tensor_storage.ne[0], hidden_size);
+ }
+ return true;
+ };
+ loader.set_n_threads(1);
+ return loader.load_tensors(on_load);
+ }
+
+ void TextualInversion::load(int64_t hidden_size, int64_t hidden_size2) {
+ std::vector<float> values, values2;
+ int count = 0;
+ dependencies.clear();
+ token_ranges.clear();
+ for (const auto& source : sources) {
+ ModelLoader loader;
+ if (!loader.init_from_file_and_convert_name(source.path))
+ throw std::runtime_error("cannot read textual inversion '" + source.path + "'");
+ ggml_init_params params{100 * 1024 * 1024, nullptr, false};
+ std::unique_ptr<ggml_context, decltype(&ggml_free)> ctx(ggml_init(params), &ggml_free);
+ if (!ctx)
+ throw std::runtime_error("cannot allocate textual inversion buffers");
+ ggml_tensor* embd = nullptr;
+ ggml_tensor* embd2 = nullptr;
+ if (!read_textual_inversion(loader, ctx.get(), hidden_size, hidden_size2,
+ GGML_TYPE_F32, GGML_TYPE_F32, embd, embd2) ||
+ (!embd && !embd2))
+ throw std::runtime_error("textual inversion '" + source.path + "' has no usable tensor for this CLIP encoder");
+ const int64_t rows = embd ? embd->ne[1] : 0;
+ const int64_t rows2 = embd2 ? embd2->ne[1] : 0;
+ const int64_t total = std::max(rows, rows2);
+ if (hidden_size2 && rows != rows2)
+ LOG_WARN("textual inversion '%s' has unequal CLIP rows, zero-padding", source.path.c_str());
+ values.resize((count + total) * hidden_size);
+ if (embd)
+ memcpy(values.data() + count * hidden_size, embd->data, rows * hidden_size * sizeof(float));
+ if (hidden_size2) {
+ values2.resize((count + total) * hidden_size2);
+ if (embd2)
+ memcpy(values2.data() + count * hidden_size2, embd2->data, rows2 * hidden_size2 * sizeof(float));
+ }
+ token_ranges.emplace(source.path, std::pair<int, int>{count, count + total});
+ count += int(total);
+ auto files = loader.dependencies();
+ dependencies.insert(dependencies.end(), files.begin(), files.end());
+ }
+ first = Tensor<float>({hidden_size, count}, std::move(values));
+ if (hidden_size2)
+ second = Tensor<float>({hidden_size2, count}, std::move(values2));
+ }
+}
diff --git a/src/conditioning/textual_inversion.h b/src/conditioning/textual_inversion.h
new file mode 100644
index 0000000..9e19fd3
--- /dev/null
+++ b/src/conditioning/textual_inversion.h
@@ -0,0 +1,21 @@
+#ifndef __SD_CONDITIONING_TEXTUAL_INVERSION_H__
+#define __SD_CONDITIONING_TEXTUAL_INVERSION_H__
+
+#include "core/tensor.hpp"
+#include "model_loader.h"
+
+namespace sd {
+ bool read_textual_inversion(ModelLoader& loader, ggml_context* ctx, int64_t hidden_size, int64_t hidden_size2, ggml_type type, ggml_type type2, ggml_tensor*& first, ggml_tensor*& second);
+
+ struct TextualInversion {
+ ModelLoader::Dependencies sources;
+ ModelLoader::Dependencies dependencies;
+ Tensor<float> first;
+ Tensor<float> second;
+ std::map<std::string, std::pair<int, int>> token_ranges;
+
+ void load(int64_t hidden_size, int64_t hidden_size2);
+ };
+}
+
+#endif // __SD_CONDITIONING_TEXTUAL_INVERSION_H__
diff --git a/src/config.cpp b/src/config.cpp
index 5cffb85..a7f3add 100644
--- a/src/config.cpp
+++ b/src/config.cpp
@@ -8,14 +8,8 @@ namespace sd {
guidance.txt_cfg = 7.0f;
guidance.img_cfg = INFINITY;
guidance.distilled_guidance = 3.5f;
- guidance.slg.layers = nullptr;
- guidance.slg.layer_count = 0;
- guidance.slg.layer_start = 0.01f;
- guidance.slg.layer_end = 0.2f;
- guidance.slg.scale = 0.f;
scheduler = SCHEDULER_COUNT;
sample_method = SAMPLE_METHOD_COUNT;
- shifted_timestep = 0;
sample_steps = 20;
eta = INFINITY;
custom_sigmas = nullptr;
@@ -51,20 +45,6 @@ namespace sd {
spectrum_stop_percent = 0.9f;
}
- HiresConfig::HiresConfig() {
- enabled = false;
- upscaler = SD_HIRES_UPSCALER_LATENT;
- model_path = nullptr;
- scale = 2.f;
- target_width = 0;
- target_height = 0;
- steps = 0;
- denoising_strength = 0.7f;
- upscale_tile_size = 128;
- custom_sigmas = nullptr;
- custom_sigmas_count = 0;
- }
-
ContextConfig::ContextConfig() {
n_threads = sd_get_num_physical_cores();
conditioning_cache_size = 4;
@@ -92,42 +72,35 @@ namespace sd {
auto_fit = true;
rpc_servers = nullptr;
model_args = nullptr;
- pulid_weights_path = nullptr;
}
ImageConfig::ImageConfig() {
- clip_skip = -1;
- ref_images_count = 0;
- ref_image_args = "";
- width = 512;
- height = 512;
- strength = 0.75f;
- seed = -1;
- batch_count = 1;
- control_strength = 0.9f;
- ip_adapter_strength = 1.0f;
- qwen_image_layers = 3;
- circular_x = false;
- circular_y = false;
- pm_params = {nullptr, 0, nullptr, 20.f};
- pulid_params = {nullptr, 1.0f};
- vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
+ clip_skip = -1;
+ ref_images_count = 0;
+ width = 512;
+ height = 512;
+ strength = 0.75f;
+ seed = -1;
+ batch_count = 1;
+ control_strength = 0.9f;
+ qwen_image_layers = 3;
+ circular_x = false;
+ circular_y = false;
+ vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
}
VideoConfig::VideoConfig() {
- high_noise_sample_params.sample_steps = -1;
- width = 512;
- height = 512;
- strength = 0.75f;
- seed = -1;
- video_frames = 6;
- fps = 16;
- moe_boundary = 0.875f;
- vace_strength = 1.f;
- vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
- circular_x = false;
- circular_y = false;
- ref_image_args = "";
+ width = 512;
+ height = 512;
+ strength = 0.75f;
+ seed = -1;
+ video_frames = 6;
+ fps = 16;
+ moe_boundary = 0.875f;
+ vace_strength = 1.f;
+ vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
+ circular_x = false;
+ circular_y = false;
}
} // namespace sd
diff --git a/src/config.h b/src/config.h
index 7331ccc..5319863 100644
--- a/src/config.h
+++ b/src/config.h
@@ -20,24 +20,10 @@ namespace sd {
const char* extra_tiling_args;
};
- struct ImagePreprocessConfig {
- // Semicolon-separated target=...,key=value rules. NULL preserves defaults.
- const char* rules;
- };
-
- struct SLGConfig {
- int* layers;
- size_t layer_count;
- float layer_start;
- float layer_end;
- float scale;
- };
-
struct GuidanceConfig {
float txt_cfg;
float img_cfg;
float distilled_guidance;
- sd::SLGConfig slg;
};
struct SampleConfig {
@@ -46,7 +32,6 @@ namespace sd {
enum sample_method_t sample_method;
int sample_steps;
float eta;
- int shifted_timestep;
float* custom_sigmas;
int custom_sigmas_count;
float flow_shift;
@@ -55,18 +40,6 @@ namespace sd {
SampleConfig();
};
- struct PhotoMakerConfig {
- sd::Image* id_images;
- int id_images_count;
- const char* id_embed_path;
- float style_strength;
- };
-
- struct PuLIDConfig {
- const char* id_embedding_path;
- float id_weight;
- };
-
struct CacheConfig {
enum sd_cache_mode_t mode;
float reuse_threshold;
@@ -96,22 +69,6 @@ namespace sd {
CacheConfig();
};
- struct HiresConfig {
- bool enabled;
- enum sd_hires_upscaler_t upscaler;
- const char* model_path;
- float scale;
- int target_width;
- int target_height;
- int steps;
- float denoising_strength;
- int upscale_tile_size;
- float* custom_sigmas;
- int custom_sigmas_count;
-
- HiresConfig();
- };
-
struct ContextConfig {
const char* model_path{};
const char* clip_l_path{};
@@ -129,12 +86,8 @@ namespace sd {
const char* audio_encoder_path{};
const char* taesd_path{};
const char* control_net_path{};
- const char* ip_adapter_path{};
- const char* motion_module_path{};
const sd_embedding_t* embeddings{};
uint32_t embedding_count{};
- const char* photo_maker_path{};
- const char* pulid_weights_path{};
const char* tensor_type_rules{};
int n_threads{};
enum sd_type_t wtype {};
@@ -150,7 +103,8 @@ namespace sd {
bool vae_conv_direct{};
bool force_sdxl_vae_conv_scale{};
enum sd_vae_format_t vae_format {};
- const char* max_vram{}; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget
+ const char* max_vram{}; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget
+ const char* max_ram{};
bool disable_prefetch{}; // Disable asynchronous next-segment weight prefetch
bool eager_load{}; // Load all params into the params backend at model-load time instead of lazily on first use
const char* backend{};
@@ -172,6 +126,20 @@ namespace sd {
ContextConfig();
};
+ struct HiresConfig {
+ bool enabled = false;
+ const char* upscaler = "latent";
+ const char* model_path{};
+ float scale = 2.f;
+ int target_width{};
+ int target_height{};
+ int steps{};
+ float denoising_strength = 0.7f;
+ int upscale_tile_size = 128;
+ float* custom_sigmas{};
+ int custom_sigmas_count{};
+ };
+
struct ImageConfig {
const sd_lora_t* loras{};
uint32_t lora_count{};
@@ -181,7 +149,6 @@ namespace sd {
sd::Image init_image{};
sd::Image* ref_images{};
int ref_images_count{};
- const char* ref_image_args{};
sd::Image mask_image{};
int width{};
int height{};
@@ -191,17 +158,12 @@ namespace sd {
int batch_count{};
sd::Image control_image{};
float control_strength{};
- sd::Image ip_adapter_image{};
- float ip_adapter_strength{};
- sd::PhotoMakerConfig pm_params{};
- sd::PuLIDConfig pulid_params{};
sd::TilingConfig vae_tiling_params{};
sd::CacheConfig cache{};
- sd::HiresConfig hires{};
+ HiresConfig hires{};
int qwen_image_layers{};
bool circular_x{};
bool circular_y{};
- sd::ImagePreprocessConfig image_preprocess{};
enum sd_control_type_t control_type {};
ImageConfig();
@@ -226,8 +188,8 @@ namespace sd {
int width{};
int height{};
sd::SampleConfig sample_params{};
- sd::SampleConfig high_noise_sample_params{};
float moe_boundary{};
+ int high_noise_steps = -1;
float strength{};
int64_t seed{};
int video_frames{};
@@ -235,52 +197,18 @@ namespace sd {
float vace_strength{};
sd::TilingConfig vae_tiling_params{};
sd::CacheConfig cache{};
- sd::HiresConfig hires{};
+ HiresConfig hires{};
bool circular_x{};
bool circular_y{};
- sd::ImagePreprocessConfig image_preprocess{};
- const char* ref_image_args{};
VideoConfig();
};
- struct VideoUpscalerConfig {
- const char* diffusion_model_path{};
- const char* vae_path{};
- // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints.
- int upscale_factor{};
- int n_threads{};
- const char* backend{};
- const char* params_backend{};
- // auto/on/off or diffusion/vae assignments; null selects auto.
- const char* flash_attn_mode{};
- const char* conv_direct_mode{};
- };
-
- struct VideoUpscaleConfig {
- const sd::Image* frames{};
- int frame_count{};
- int fps = 24;
- int64_t seed = 42;
- // RelayVSR reference window, including both endpoints; defaults to 16.
- int window_size = 16;
- // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5.
- int temporal_chunk_size = 5;
- };
-
- struct ADetailConfig {
- const char* prompt{};
- const char* negative_prompt{};
- const char* extra_ad_args{};
- };
-
struct PostprocessConfig {
sd::Image image{};
bool conv_direct = false;
int tile_size = 128;
- VideoUpscalerConfig video_upscaler;
- VideoUpscaleConfig video_upscale;
- ADetailConfig adetail;
+ int repeats = 1;
};
struct OptConfig {
diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp
index 123894d..9df9c61 100644
--- a/src/core/backend_fit.cpp
+++ b/src/core/backend_fit.cpp
@@ -21,6 +21,7 @@
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "ggml-backend.h"
+#include "name_conversion.h"
namespace sd::backend_fit {
@@ -75,7 +76,11 @@ namespace sd::backend_fit {
static bool classify_tensor(const std::string& name, ComponentKind& out) {
auto contains = [&](const char* s) { return name.find(s) != std::string::npos; };
- if (contains("model.diffusion_model.") || contains("unet.")) {
+ if (is_cond_stage_model_name(name)) {
+ out = ComponentKind::CONDITIONER;
+ return true;
+ }
+ if (contains("model.diffusion_model.") || contains("unet.") || name.rfind("diffusion.", 0) == 0) {
out = ComponentKind::DIT;
return true;
}
@@ -90,7 +95,6 @@ namespace sd::backend_fit {
contains("te.text_model.") ||
contains("conditioner") ||
name.rfind("text_encoder.", 0) == 0 ||
- name.rfind("text_embedding_projection.", 0) == 0 ||
contains(".aggregate_embed.")) {
out = ComponentKind::CONDITIONER;
return true;
@@ -98,13 +102,13 @@ namespace sd::backend_fit {
return false;
}
- static std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype) {
+ static std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype, const char* standalone_name = nullptr) {
int64_t bytes[3] = {0, 0, 0};
int64_t largest_tensor[3] = {0, 0, 0};
for (const auto& [name, stored_tensor] : loader.get_tensor_storage_map()) {
- TensorStorage ts = stored_tensor;
- ComponentKind kind;
- if (!classify_tensor(ts.name, kind)) {
+ TensorStorage ts = stored_tensor;
+ ComponentKind kind = ComponentKind::DIT;
+ if (!standalone_name && !classify_tensor(ts.name, kind)) {
continue;
}
if (ts.expected_type != GGML_TYPE_COUNT) {
@@ -118,7 +122,7 @@ namespace sd::backend_fit {
}
return {
- {ComponentKind::DIT, "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, largest_tensor[int(ComponentKind::DIT)]},
+ {ComponentKind::DIT, standalone_name ? standalone_name : "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, largest_tensor[int(ComponentKind::DIT)]},
{ComponentKind::CONDITIONER, "Conditioner", bytes[int(ComponentKind::CONDITIONER)], 2048 * MiB, largest_tensor[int(ComponentKind::CONDITIONER)]},
{ComponentKind::VAE, "VAE", bytes[int(ComponentKind::VAE)], 1024 * MiB, largest_tensor[int(ComponentKind::VAE)]},
};
@@ -130,7 +134,8 @@ namespace sd::backend_fit {
}
static std::vector<Device> enumerate_gpu_devices(const sd::ggml_graph_cut::MaxVramAssignment& budgets,
- bool include_other_devices) {
+ bool include_other_devices,
+ const AvailableVRAM& available_vram) {
std::vector<Device> out;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
@@ -145,6 +150,8 @@ namespace sd::backend_fit {
device.type = type;
size_t free_bytes = 0, total_bytes = 0;
ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
+ if (available_vram)
+ free_bytes = available_vram(dev, free_bytes, total_bytes);
device.free_bytes = (int64_t)free_bytes;
float gib = budgets.default_gib;
@@ -164,12 +171,13 @@ namespace sd::backend_fit {
return out;
}
- static int64_t available_ram_bytes() {
+ int64_t available_ram_bytes() {
#if defined(_WIN32)
MEMORYSTATUSEX status{};
status.dwLength = sizeof(status);
if (GlobalMemoryStatusEx(&status)) {
- return (int64_t)status.ullAvailPhys;
+ // Commit exhaustion can reject allocations while physical RAM is still available.
+ return (int64_t)std::min(status.ullAvailPhys, status.ullAvailPageFile);
}
#elif defined(__linux__)
std::ifstream meminfo("/proc/meminfo");
@@ -195,6 +203,16 @@ namespace sd::backend_fit {
return -1;
}
+ int64_t available_commit_bytes() {
+#ifdef _WIN32
+ MEMORYSTATUSEX status{};
+ status.dwLength = sizeof(status);
+ if (GlobalMemoryStatusEx(&status))
+ return static_cast<int64_t>(status.ullAvailPageFile);
+#endif
+ return -1;
+ }
+
static size_t select_main_device(const std::vector<Device>& devices) {
// Shared system memory must not make an integrated GPU outrank a discrete GPU.
for (bool discrete : {true, false}) {
@@ -427,11 +445,12 @@ namespace sd::backend_fit {
return true;
}
- bool derive_backend_specs(ModelLoader& loader,
- ggml_type override_wtype,
- sd::ggml_graph_cut::MaxVramAssignment& budgets,
- std::string& runtime_spec,
- std::string& params_spec) {
+ static bool derive_component_specs(const std::vector<Component>& components,
+ sd::ggml_graph_cut::MaxVramAssignment& budgets,
+ std::string& runtime_spec,
+ std::string& params_spec,
+ const AvailableVRAM& available_vram,
+ size_t max_ram_bytes) {
std::string error;
if (!budgets.canonicalize_backend_keys(&error)) {
LOG_ERROR("%s", error.c_str());
@@ -440,16 +459,17 @@ namespace sd::backend_fit {
// Resolve once to ensure dynamic backends are loaded before enumerating devices.
sd_backend_resolve_name("");
- const auto components = estimate_components(loader, override_wtype);
- const auto devices = enumerate_gpu_devices(budgets, !runtime_spec.empty());
+ const auto devices = enumerate_gpu_devices(budgets, !runtime_spec.empty(), available_vram);
std::vector<Runtime> runtimes;
if (!runtime_spec.empty() && !resolve_runtimes(components, devices, runtime_spec, runtimes, error)) {
LOG_ERROR("%s", error.c_str());
return false;
}
- const int64_t free_ram = available_ram_bytes();
- const int64_t ram_budget = std::max<int64_t>(free_ram - std::max<int64_t>(2048 * MiB, free_ram / 10), 0);
- const auto plan = compute_plan(components, devices, ram_budget, runtimes);
+ const int64_t free_ram = available_ram_bytes();
+ int64_t ram_budget = std::max<int64_t>(free_ram - std::max<int64_t>(2048 * MiB, free_ram / 10), 0);
+ if (max_ram_bytes)
+ ram_budget = static_cast<int64_t>(std::min<uint64_t>(ram_budget, max_ram_bytes));
+ const auto plan = compute_plan(components, devices, ram_budget, runtimes);
params_spec.clear();
if (!plan.valid) {
if (devices.empty()) {
@@ -490,6 +510,32 @@ namespace sd::backend_fit {
return true;
}
+ std::string select_default_backend(const sd::ggml_graph_cut::MaxVramAssignment& budgets, const AvailableVRAM& available_vram) {
+ const auto fallback = sd_backend_resolve_name("");
+ const auto devices = enumerate_gpu_devices(budgets, false, available_vram);
+ const auto selected = select_main_device(devices);
+ return selected == SIZE_MAX ? fallback : devices[selected].name;
+ }
+
+ bool derive_backend_specs(ModelLoader& loader,
+ ggml_type override_wtype,
+ sd::ggml_graph_cut::MaxVramAssignment& budgets,
+ std::string& runtime_spec,
+ std::string& params_spec,
+ const AvailableVRAM& available_vram,
+ size_t max_ram_bytes) {
+ return derive_component_specs(estimate_components(loader, override_wtype), budgets, runtime_spec, params_spec,
+ available_vram, max_ram_bytes);
+ }
+
+ bool derive_backend_specs(ModelLoader& loader,
+ ggml_type override_wtype,
+ sd::ggml_graph_cut::MaxVramAssignment& budgets,
+ std::string& runtime_spec,
+ std::string& params_spec) {
+ return derive_backend_specs(loader, override_wtype, budgets, runtime_spec, params_spec, {});
+ }
+
bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params,
bool prefer_temporal_tiling,
ggml_status status,
diff --git a/src/core/backend_fit.h b/src/core/backend_fit.h
index 61393ad..0d45c8d 100644
--- a/src/core/backend_fit.h
+++ b/src/core/backend_fit.h
@@ -1,21 +1,40 @@
#ifndef __SD_BACKEND_FIT_H__
#define __SD_BACKEND_FIT_H__
+#include <functional>
#include <string>
-#include "config.h"
#include "core/ggml_graph_cut.h"
#include "model_loader.h"
#include "stable-diffusion.h"
+namespace sd {
+ struct TilingConfig;
+}
+
namespace sd::backend_fit {
+ int64_t available_ram_bytes();
+ int64_t available_commit_bytes();
+
+ using AvailableVRAM = std::function<size_t(ggml_backend_dev_t, size_t, size_t)>;
+
+ std::string select_default_backend(const sd::ggml_graph_cut::MaxVramAssignment& budgets, const AvailableVRAM& available_vram = {});
+
bool derive_backend_specs(ModelLoader& loader,
ggml_type override_wtype,
sd::ggml_graph_cut::MaxVramAssignment& budgets,
std::string& runtime_spec,
std::string& params_spec);
+ bool derive_backend_specs(ModelLoader& loader,
+ ggml_type override_wtype,
+ sd::ggml_graph_cut::MaxVramAssignment& budgets,
+ std::string& runtime_spec,
+ std::string& params_spec,
+ const AvailableVRAM& available_vram,
+ size_t max_ram_bytes = 0);
+
bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params,
bool prefer_temporal_tiling,
ggml_status status,
diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp
index 128e0c3..d836341 100644
--- a/src/core/compute_workspace.cpp
+++ b/src/core/compute_workspace.cpp
@@ -2,6 +2,7 @@
#include <algorithm>
#include <exception>
+#include <limits>
#include <map>
#include <unordered_map>
#include <unordered_set>
@@ -25,7 +26,13 @@ namespace sd {
}
}
- ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) {
+ ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size, const std::function<bool(size_t)>& ensure_host_memory) {
+ // Round up ggml's 30 split inputs and two tensor copies per graph node.
+ const auto per_node = 64 * ggml_tensor_overhead();
+ const auto graph_bytes = ggml_graph_overhead_custom(graph_size, false);
+ if (graph_size > (std::numeric_limits<size_t>::max() - graph_bytes) / per_node ||
+ (ensure_host_memory && !ensure_host_memory(graph_size * per_node + graph_bytes)))
+ return nullptr;
std::vector<ggml_backend_t> backends{backend_};
backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end());
if (!sd_backend_is_cpu(backend_)) {
@@ -69,17 +76,26 @@ namespace sd {
release_scheduler();
}
- bool ComputeWorkspace::allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes) {
+ bool ComputeWorkspace::allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes, const std::function<bool(size_t)>& ensure_host_memory) {
GGML_ASSERT(!active_);
const size_t capacity = static_cast<size_t>(ggml_graph_n_nodes(graph) + ggml_graph_n_leafs(graph)) + 8;
if (scheduler_ == nullptr || capacity > scheduler_capacity_) {
release_scheduler();
- scheduler_ = make_scheduler(capacity);
+ scheduler_ = make_scheduler(capacity, ensure_host_memory);
scheduler_capacity_ = capacity;
}
if (scheduler_ == nullptr) {
return false;
}
+ size_t host_bytes = 0;
+ for (const auto& size : measurement.buffers) {
+ if (sd_backend_is_cpu(size.backend)) {
+ const auto pending = size.bytes - std::min(size.bytes, bytes(size.backend));
+ host_bytes += std::min(pending, SIZE_MAX - host_bytes);
+ }
+ }
+ if (ensure_host_memory && !ensure_host_memory(host_bytes))
+ return false;
ggml_backend_sched_reset(scheduler_);
assign_nodes(scheduler_, graph);
// Scheduler allocation rewrites sources. Split the execution graph only once.
@@ -99,7 +115,8 @@ namespace sd {
ggml_cgraph* graph,
const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
const AssignNodes& assign_nodes,
- bool cache_measurement) {
+ bool cache_measurement,
+ const std::function<bool(size_t)>& ensure_host_memory) {
std::vector<const ggml_tensor*> tensors;
std::unordered_set<const ggml_tensor*> seen;
auto visit = [&](const ggml_tensor* tensor) {
@@ -224,7 +241,7 @@ namespace sd {
};
GGML_ASSERT(ggml_graph_cpy_mapped(graph, copy_graph, map_tensor, &copies));
Measurement result;
- auto scheduler = make_scheduler(graph_size);
+ auto scheduler = make_scheduler(graph_size, ensure_host_memory);
if (scheduler != nullptr) {
assign_nodes(scheduler, copy_graph);
std::vector<size_t> sizes(extra_backends_.size() + 2);
@@ -285,7 +302,22 @@ namespace sd {
if (active_) {
return false;
}
- ggml_backend_sched_free(scheduler_);
+ if (scheduler_ != nullptr) {
+ const int n_backends = ggml_backend_sched_get_n_backends(scheduler_);
+ for (int i = 0; i < n_backends; ++i) {
+ auto backend = ggml_backend_sched_get_backend(scheduler_, i);
+ const size_t size = bytes(backend);
+ if (size == 0) {
+ continue;
+ }
+ auto buft = ggml_backend_sched_get_buffer_type(scheduler_, backend);
+ LOG_VERBOSE("free compute buffer: %.2f MB(%s) on %s",
+ size / (1024.0 * 1024.0),
+ ggml_backend_buft_is_host(buft) ? "RAM" : "VRAM",
+ ggml_backend_name(backend));
+ }
+ ggml_backend_sched_free(scheduler_);
+ }
scheduler_ = nullptr;
scheduler_capacity_ = 0;
peak_scheduler_bytes_.clear();
diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h
index 6cfbd5c..f6f14e0 100644
--- a/src/core/compute_workspace.h
+++ b/src/core/compute_workspace.h
@@ -27,7 +27,7 @@ namespace sd {
};
std::array<CachedMeasurement, 2> measurement_cache_;
- ggml_backend_sched_t make_scheduler(size_t graph_size);
+ ggml_backend_sched_t make_scheduler(size_t graph_size, const std::function<bool(size_t)>& ensure_host_memory);
bool release_scheduler();
public:
@@ -45,12 +45,13 @@ namespace sd {
void set_extra_backends(const std::vector<ggml_backend_t>& backends);
bool prepare(const Measurement& measurement);
bool release_excess(const Measurement& measurement);
- bool allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes);
+ bool allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes, const std::function<bool(size_t)>& ensure_host_memory = {});
Measurement measure(
ggml_cgraph* graph,
const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
const AssignNodes& assign_nodes,
- bool cache_measurement = false);
+ bool cache_measurement = false,
+ const std::function<bool(size_t)>& ensure_host_memory = {});
void synchronize() const;
bool segment_end() noexcept;
bool release();
diff --git a/src/core/execution_engine.cpp b/src/core/execution_engine.cpp
index 867fc73..9c388ba 100644
--- a/src/core/execution_engine.cpp
+++ b/src/core/execution_engine.cpp
@@ -44,7 +44,7 @@ namespace sd {
bool assign_graph_cut_layer_split_backends(ggml_cgraph* graph);
ComputeWorkspace::Measurement measure(ggml_cgraph* graph, const std::unordered_set<const ggml_tensor*>* external_outputs = nullptr);
std::vector<DeviceMemoryRequest> memory_requests(const std::vector<BackendBufferSize>& sizes, size_t pending_cache_bytes) const;
- bool fits(const std::vector<DeviceMemoryRequest>& requests, const std::vector<ggml_tensor*>& params) const;
+ bool fits(const std::vector<DeviceMemoryRequest>& requests, const std::vector<ggml_tensor*>& params, bool allow_reclamation = false) const;
bool execute_segment(ggml_cgraph* graph, int n_threads, const std::function<void()>& on_before_compute);
bool execute_graph(ggml_cgraph* graph, int n_threads, bool no_return, const std::function<bool()>& read_outputs);
bool is_multi_device() const { return !session_.extra_backends_.empty(); }
@@ -352,6 +352,13 @@ namespace sd {
}
sd::GraphCutLayerSplitAssignment assignment;
+ auto backend_limits = session_.backend_limits_;
+ backend_limits.resize(split_backends.size(), session_.max_backend_bytes_);
+ for (size_t i = 0; i < split_backends.size(); ++i) {
+ const auto context_limit = manager->context_vram_limit(split_backends[i]);
+ if (context_limit > 0)
+ backend_limits[i] = backend_limits[i] > 0 ? std::min(backend_limits[i], context_limit) : context_limit;
+ }
auto canonicalize_param = [this](ggml_tensor* tensor) {
return canonical_param_tensor(tensor);
};
@@ -359,7 +366,7 @@ namespace sd {
gf,
plan,
split_backends,
- session_.backend_limits_,
+ backend_limits,
session_.max_backend_bytes_,
session_.weight_assignments_,
canonicalize_param,
@@ -399,7 +406,9 @@ namespace sd {
auto assign_nodes = [&](ggml_backend_sched_t scheduler, ggml_cgraph* copy) {
pin_multi_device_nodes(scheduler, copy, graph);
};
- return session_.workspace_.measure(graph, external_backend, assign_nodes, !is_multi_device());
+ return session_.workspace_.measure(graph, external_backend, assign_nodes, !is_multi_device(), [&](size_t bytes) {
+ return manager_ == nullptr || manager_->ensure_host_memory(bytes, task_.weights);
+ });
}
std::vector<DeviceMemoryRequest> GraphExecution::memory_requests(
@@ -429,13 +438,14 @@ namespace sd {
}
bool GraphExecution::fits(const std::vector<DeviceMemoryRequest>& requests,
- const std::vector<ggml_tensor*>& params) const {
+ const std::vector<ggml_tensor*>& params,
+ bool allow_reclamation) const {
auto manager = manager_;
if (manager == nullptr) {
return params.empty();
}
for (const auto& request : requests) {
- if (!manager->fits_compute_backend_capacity(request, params)) {
+ if (!manager->fits_compute_backend_capacity(request, params, allow_reclamation)) {
return false;
}
}
@@ -528,10 +538,10 @@ namespace sd {
const auto& cached_plan = resolve_graph_cut_plan(graph);
auto manager = manager_;
const bool can_segment = !is_multi_device() && !sd_backend_is_cpu(session_.backend_) &&
- manager != nullptr && manager->segmented_compute_enabled() &&
+ session_.segmented_compute_ &&
cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1;
// An empty workspace request is a lower bound on the full graph's memory demand.
- bool segmented = can_segment && !fits(memory_requests({{session_.backend_, 0}}, 0), params);
+ bool segmented = can_segment && !fits(memory_requests({{session_.backend_, 0}}, 0), params, true);
ComputeWorkspace::Measurement full_measurement;
if (!segmented) {
full_measurement = measure(graph);
@@ -546,7 +556,7 @@ namespace sd {
for (auto& request : requests) {
request.pending_allocation_bytes = add_bytes(request.pending_allocation_bytes, planning_headroom);
}
- segmented = !fits(requests, params);
+ segmented = !fits(requests, params, true);
}
}
ggml_graph_cut::Plan monolithic_plan;
@@ -579,7 +589,7 @@ namespace sd {
SegmentGraphBindings bindings(session_.cut_cache_, plan, graph);
SegmentWeightPipeline weights(manager, session_.backend_, session_.owner_id(),
graph, plan, params_tensor_set_,
- segmented && manager != nullptr && manager->prefetch_enabled());
+ segmented && session_.prefetch_enabled_);
std::map<ggml_backend_t, size_t> peak_compute_bytes;
auto track_compute_buffer = [&](ggml_backend_t backend) {
@@ -669,10 +679,21 @@ namespace sd {
last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
return fail_segment("output cache allocation");
}
- new_cache_bytes = add_bytes(last ? 0 : session_.cut_cache_.estimate_output_bytes(graph, segment), session_.cache_.pending_bytes(segment_graph));
- if (!session_.workspace_.allocate(segment_graph, measurement, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) {
- pin_multi_device_nodes(scheduler, current);
- })) {
+ new_cache_bytes = add_bytes(last ? 0 : session_.cut_cache_.estimate_output_bytes(graph, segment), session_.cache_.pending_bytes(segment_graph));
+ auto ensure_host_memory = [&](size_t bytes) {
+ size_t device_commit = 0;
+#ifdef _WIN32
+ for (const auto& size : measurement.buffers) {
+ if (sd_backend_is(size.backend, "CUDA")) {
+ const auto pending = size.bytes - std::min(size.bytes, session_.workspace_.bytes(size.backend));
+ device_commit = add_bytes(device_commit, pending);
+ }
+ }
+#endif
+ return manager == nullptr || manager->ensure_host_memory(bytes, weights.params(index), device_commit);
+ };
+ if (!session_.workspace_.allocate(
+ segment_graph, measurement, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) { pin_multi_device_nodes(scheduler, current); }, ensure_host_memory)) {
last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
return fail_segment("workspace allocation");
}
@@ -736,7 +757,13 @@ namespace sd {
}
ExecutionSession::ExecutionSession(ggml_backend_t backend, const std::shared_ptr<DeviceResidencyManager>& manager)
- : backend_(backend), residency_manager_(manager), cache_(backend), cut_cache_(backend), workspace_(backend) {
+ : backend_(backend),
+ residency_manager_(manager),
+ cache_(backend),
+ cut_cache_(backend),
+ workspace_(backend),
+ segmented_compute_(manager && manager->segmented_compute_enabled()),
+ prefetch_enabled_(manager && manager->prefetch_enabled()) {
GGML_ASSERT(backend_ != nullptr);
}
@@ -787,7 +814,7 @@ namespace sd {
}
ExecutionSession& ExecutionEngine::create_session(ggml_backend_t backend, const std::shared_ptr<DeviceResidencyManager>& manager) {
- GGML_ASSERT(manager.get() == manager_);
+ GGML_ASSERT(manager ? manager->execution_engine().get() == this : manager_ == nullptr);
auto session = std::make_unique<ExecutionSession>(backend, manager);
auto& result = *session;
sessions_.emplace(session->owner_id(), std::move(session));
@@ -865,7 +892,7 @@ namespace sd {
}
for (auto& entry : sessions_) {
auto& session = *entry.second;
- if (session.owner_id() != request.owner_id && session.workspace_.release()) {
+ if (session.owner_id() != request.owner_id && !session.executing_ && session.workspace_.release()) {
sync_residency(session);
if (capacity().fits(true)) {
return true;
@@ -895,11 +922,15 @@ namespace sd {
return position;
};
std::stable_sort(candidates.begin(), candidates.end(), [&](const auto& lhs, const auto& rhs) {
+ if (lhs.next_use != rhs.next_use)
+ return lhs.next_use > rhs.next_use;
+ const auto left = preferred_position(lhs), right = preferred_position(rhs);
+ if (manager.retains_idle_weights() && (left == SIZE_MAX) != (right == SIZE_MAX))
+ return left == SIZE_MAX;
if (lhs.requires_demotion != rhs.requires_demotion) {
return !lhs.requires_demotion;
}
if (!lhs.requires_demotion) {
- const auto left = preferred_position(lhs), right = preferred_position(rhs);
if (left != right) {
return left < right;
}
diff --git a/src/core/execution_engine.h b/src/core/execution_engine.h
index 79ced1d..11d9ed4 100644
--- a/src/core/execution_engine.h
+++ b/src/core/execution_engine.h
@@ -57,6 +57,8 @@ namespace sd {
size_t logged_segment_count_ = 0;
bool started_ = false;
bool executing_ = false;
+ bool segmented_compute_ = false;
+ bool prefetch_enabled_ = false;
size_t resident_bytes(ggml_backend_t backend) const;
uintptr_t owner_id() const { return reinterpret_cast<uintptr_t>(this); }
diff --git a/src/core/file_access.h b/src/core/file_access.h
new file mode 100644
index 0000000..a941352
--- /dev/null
+++ b/src/core/file_access.h
@@ -0,0 +1,44 @@
+#ifndef __SD_CORE_FILE_ACCESS_H__
+#define __SD_CORE_FILE_ACCESS_H__
+
+#include <stdexcept>
+#include <string>
+
+#include "stable-diffusion.h"
+
+namespace sd {
+ struct FileAccessPolicy {
+ sd_dgl_file_access_cb_t callback = nullptr;
+ void* userdata = nullptr;
+ };
+
+ inline thread_local FileAccessPolicy active_file_access;
+
+ class FileAccessScope {
+ FileAccessPolicy previous_;
+
+ public:
+ explicit FileAccessScope(FileAccessPolicy policy)
+ : previous_(active_file_access) { active_file_access = policy; }
+ ~FileAccessScope() { active_file_access = previous_; }
+ FileAccessScope(const FileAccessScope&) = delete;
+ FileAccessScope& operator=(const FileAccessScope&) = delete;
+ };
+
+ class FileAccessDenied : public std::runtime_error {
+ public:
+ using std::runtime_error::runtime_error;
+ };
+
+ inline void require_file_access(const std::string& path, sd_dgl_file_access_t access) {
+ const auto& policy = active_file_access;
+ if (policy.callback &&
+ (path.find('\0') != std::string::npos || !policy.callback(path.c_str(), access, policy.userdata))) {
+ const char* action = access == SD_DGL_FILE_WRITE ? "write" : access == SD_DGL_FILE_READ_MODEL ? "model read"
+ : "read";
+ throw FileAccessDenied(std::string("file ") + action + " denied for '" + path + "'");
+ }
+ }
+}
+
+#endif // __SD_CORE_FILE_ACCESS_H__
diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp
index fa4a3ef..e399092 100644
--- a/src/core/ggml_extend_backend.cpp
+++ b/src/core/ggml_extend_backend.cpp
@@ -896,6 +896,15 @@ ggml_backend_t SDBackendManager::runtime_backend(SDBackendModule module) {
return init_cached_backend(primary_device_name(runtime_assignment_.get(module)));
}
+ggml_backend_t SDBackendManager::existing_backend(ggml_backend_dev_t device) const {
+ const auto& backends = shared_backends_ ? *shared_backends_ : backends_;
+ for (const auto& entry : backends) {
+ if (ggml_backend_get_device(entry.second.get()) == device)
+ return entry.second.get();
+ }
+ return nullptr;
+}
+
std::vector<ggml_backend_t> SDBackendManager::runtime_backends(SDBackendModule module) {
std::vector<ggml_backend_t> backends;
for (const std::string& name : split_device_list(runtime_assignment_.get(module))) {
@@ -1110,13 +1119,14 @@ bool SDBackendManager::validate(std::string* error) const {
}
ggml_backend_t SDBackendManager::init_cached_backend(const std::string& name) {
+ auto& backends = shared_backends_ ? *shared_backends_ : backends_;
std::string resolved = sd_backend_resolve_name(name);
std::string key = lower_copy(resolved);
ggml_backend_t backend = nullptr;
if (!key.empty()) {
- auto it = backends_.find(key);
- if (it != backends_.end()) {
+ auto it = backends.find(key);
+ if (it != backends.end()) {
return it->second.get();
}
} else if (!is_default_backend_token(name)) {
@@ -1135,15 +1145,15 @@ ggml_backend_t SDBackendManager::init_cached_backend(const std::string& name) {
actual_key = !key.empty() ? key : lower_copy(trim_copy(name));
}
- auto it = backends_.find(actual_key);
- if (it != backends_.end()) {
+ auto it = backends.find(actual_key);
+ if (it != backends.end()) {
ggml_backend_free(backend);
return it->second.get();
}
SDBackendHandle handle(backend);
sd_ggml_register_extensions(backend);
- backends_.emplace(actual_key, std::move(handle));
+ backends.emplace(actual_key, std::move(handle));
return backend;
}
diff --git a/src/core/ggml_extend_backend.h b/src/core/ggml_extend_backend.h
index a535ab9..181c0d1 100644
--- a/src/core/ggml_extend_backend.h
+++ b/src/core/ggml_extend_backend.h
@@ -39,6 +39,7 @@ struct SDBackendHandleDeleter {
};
using SDBackendHandle = std::unique_ptr<struct ggml_backend, SDBackendHandleDeleter>;
+using SDBackendPool = std::unordered_map<std::string, SDBackendHandle>;
enum class SDSplitMode {
LAYER,
@@ -50,10 +51,13 @@ private:
SDBackendAssignment runtime_assignment_;
SDBackendAssignment params_assignment_;
SDBackendAssignment split_mode_assignment_;
- std::unordered_map<std::string, SDBackendHandle> backends_;
+ SDBackendPool backends_;
+ std::shared_ptr<SDBackendPool> shared_backends_;
public:
SDBackendManager() = default;
+ explicit SDBackendManager(std::shared_ptr<SDBackendPool> backends)
+ : shared_backends_(std::move(backends)) {}
~SDBackendManager();
SDBackendManager(const SDBackendManager&) = delete;
@@ -69,6 +73,7 @@ public:
ggml_backend_t params_backend(SDBackendModule module);
std::vector<ggml_backend_t> runtime_backends(SDBackendModule module);
+ ggml_backend_t existing_backend(ggml_backend_dev_t device) const;
SDSplitMode split_mode(SDBackendModule module) const;
ggml_backend_buffer_type_t split_buffer_type(ggml_backend_t backend,
diff --git a/src/core/ggml_graph_cut.cpp b/src/core/ggml_graph_cut.cpp
index fc6d8c6..8347306 100644
--- a/src/core/ggml_graph_cut.cpp
+++ b/src/core/ggml_graph_cut.cpp
@@ -1029,9 +1029,9 @@ namespace sd::ggml_graph_cut {
plans.pop_back();
}
if (log_desc != nullptr) {
- LOG_INFO("%s build cached graph cut plan done (taking %lld ms)",
- log_desc,
- ggml_time_ms() - t_plan_begin);
+ LOG_VERBOSE("%s build cached graph cut plan done (taking %lld ms)",
+ log_desc,
+ ggml_time_ms() - t_plan_begin);
}
return plans.front();
}
diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp
index 7a27811..1843021 100644
--- a/src/core/ggml_runner.cpp
+++ b/src/core/ggml_runner.cpp
@@ -193,6 +193,7 @@ GGMLRunnerContext GGMLRunner::get_context(ggml_cgraph* graph) {
runner_ctx.backend = runtime_backend;
runner_ctx.flash_attn_enabled = flash_attn_enabled;
runner_ctx.sage_attn_enabled = sage_attn_enabled;
+ runner_ctx.conv_scale = conv_scale;
runner_ctx.linear_scale = linear_scale;
runner_ctx.attn_scale = attn_scale;
runner_ctx.conv2d_direct_enabled = conv2d_direct_enabled;
diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h
index 9524824..ab785d7 100644
--- a/src/core/ggml_runner.h
+++ b/src/core/ggml_runner.h
@@ -23,6 +23,11 @@
struct GGMLRunnerContext;
+struct AttentionProcessor {
+ virtual ~AttentionProcessor() = default;
+ virtual ggml_tensor* apply(GGMLRunnerContext& ctx, const std::string& layer, ggml_tensor* query, ggml_tensor* output, int64_t heads) const = 0;
+};
+
struct WeightAdapter {
struct ForwardParams {
enum class op_type_t {
@@ -75,6 +80,7 @@ struct GGMLRunnerContext {
bool flash_attn_enabled = false;
bool sage_attn_enabled = false;
float linear_scale = 0.f;
+ float conv_scale = 0.f;
float attn_scale = 0.f;
bool conv2d_direct_enabled = false;
bool conv3d_direct_enabled = false;
@@ -82,6 +88,7 @@ struct GGMLRunnerContext {
bool circular_y_enabled = false;
ggml_tensor* ip_context = nullptr;
float ip_scale = 1.0f;
+ const AttentionProcessor* attn2_processor = nullptr;
std::shared_ptr<WeightAdapter> weight_adapter = nullptr;
std::vector<std::pair<ggml_tensor*, std::string>>* debug_tensors = nullptr;
std::function<ggml_tensor*(const std::string&)> get_cache_tensor;
@@ -162,6 +169,7 @@ protected:
bool flash_attn_enabled = false;
bool sage_attn_enabled = false;
float linear_scale = 0.f;
+ float conv_scale = 0.f;
float attn_scale = 0.f;
bool conv2d_direct_enabled = false;
bool conv3d_direct_enabled = false;
@@ -306,9 +314,10 @@ public:
}
}
- void set_scale_overrides(float linear_scale, float attn_scale) {
+ void set_scale_overrides(float linear_scale, float attn_scale, float conv_scale = 0.f) {
this->linear_scale = linear_scale;
this->attn_scale = attn_scale;
+ this->conv_scale = conv_scale;
}
void set_conv2d_direct_enabled(bool enabled) {
diff --git a/src/core/image_geometry.cpp b/src/core/image_geometry.cpp
new file mode 100644
index 0000000..5ec86f1
--- /dev/null
+++ b/src/core/image_geometry.cpp
@@ -0,0 +1,95 @@
+#include "core/image_geometry.h"
+
+#include "core/util.h"
+
+namespace sd {
+ int64_t align_image_dimension(int64_t size, int64_t multiple) {
+ if (size <= 0 || multiple <= 0)
+ throw std::invalid_argument("image dimensions and alignment must be positive");
+ const auto remainder = size % multiple;
+ const auto rounded = size - remainder;
+ if (remainder >= multiple - remainder) {
+ if (rounded > std::numeric_limits<int64_t>::max() - multiple)
+ throw std::invalid_argument("aligned image dimension is too large");
+ return rounded + multiple;
+ }
+ return std::max(multiple, rounded);
+ }
+
+ int64_t align_video_frame_count(int64_t frames, int64_t multiple, int64_t minimum) {
+ if (frames <= 0 || multiple <= 0 || minimum <= 0)
+ throw std::invalid_argument("video frame count and alignment must be positive");
+ const auto count = std::max(frames, minimum);
+ const auto remainder = (count - minimum) % multiple;
+ const auto pad = remainder ? multiple - remainder : 0;
+ if (count > std::numeric_limits<int64_t>::max() - pad)
+ throw std::invalid_argument("aligned video frame count is too large");
+ return count + pad;
+ }
+
+ Tensor<float> fit_image_tensor(const Tensor<float>& image, int64_t multiple, const char* operation) {
+ if (image.empty() || (image.dim() != 4 && image.dim() != 5))
+ throw std::invalid_argument("expected an image or video pixel tensor");
+ auto shape = image.shape();
+ shape[0] = align_image_dimension(shape[0], multiple);
+ shape[1] = align_image_dimension(shape[1], multiple);
+ if (shape == image.shape())
+ return {};
+ LOG_VERBOSE("%s: resizing pixels from %lldx%lld to %lldx%lld (alignment %lld)", operation,
+ (long long)image.shape()[0], (long long)image.shape()[1],
+ (long long)shape[0], (long long)shape[1], (long long)multiple);
+ return ops::interpolate(image, std::move(shape), ops::InterpolateMode::Lanczos, false, true);
+ }
+
+ Tensor<float> pad_video_tensor(const Tensor<float>& video, int64_t frames) {
+ if (video.empty() || video.dim() != 5 || frames < video.shape()[2])
+ throw std::invalid_argument("video padding requires a nonempty video and cannot remove frames");
+ if (frames == video.shape()[2])
+ return {};
+ auto shape = video.shape();
+ shape[2] = frames;
+ Tensor<float> output(shape);
+ ops::slice_assign(&output, 2, 0, video.shape()[2], video);
+ auto last = ops::slice(video, 2, video.shape()[2] - 1, video.shape()[2]);
+ for (auto i = video.shape()[2]; i < frames; ++i)
+ ops::slice_assign(&output, 2, i, i + 1, last);
+ return output;
+ }
+
+ std::pair<int, int> select_image_size_bucket(int source_width, int source_height, int area, int factor, int fallback_width, int fallback_height) {
+ int width = fallback_width;
+ int height = fallback_height;
+ double ratio = static_cast<double>(source_width) / source_height;
+ double best_crop = INFINITY;
+ int64_t best_area_error = INT64_MAX;
+ int min_size = 4 * factor;
+ for (int w = min_size; w < 2LL * area / min_size; w += factor) {
+ for (int h = min_size; h < 2LL * area / w; h += factor) {
+ int64_t pixels = int64_t(w) * h;
+ if (pixels < 0.9 * area || pixels > 1.1 * area) {
+ continue;
+ }
+ double candidate_ratio = static_cast<double>(w) / h;
+ double crop = std::round((1.0 - std::min(candidate_ratio / ratio, ratio / candidate_ratio)) * 1e9);
+ int64_t area_error = std::abs(pixels - area);
+ if (crop < best_crop || (crop == best_crop && (area_error < best_area_error || (area_error == best_area_error && w > width)))) {
+ width = w;
+ height = h;
+ best_crop = crop;
+ best_area_error = area_error;
+ }
+ }
+ }
+ return {width, height};
+ }
+
+ Tensor<float> resize_and_center_crop(const Tensor<float>& image, int width, int height, ops::InterpolateMode filter) {
+ double scale = std::max(static_cast<double>(width) / image.shape()[0], static_cast<double>(height) / image.shape()[1]);
+ int w = static_cast<int>(std::ceil(image.shape()[0] * scale));
+ int h = static_cast<int>(std::ceil(image.shape()[1] * scale));
+ auto resized = ops::interpolate(image, {w, h, image.shape()[2], image.shape()[3]}, filter);
+ resized = ops::slice(resized, 0, (w - width) / 2, (w - width) / 2 + width);
+ return ops::slice(resized, 1, (h - height) / 2, (h - height) / 2 + height);
+ }
+
+}
diff --git a/src/core/image_geometry.h b/src/core/image_geometry.h
new file mode 100644
index 0000000..229939e
--- /dev/null
+++ b/src/core/image_geometry.h
@@ -0,0 +1,16 @@
+#ifndef __SD_CORE_IMAGE_GEOMETRY_H__
+#define __SD_CORE_IMAGE_GEOMETRY_H__
+
+#include "core/tensor.hpp"
+
+namespace sd {
+ std::pair<int, int> select_image_size_bucket(int source_width, int source_height, int area, int factor, int fallback_width, int fallback_height);
+ Tensor<float> resize_and_center_crop(const Tensor<float>& image, int width, int height, ops::InterpolateMode filter = ops::InterpolateMode::Lanczos);
+ int64_t align_image_dimension(int64_t size, int64_t multiple);
+ int64_t align_video_frame_count(int64_t frames, int64_t multiple, int64_t minimum = 1);
+ // An empty result means no adjustment is needed; borrow the original tensor.
+ Tensor<float> fit_image_tensor(const Tensor<float>& image, int64_t multiple, const char* operation);
+ Tensor<float> pad_video_tensor(const Tensor<float>& video, int64_t frames);
+}
+
+#endif
diff --git a/src/core/image_tensor.cpp b/src/core/image_tensor.cpp
new file mode 100644
index 0000000..4a60983
--- /dev/null
+++ b/src/core/image_tensor.cpp
@@ -0,0 +1,39 @@
+#include "core/image_tensor.h"
+#include "ggml.h"
+
+namespace sd {
+ sd::Tensor<float> ensure_image_tensor_channels(const sd::Tensor<float>& image, int channels) {
+ if (image.empty()) {
+ return image;
+ }
+ GGML_ASSERT(image.dim() == 4 || image.dim() == 5);
+ const size_t channel_axis = image.dim() == 5 ? 3 : 2;
+ int64_t current_channels = image.shape()[channel_axis];
+ if (current_channels == channels) {
+ return image;
+ }
+ sd::Tensor<float> expanded;
+ const auto* source = ℑ
+ if (current_channels == 1 && (channels == 3 || channels == 4)) {
+ auto rg = sd::ops::concat(image, image, channel_axis);
+ expanded = sd::ops::concat(rg, image, channel_axis);
+ if (channels == 3) {
+ return expanded;
+ }
+ source = &expanded;
+ current_channels = 3;
+ }
+ if (channels == 4 && current_channels == 3) {
+ auto alpha_shape = image.shape();
+ alpha_shape[channel_axis] = 1;
+ return sd::ops::concat(*source, sd::full<float>(alpha_shape, 1.f), channel_axis);
+ }
+ if (channels == 3 && current_channels >= 3) {
+ return sd::ops::slice(image, channel_axis, 0, 3);
+ }
+ GGML_ABORT("cannot convert image tensor from %lld to %d channels",
+ (long long)current_channels,
+ channels);
+ }
+
+}
diff --git a/src/core/image_tensor.h b/src/core/image_tensor.h
new file mode 100644
index 0000000..cb57045
--- /dev/null
+++ b/src/core/image_tensor.h
@@ -0,0 +1,10 @@
+#ifndef __SD_CORE_IMAGE_TENSOR_H__
+#define __SD_CORE_IMAGE_TENSOR_H__
+
+#include "core/tensor.hpp"
+
+namespace sd {
+ Tensor<float> ensure_image_tensor_channels(const Tensor<float>& image, int channels);
+}
+
+#endif
diff --git a/src/core/option_types.cpp b/src/core/option_types.cpp
index 2c9a1c1..83e46c6 100644
--- a/src/core/option_types.cpp
+++ b/src/core/option_types.cpp
@@ -113,7 +113,7 @@ static constexpr const char* prediction_to_str[] = {
"eps",
"v",
"edm_v",
- "sd3_flow",
+ "flow",
"flux_flow",
"sefi_flow",
"minit2i_flow",
diff --git a/src/core/probe/backend_probe.cpp b/src/core/probe/backend_probe.cpp
index 57a90ad..c35f586 100644
--- a/src/core/probe/backend_probe.cpp
+++ b/src/core/probe/backend_probe.cpp
@@ -6,9 +6,29 @@
#include <exception>
#include "core/ggml_extend_backend.h"
+#include "core/probe/conv_direct.h"
+#include "core/probe/flash_attn.h"
#include "core/util.h"
#include "ggml-alloc.h"
+bool BackendProbeCache::supports(BackendProbeOperation operation, ggml_backend_t backend, int n_threads) {
+ if (backend == nullptr)
+ return false;
+ const int threads = operation == BackendProbeOperation::FlashAttention && sd_backend_is_cpu(backend)
+ ? (n_threads > 0 ? n_threads : sd_get_num_physical_cores())
+ : 0;
+ const auto key = std::make_pair(operation, threads);
+ std::lock_guard<std::mutex> lock(mutex_);
+ auto& results = results_[backend];
+ const auto found = results.find(key);
+ if (found != results.end())
+ return found->second;
+ const bool supported = operation == BackendProbeOperation::FlashAttention ? sd_probe_flash_attention(backend, n_threads)
+ : sd_probe_conv_direct(backend, operation == BackendProbeOperation::Conv3D);
+ results.emplace(key, supported);
+ return supported;
+}
+
BackendProbeGraph::BackendProbeGraph(ggml_backend_t backend, size_t graph_size)
: backend_(backend), graph_size_(graph_size) {
ctx = ggml_init({graph_size * ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false), nullptr, true});
diff --git a/src/core/probe/backend_probe.h b/src/core/probe/backend_probe.h
index 55b2776..9dc6761 100644
--- a/src/core/probe/backend_probe.h
+++ b/src/core/probe/backend_probe.h
@@ -2,6 +2,8 @@
#define __SD_CORE_PROBE_BACKEND_PROBE_H__
#include <functional>
+#include <map>
+#include <mutex>
#include <string>
#include <vector>
@@ -56,6 +58,18 @@ struct BackendProbeResult {
std::string error;
};
+enum class BackendProbeOperation { FlashAttention,
+ Conv2D,
+ Conv3D };
+
+class BackendProbeCache {
+ std::mutex mutex_;
+ std::map<ggml_backend_t, std::map<std::pair<BackendProbeOperation, int>, bool>> results_;
+
+public:
+ bool supports(BackendProbeOperation operation, ggml_backend_t backend, int n_threads);
+};
+
BackendProbeResult sd_run_backend_probe(ggml_backend_t backend,
int n_threads,
const std::vector<BackendProbeCase>& cases,
diff --git a/src/core/progress.h b/src/core/progress.h
new file mode 100644
index 0000000..ce1c6dd
--- /dev/null
+++ b/src/core/progress.h
@@ -0,0 +1,33 @@
+#ifndef __SD_CORE_PROGRESS_H__
+#define __SD_CORE_PROGRESS_H__
+
+#include <cstdint>
+
+void pretty_progress(int step, int steps, float time);
+void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds);
+
+namespace sd {
+ class ByteProgressScope {
+ using Callback = void (*)(int completed, int total, void* data);
+ Callback previous_callback_;
+ void* previous_data_;
+
+ public:
+ ByteProgressScope(Callback callback, void* data);
+ ~ByteProgressScope();
+ ByteProgressScope(const ByteProgressScope&) = delete;
+ ByteProgressScope& operator=(const ByteProgressScope&) = delete;
+ };
+
+ class ByteProgressConsoleScope {
+ bool previous_;
+
+ public:
+ explicit ByteProgressConsoleScope(bool enabled = true);
+ ~ByteProgressConsoleScope();
+ ByteProgressConsoleScope(const ByteProgressConsoleScope&) = delete;
+ ByteProgressConsoleScope& operator=(const ByteProgressConsoleScope&) = delete;
+ };
+}
+
+#endif
diff --git a/src/core/rng_philox.hpp b/src/core/rng_philox.hpp
index 9799a87..c31ee2d 100644
--- a/src/core/rng_philox.hpp
+++ b/src/core/rng_philox.hpp
@@ -57,9 +57,9 @@ private:
}
public:
- PhiloxRNG(uint64_t seed = 0) {
+ PhiloxRNG(uint64_t seed = 0, uint32_t offset = 0) {
this->seed = seed;
- this->offset = 0;
+ this->offset = offset;
}
std::shared_ptr<RNG> clone() const override {
diff --git a/src/core/util.cpp b/src/core/util.cpp
index 220134f..00b0a31 100644
--- a/src/core/util.cpp
+++ b/src/core/util.cpp
@@ -624,7 +624,36 @@ void pretty_progress(int step, int steps, float time) {
print_progress_line(step, steps, sd_format("%.2f%s", speed, unit), '=', true, remainder_text);
}
+static thread_local void (*byte_progress_callback)(int, int, void*) = nullptr;
+static thread_local void* byte_progress_data = nullptr;
+static thread_local bool byte_progress_console = false;
+
+sd::ByteProgressConsoleScope::ByteProgressConsoleScope(bool enabled)
+ : previous_(byte_progress_console) {
+ byte_progress_console = enabled;
+}
+
+sd::ByteProgressConsoleScope::~ByteProgressConsoleScope() {
+ byte_progress_console = previous_;
+}
+
+sd::ByteProgressScope::ByteProgressScope(Callback callback, void* data)
+ : previous_callback_(byte_progress_callback), previous_data_(byte_progress_data) {
+ byte_progress_callback = callback;
+ byte_progress_data = data;
+}
+
+sd::ByteProgressScope::~ByteProgressScope() {
+ byte_progress_callback = previous_callback_;
+ byte_progress_data = previous_data_;
+}
+
void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds) {
+ if (byte_progress_callback) {
+ byte_progress_callback(step, steps, byte_progress_data);
+ if (!byte_progress_console)
+ return;
+ }
if (sd_progress_cb) {
float time = elapsed_seconds / (step + 1e-6f);
sd_progress_cb(step, steps, time, sd_progress_cb_data);
diff --git a/src/core/util.h b/src/core/util.h
index 082565a..1274e64 100644
--- a/src/core/util.h
+++ b/src/core/util.h
@@ -10,6 +10,7 @@
#include <vector>
#include "core/option_types.h"
+#include "core/progress.h"
#include "core/tensor.hpp"
#include "ggml-backend.h"
#include "stable-diffusion.h"
@@ -87,9 +88,6 @@ bool parse_strict_float(const std::string& text, float& value);
bool parse_strict_int(const std::string& text, int& value);
bool parse_strict_bool(const std::string& text, bool& value);
-void pretty_progress(int step, int steps, float time);
-void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds);
-
void log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
void sd_ggml_log_callback(ggml_log_level level, const char* text, void*);
diff --git a/src/detailer.cpp b/src/detailer.cpp
deleted file mode 100644
index 234afce..0000000
--- a/src/detailer.cpp
+++ /dev/null
@@ -1,1067 +0,0 @@
-#include "detailer.h"
-#include "api/options.h"
-#include "api/result.h"
-#include "core/option_types.h"
-#include "generation_info.h"
-#include "pipeline/generation.h"
-#include "pipeline/request.h"
-
-#include <algorithm>
-#include <cctype>
-#include <cmath>
-#include <cstdlib>
-#include <cstring>
-#include <limits>
-#include <sstream>
-#include <utility>
-
-#include "core/util.h"
-#include "json.hpp"
-#include "model_loader.h"
-
-struct sd_adetailer {
- std::unique_ptr<ADetailerGGML> detailer;
-};
-
-struct LetterboxInput {
- sd::Tensor<float> tensor;
- float scale = 1.f;
- int pad_x = 0;
- int pad_y = 0;
-};
-
-struct Mask {
- int width = 0;
- int height = 0;
- std::vector<uint8_t> data;
-};
-
-struct CropRegion {
- int x1 = 0;
- int y1 = 0;
- int x2 = 0;
- int y2 = 0;
-};
-
-static std::string trim_copy(const std::string& value) {
- size_t first = value.find_first_not_of(" \t\r\n");
- if (first == std::string::npos) {
- return "";
- }
- size_t last = value.find_last_not_of(" \t\r\n");
- return value.substr(first, last - first + 1);
-}
-
-static std::string lower_copy(std::string value) {
- std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
- return static_cast<char>(std::tolower(c));
- });
- return value;
-}
-
-static bool parse_bool(const std::string& value, bool* result) {
- const std::string lower = lower_copy(trim_copy(value));
- if (lower == "1" || lower == "true" || lower == "yes" || lower == "on") {
- *result = true;
- return true;
- }
- if (lower == "0" || lower == "false" || lower == "no" || lower == "off") {
- *result = false;
- return true;
- }
- return false;
-}
-
-static bool parse_int(const std::string& value, int* result) {
- try {
- size_t used = 0;
- int parsed = std::stoi(trim_copy(value), &used);
- if (used != trim_copy(value).size()) {
- return false;
- }
- *result = parsed;
- return true;
- } catch (...) {
- return false;
- }
-}
-
-static bool parse_float(const std::string& value, float* result) {
- try {
- size_t used = 0;
- float parsed = std::stof(trim_copy(value), &used);
- if (used != trim_copy(value).size() || !std::isfinite(parsed)) {
- return false;
- }
- *result = parsed;
- return true;
- } catch (...) {
- return false;
- }
-}
-
-static float image_channel(const sd::Image& image, int x, int y, int channel) {
- if (image.data == nullptr || image.width == 0 || image.height == 0 || image.channel == 0) {
- return 0.f;
- }
- x = std::clamp(x, 0, static_cast<int>(image.width) - 1);
- y = std::clamp(y, 0, static_cast<int>(image.height) - 1);
- int source_channel = image.channel == 1 ? 0 : std::min(channel, static_cast<int>(image.channel) - 1);
- size_t index = (static_cast<size_t>(y) * image.width + x) * image.channel + source_channel;
- return image.data[index] / 255.f;
-}
-
-static float bilinear_channel(const sd::Image& image, float x, float y, int channel) {
- int x0 = static_cast<int>(std::floor(x));
- int y0 = static_cast<int>(std::floor(y));
- int x1 = x0 + 1;
- int y1 = y0 + 1;
- float wx = x - x0;
- float wy = y - y0;
- float a = image_channel(image, x0, y0, channel) * (1.f - wx) + image_channel(image, x1, y0, channel) * wx;
- float b = image_channel(image, x0, y1, channel) * (1.f - wx) + image_channel(image, x1, y1, channel) * wx;
- return a * (1.f - wy) + b * wy;
-}
-
-static LetterboxInput make_letterbox_input(sd::Image image, int input_size) {
- LetterboxInput result;
- result.tensor = sd::full<float>({input_size, input_size, 3, 1}, 114.f / 255.f);
- result.scale = std::min(static_cast<float>(input_size) / image.width,
- static_cast<float>(input_size) / image.height);
- int resized_width = std::max(1, static_cast<int>(std::round(image.width * result.scale)));
- int resized_height = std::max(1, static_cast<int>(std::round(image.height * result.scale)));
- result.pad_x = (input_size - resized_width) / 2;
- result.pad_y = (input_size - resized_height) / 2;
-
- for (int y = 0; y < resized_height; ++y) {
- float source_y = (y + 0.5f) / result.scale - 0.5f;
- for (int x = 0; x < resized_width; ++x) {
- float source_x = (x + 0.5f) / result.scale - 0.5f;
- for (int c = 0; c < 3; ++c) {
- result.tensor.index(x + result.pad_x, y + result.pad_y, c, 0) = bilinear_channel(image, source_x, source_y, c);
- }
- }
- }
- return result;
-}
-
-static float logistic(float x) {
- if (x >= 0.f) {
- float z = std::exp(-x);
- return 1.f / (1.f + z);
- }
- float z = std::exp(x);
- return z / (1.f + z);
-}
-
-static float dfl_expectation(const float* values, int64_t anchor_count, int channel_offset, int reg_max, int64_t anchor) {
- float maximum = -std::numeric_limits<float>::infinity();
- for (int i = 0; i < reg_max; ++i) {
- maximum = std::max(maximum, values[anchor + anchor_count * (channel_offset + i)]);
- }
- float denominator = 0.f;
- float numerator = 0.f;
- for (int i = 0; i < reg_max; ++i) {
- float probability = std::exp(values[anchor + anchor_count * (channel_offset + i)] - maximum);
- denominator += probability;
- numerator += probability * i;
- }
- return denominator > 0.f ? numerator / denominator : 0.f;
-}
-
-static float box_iou(const ADetailerDetection& a, const ADetailerDetection& b) {
- float x1 = std::max(a.x1, b.x1);
- float y1 = std::max(a.y1, b.y1);
- float x2 = std::min(a.x2, b.x2);
- float y2 = std::min(a.y2, b.y2);
- float intersection = std::max(0.f, x2 - x1) * std::max(0.f, y2 - y1);
- float area_a = std::max(0.f, a.x2 - a.x1) * std::max(0.f, a.y2 - a.y1);
- float area_b = std::max(0.f, b.x2 - b.x1) * std::max(0.f, b.y2 - b.y1);
- float union_area = area_a + area_b - intersection;
- return union_area > 0.f ? intersection / union_area : 0.f;
-}
-
-static std::vector<ADetailerDetection> decode_detections(const sd::Tensor<float>& raw,
- const YOLOv8Config& config,
- const LetterboxInput& letterbox,
- sd::Image image,
- const ADetailerParams& params) {
- std::vector<ADetailerDetection> candidates;
- if (raw.empty() || raw.dim() < 2) {
- return candidates;
- }
- int64_t anchor_count = raw.shape()[0];
- int64_t channels = raw.shape()[1];
- if (channels != config.reg_max * 4 + config.num_classes) {
- LOG_ERROR("unexpected YOLOv8 output channels: %lld", static_cast<long long>(channels));
- return candidates;
- }
-
- const std::array<int, 3> strides = {8, 16, 32};
- std::array<int64_t, 4> offsets = {0, 0, 0, 0};
- for (int i = 0; i < 3; ++i) {
- int grid = params.input_size / strides[i];
- offsets[i + 1] = offsets[i] + static_cast<int64_t>(grid) * grid;
- }
- if (offsets[3] != anchor_count) {
- LOG_ERROR("unexpected YOLOv8 anchor count: %lld (expected %lld)",
- static_cast<long long>(anchor_count),
- static_cast<long long>(offsets[3]));
- return candidates;
- }
-
- const float* values = raw.data();
- for (int scale_index = 0; scale_index < 3; ++scale_index) {
- int stride = strides[scale_index];
- int grid = params.input_size / stride;
- for (int64_t anchor = offsets[scale_index]; anchor < offsets[scale_index + 1]; ++anchor) {
- int64_t local = anchor - offsets[scale_index];
- int grid_x = static_cast<int>(local % grid);
- int grid_y = static_cast<int>(local / grid);
-
- float confidence = 0.f;
- int class_id = 0;
- for (int c = 0; c < config.num_classes; ++c) {
- float score = logistic(values[anchor + anchor_count * (config.reg_max * 4 + c)]);
- if (score > confidence) {
- confidence = score;
- class_id = c;
- }
- }
- if (confidence < params.confidence) {
- continue;
- }
-
- float left = dfl_expectation(values, anchor_count, 0 * config.reg_max, config.reg_max, anchor);
- float top = dfl_expectation(values, anchor_count, 1 * config.reg_max, config.reg_max, anchor);
- float right = dfl_expectation(values, anchor_count, 2 * config.reg_max, config.reg_max, anchor);
- float bottom = dfl_expectation(values, anchor_count, 3 * config.reg_max, config.reg_max, anchor);
- float center_x = (grid_x + 0.5f) * stride;
- float center_y = (grid_y + 0.5f) * stride;
-
- ADetailerDetection detection;
- detection.x1 = (center_x - left * stride - letterbox.pad_x) / letterbox.scale;
- detection.y1 = (center_y - top * stride - letterbox.pad_y) / letterbox.scale;
- detection.x2 = (center_x + right * stride - letterbox.pad_x) / letterbox.scale;
- detection.y2 = (center_y + bottom * stride - letterbox.pad_y) / letterbox.scale;
- detection.x1 = std::clamp(detection.x1, 0.f, static_cast<float>(image.width));
- detection.y1 = std::clamp(detection.y1, 0.f, static_cast<float>(image.height));
- detection.x2 = std::clamp(detection.x2, 0.f, static_cast<float>(image.width));
- detection.y2 = std::clamp(detection.y2, 0.f, static_cast<float>(image.height));
- detection.confidence = confidence;
- detection.class_id = class_id;
- if (detection.x2 > detection.x1 && detection.y2 > detection.y1) {
- candidates.push_back(detection);
- }
- }
- }
-
- std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) {
- return a.confidence > b.confidence;
- });
- std::vector<ADetailerDetection> selected;
- for (const auto& candidate : candidates) {
- bool suppressed = false;
- for (const auto& kept : selected) {
- if (candidate.class_id == kept.class_id && box_iou(candidate, kept) > params.nms_threshold) {
- suppressed = true;
- break;
- }
- }
- if (!suppressed) {
- selected.push_back(candidate);
- if (static_cast<int>(selected.size()) >= params.max_detections) {
- break;
- }
- }
- }
- return selected;
-}
-
-static float detection_area(const ADetailerDetection& detection) {
- return std::max(0.f, detection.x2 - detection.x1) * std::max(0.f, detection.y2 - detection.y1);
-}
-
-static void filter_and_sort_detections(std::vector<ADetailerDetection>* detections,
- int width,
- int height,
- const ADetailerParams& params) {
- const float image_area = static_cast<float>(width) * height;
- detections->erase(std::remove_if(detections->begin(), detections->end(), [&](const auto& detection) {
- float ratio = detection_area(detection) / image_area;
- return ratio < params.mask_min_ratio || ratio > params.mask_max_ratio;
- }),
- detections->end());
-
- if (params.mask_k_largest > 0 && static_cast<int>(detections->size()) > params.mask_k_largest) {
- std::partial_sort(detections->begin(),
- detections->begin() + params.mask_k_largest,
- detections->end(),
- [&](const auto& a, const auto& b) {
- return params.mask_filter_by_confidence ? a.confidence > b.confidence : detection_area(a) > detection_area(b);
- });
- detections->resize(params.mask_k_largest);
- }
-
- if (params.sort_by == ADETAILER_SORT_LEFT_TO_RIGHT) {
- std::sort(detections->begin(), detections->end(), [](const auto& a, const auto& b) { return a.x1 < b.x1; });
- } else if (params.sort_by == ADETAILER_SORT_CENTER_TO_EDGE) {
- const float cx = width * 0.5f;
- const float cy = height * 0.5f;
- std::sort(detections->begin(), detections->end(), [&](const auto& a, const auto& b) {
- float ax = (a.x1 + a.x2) * 0.5f - cx;
- float ay = (a.y1 + a.y2) * 0.5f - cy;
- float bx = (b.x1 + b.x2) * 0.5f - cx;
- float by = (b.y1 + b.y2) * 0.5f - cy;
- return ax * ax + ay * ay < bx * bx + by * by;
- });
- } else if (params.sort_by == ADETAILER_SORT_AREA) {
- std::sort(detections->begin(), detections->end(), [](const auto& a, const auto& b) {
- return detection_area(a) > detection_area(b);
- });
- }
-}
-
-static Mask bbox_mask(const ADetailerDetection& detection, int width, int height) {
- Mask mask;
- mask.width = width;
- mask.height = height;
- mask.data.assign(static_cast<size_t>(width) * height, 0);
- int x1 = std::clamp(static_cast<int>(std::floor(detection.x1)), 0, width);
- int y1 = std::clamp(static_cast<int>(std::floor(detection.y1)), 0, height);
- int x2 = std::clamp(static_cast<int>(std::ceil(detection.x2)), 0, width);
- int y2 = std::clamp(static_cast<int>(std::ceil(detection.y2)), 0, height);
- for (int y = y1; y < y2; ++y) {
- std::fill(mask.data.begin() + static_cast<size_t>(y) * width + x1,
- mask.data.begin() + static_cast<size_t>(y) * width + x2,
- 255);
- }
- return mask;
-}
-
-static Mask offset_mask(const Mask& source, int x_offset, int y_offset) {
- Mask output{source.width, source.height, std::vector<uint8_t>(source.data.size(), 0)};
- for (int y = 0; y < source.height; ++y) {
- int target_y = y - y_offset;
- if (target_y < 0 || target_y >= source.height) {
- continue;
- }
- for (int x = 0; x < source.width; ++x) {
- int target_x = x + x_offset;
- if (target_x >= 0 && target_x < source.width) {
- output.data[static_cast<size_t>(target_y) * source.width + target_x] = source.data[static_cast<size_t>(y) * source.width + x];
- }
- }
- }
- return output;
-}
-
-static Mask morphology_mask(const Mask& source, int amount) {
- if (amount == 0) {
- return source;
- }
- int kernel = std::abs(amount);
- int before = kernel / 2;
- int after = kernel - before - 1;
- Mask output{source.width, source.height, std::vector<uint8_t>(source.data.size(), amount > 0 ? 0 : 255)};
- for (int y = 0; y < source.height; ++y) {
- for (int x = 0; x < source.width; ++x) {
- uint8_t value = amount > 0 ? 0 : 255;
- for (int ky = -before; ky <= after; ++ky) {
- for (int kx = -before; kx <= after; ++kx) {
- int sx = x + kx;
- int sy = y + ky;
- uint8_t sample = (sx >= 0 && sx < source.width && sy >= 0 && sy < source.height)
- ? source.data[static_cast<size_t>(sy) * source.width + sx]
- : 0;
- value = amount > 0 ? std::max(value, sample) : std::min(value, sample);
- }
- }
- output.data[static_cast<size_t>(y) * source.width + x] = value;
- }
- }
- return output;
-}
-
-static Mask gaussian_blur_mask(const Mask& source, int radius) {
- if (radius <= 0) {
- return source;
- }
- float sigma = static_cast<float>(radius);
- int kernel_radius = std::max(1, static_cast<int>(std::ceil(2.5f * sigma)));
- std::vector<float> kernel(kernel_radius * 2 + 1);
- float sum = 0.f;
- for (int i = -kernel_radius; i <= kernel_radius; ++i) {
- float value = std::exp(-(i * i) / (2.f * sigma * sigma));
- kernel[i + kernel_radius] = value;
- sum += value;
- }
- for (float& value : kernel) {
- value /= sum;
- }
-
- std::vector<float> horizontal(source.data.size());
- for (int y = 0; y < source.height; ++y) {
- for (int x = 0; x < source.width; ++x) {
- float value = 0.f;
- for (int k = -kernel_radius; k <= kernel_radius; ++k) {
- int sx = std::clamp(x + k, 0, source.width - 1);
- value += source.data[static_cast<size_t>(y) * source.width + sx] * kernel[k + kernel_radius];
- }
- horizontal[static_cast<size_t>(y) * source.width + x] = value;
- }
- }
-
- Mask output{source.width, source.height, std::vector<uint8_t>(source.data.size())};
- for (int y = 0; y < source.height; ++y) {
- for (int x = 0; x < source.width; ++x) {
- float value = 0.f;
- for (int k = -kernel_radius; k <= kernel_radius; ++k) {
- int sy = std::clamp(y + k, 0, source.height - 1);
- value += horizontal[static_cast<size_t>(sy) * source.width + x] * kernel[k + kernel_radius];
- }
- output.data[static_cast<size_t>(y) * source.width + x] = static_cast<uint8_t>(std::clamp(value, 0.f, 255.f) + 0.5f);
- }
- }
- return output;
-}
-
-static std::vector<Mask> make_masks(const std::vector<ADetailerDetection>& detections,
- int width,
- int height,
- const ADetailerParams& params) {
- std::vector<Mask> masks;
- for (const auto& detection : detections) {
- Mask mask = bbox_mask(detection, width, height);
- if (params.x_offset != 0 || params.y_offset != 0) {
- mask = offset_mask(mask, params.x_offset, params.y_offset);
- }
- mask = morphology_mask(mask, params.dilate_erode);
- if (std::any_of(mask.data.begin(), mask.data.end(), [](uint8_t value) { return value != 0; })) {
- masks.push_back(std::move(mask));
- }
- }
- if (params.merge_masks && !masks.empty()) {
- Mask merged{width, height, std::vector<uint8_t>(static_cast<size_t>(width) * height, 0)};
- for (const Mask& mask : masks) {
- for (size_t i = 0; i < merged.data.size(); ++i) {
- merged.data[i] = std::max(merged.data[i], mask.data[i]);
- }
- }
- masks = {std::move(merged)};
- }
- if (params.invert_mask) {
- for (Mask& mask : masks) {
- for (uint8_t& value : mask.data) {
- value = 255 - value;
- }
- }
- }
- return masks;
-}
-
-static bool mask_bbox(const Mask& mask, CropRegion* region) {
- int x1 = mask.width;
- int y1 = mask.height;
- int x2 = 0;
- int y2 = 0;
- for (int y = 0; y < mask.height; ++y) {
- for (int x = 0; x < mask.width; ++x) {
- if (mask.data[static_cast<size_t>(y) * mask.width + x] != 0) {
- x1 = std::min(x1, x);
- y1 = std::min(y1, y);
- x2 = std::max(x2, x + 1);
- y2 = std::max(y2, y + 1);
- }
- }
- }
- if (x2 <= x1 || y2 <= y1) {
- return false;
- }
- *region = {x1, y1, x2, y2};
- return true;
-}
-
-static CropRegion expand_crop(CropRegion crop,
- int image_width,
- int image_height,
- int padding,
- int target_width,
- int target_height) {
- crop.x1 = std::max(0, crop.x1 - padding);
- crop.y1 = std::max(0, crop.y1 - padding);
- crop.x2 = std::min(image_width, crop.x2 + padding);
- crop.y2 = std::min(image_height, crop.y2 + padding);
-
- float target_aspect = static_cast<float>(target_width) / target_height;
- int width = crop.x2 - crop.x1;
- int height = crop.y2 - crop.y1;
- int desired_width = width;
- int desired_height = height;
- if (static_cast<float>(width) / height < target_aspect) {
- desired_width = static_cast<int>(std::ceil(height * target_aspect));
- } else {
- desired_height = static_cast<int>(std::ceil(width / target_aspect));
- }
- desired_width = std::min(desired_width, image_width);
- desired_height = std::min(desired_height, image_height);
- int center_x = (crop.x1 + crop.x2) / 2;
- int center_y = (crop.y1 + crop.y2) / 2;
- crop.x1 = std::clamp(center_x - desired_width / 2, 0, image_width - desired_width);
- crop.y1 = std::clamp(center_y - desired_height / 2, 0, image_height - desired_height);
- crop.x2 = crop.x1 + desired_width;
- crop.y2 = crop.y1 + desired_height;
- return crop;
-}
-
-static sd::Image resize_crop_image(const sd::Image& source, const CropRegion& crop, int width, int height) {
- sd::Image output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 3, nullptr};
- output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height * 3));
- if (output.data == nullptr) {
- return {0, 0, 0, nullptr};
- }
- float scale_x = static_cast<float>(crop.x2 - crop.x1) / width;
- float scale_y = static_cast<float>(crop.y2 - crop.y1) / height;
- for (int y = 0; y < height; ++y) {
- float source_y = crop.y1 + (y + 0.5f) * scale_y - 0.5f;
- for (int x = 0; x < width; ++x) {
- float source_x = crop.x1 + (x + 0.5f) * scale_x - 0.5f;
- for (int c = 0; c < 3; ++c) {
- float value = bilinear_channel(source, source_x, source_y, c);
- output.data[(static_cast<size_t>(y) * width + x) * 3 + c] = static_cast<uint8_t>(std::clamp(value * 255.f, 0.f, 255.f) + 0.5f);
- }
- }
- }
- return output;
-}
-
-static sd::Image resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) {
- sd::Image output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 1, nullptr};
- output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height));
- if (output.data == nullptr) {
- return {0, 0, 0, nullptr};
- }
- float scale_x = static_cast<float>(crop.x2 - crop.x1) / width;
- float scale_y = static_cast<float>(crop.y2 - crop.y1) / height;
- for (int y = 0; y < height; ++y) {
- int source_y = std::clamp(crop.y1 + static_cast<int>((y + 0.5f) * scale_y), 0, source.height - 1);
- for (int x = 0; x < width; ++x) {
- int source_x = std::clamp(crop.x1 + static_cast<int>((x + 0.5f) * scale_x), 0, source.width - 1);
- output.data[static_cast<size_t>(y) * width + x] = source.data[static_cast<size_t>(source_y) * source.width + source_x];
- }
- }
- return output;
-}
-
-static sd::Image copy_for_detailing(const sd::Image& source) {
- if (source.channel == 4) {
- sd::Image output = source;
- size_t bytes = static_cast<size_t>(source.width) * source.height * source.channel;
- output.data = static_cast<uint8_t*>(malloc(bytes));
- if (output.data != nullptr) {
- memcpy(output.data, source.data, bytes);
- }
- return output;
- }
- CropRegion full{0, 0, static_cast<int>(source.width), static_cast<int>(source.height)};
- return resize_crop_image(source, full, source.width, source.height);
-}
-
-static void composite_crop(sd::Image* destination,
- const sd::Image& generated,
- const Mask& feather_mask,
- const CropRegion& crop) {
- int crop_width = crop.x2 - crop.x1;
- int crop_height = crop.y2 - crop.y1;
- float scale_x = static_cast<float>(generated.width) / crop_width;
- float scale_y = static_cast<float>(generated.height) / crop_height;
- for (int y = crop.y1; y < crop.y2; ++y) {
- float source_y = (y - crop.y1 + 0.5f) * scale_y - 0.5f;
- for (int x = crop.x1; x < crop.x2; ++x) {
- float alpha = feather_mask.data[static_cast<size_t>(y) * feather_mask.width + x] / 255.f;
- if (alpha <= 0.f) {
- continue;
- }
- float source_x = (x - crop.x1 + 0.5f) * scale_x - 0.5f;
- size_t destination_index = (static_cast<size_t>(y) * destination->width + x) * destination->channel;
- for (int c = 0; c < 3; ++c) {
- float generated_value = bilinear_channel(generated, source_x, source_y, c) * 255.f;
- float original_value = destination->data[destination_index + c];
- float blended = original_value * (1.f - alpha) + generated_value * alpha;
- destination->data[destination_index + c] = static_cast<uint8_t>(std::clamp(blended, 0.f, 255.f) + 0.5f);
- }
- }
- }
-}
-
-static std::vector<std::string> split_prompts(const std::string& prompt) {
- std::vector<std::string> result;
- size_t start = 0;
- while (true) {
- size_t separator = prompt.find("[SEP]", start);
- result.push_back(trim_copy(prompt.substr(start, separator == std::string::npos ? std::string::npos : separator - start)));
- if (separator == std::string::npos) {
- break;
- }
- start = separator + 5;
- }
- return result;
-}
-
-static std::string resolve_prompt(const char* prompt_template, const char* base_prompt, int index) {
- std::string base = base_prompt == nullptr ? "" : base_prompt;
- std::string templ = prompt_template == nullptr ? "" : prompt_template;
- auto prompts = split_prompts(templ);
- std::string selected = prompts[std::min(index, static_cast<int>(prompts.size()) - 1)];
- if (selected.empty()) {
- return base;
- }
- size_t position = 0;
- while ((position = selected.find("[PROMPT]", position)) != std::string::npos) {
- selected.replace(position, 8, base);
- position += base.size();
- }
- return selected;
-}
-
-static bool prompt_is_skip(const std::string& prompt) {
- return trim_copy(prompt) == "[SKIP]";
-}
-
-static bool validate_params(const ADetailerParams& params) {
- return params.input_size >= 32 && params.input_size % 32 == 0 &&
- params.confidence >= 0.f && params.confidence <= 1.f &&
- params.nms_threshold >= 0.f && params.nms_threshold <= 1.f &&
- params.max_detections > 0 && params.mask_k_largest >= 0 &&
- params.mask_min_ratio >= 0.f && params.mask_max_ratio <= 1.f &&
- params.mask_min_ratio <= params.mask_max_ratio && params.mask_blur >= 0 &&
- params.inpaint_padding >= 0 && params.inpaint_width > 0 && params.inpaint_height > 0 &&
- params.denoising_strength >= 0.f && params.denoising_strength <= 1.f && params.steps >= 0 &&
- (params.cfg_scale < 0.f || std::isfinite(params.cfg_scale));
-}
-
-static std::vector<std::string> parse_class_names(const std::string& value, int class_count) {
- std::vector<std::string> names(static_cast<size_t>(class_count));
- try {
- nlohmann::json parsed = nlohmann::json::parse(value);
- if (parsed.is_array()) {
- for (size_t i = 0; i < names.size() && i < parsed.size(); ++i) {
- if (parsed[i].is_string()) {
- names[i] = parsed[i].get<std::string>();
- }
- }
- } else if (parsed.is_object()) {
- for (const auto& item : parsed.items()) {
- int class_id = 0;
- if (parse_int(item.key(), &class_id) && class_id >= 0 && class_id < class_count &&
- item.value().is_string()) {
- names[static_cast<size_t>(class_id)] = item.value().get<std::string>();
- }
- }
- }
- } catch (const std::exception&) {
- }
- return names;
-}
-
-ADetailerGGML::ADetailerGGML(int n_threads,
- std::string backend_spec,
- std::string params_backend_spec)
- : n_threads(n_threads > 0 ? n_threads : 1),
- backend_spec(std::move(backend_spec)),
- params_backend_spec(std::move(params_backend_spec)) {
-}
-
-ADetailerGGML::~ADetailerGGML() {
- model_manager.reset();
- detector.reset();
-}
-
-bool ADetailerGGML::load_from_file(const std::string& detector_path) {
- std::string error;
- if (!backend_manager.init(backend_spec.c_str(), params_backend_spec.c_str(), nullptr, &error)) {
- LOG_ERROR("ADetailer backend config failed: %s", error.c_str());
- return false;
- }
- ggml_backend_t backend = backend_manager.runtime_backend(SDBackendModule::DETECTOR);
- ggml_backend_t params_backend = backend_manager.params_backend(SDBackendModule::DETECTOR);
- if (backend == nullptr || params_backend == nullptr) {
- LOG_ERROR("failed to initialize detector backend");
- return false;
- }
-
- model_manager = std::make_shared<ModelManager>();
- model_manager->set_n_threads(n_threads);
- model_manager->set_enable_mmap(false);
- ModelLoader loader;
- if (!loader.init_from_file(detector_path)) {
- LOG_ERROR("failed to load ADetailer detector: '%s'", detector_path.c_str());
- return false;
- }
-
- detector = std::make_shared<YOLOv8Runner>(backend, loader.get_tensor_storage_map(), model_manager);
- if (!detector || !detector->model || !detector->config.valid) {
- LOG_ERROR("unsupported YOLOv8 detection weights: '%s'", detector_path.c_str());
- return false;
- }
-
- class_names.assign(static_cast<size_t>(detector->config.num_classes), "");
- auto names_item = loader.get_metadata().find("yolov8.names");
- if (names_item != loader.get_metadata().end()) {
- class_names = parse_class_names(names_item->second, detector->config.num_classes);
- }
-
- std::map<std::string, ggml_tensor*> tensors;
- detector->get_param_tensors(tensors);
- if (!model_manager->set_loader(loader) ||
- !model_manager->register_param_tensors(ModelComponent::Detector,
- std::move(tensors),
- backend_manager.params_backend_is_disk(SDBackendModule::DETECTOR)
- ? ModelManager::ResidencyMode::Disk
- : ModelManager::ResidencyMode::ParamBackend,
- backend,
- params_backend) ||
- !model_manager->validate_registered_tensors()) {
- LOG_ERROR("failed to register YOLOv8 detector tensors");
- return false;
- }
- return true;
-}
-
-std::vector<ADetailerDetection> ADetailerGGML::predict(sd::Image image,
- const ADetailerParams& params) {
- LetterboxInput input = make_letterbox_input(image, params.input_size);
- int64_t start = ggml_time_ms();
- sd::Tensor<float> raw = detector->compute(n_threads, input.tensor);
- detector->runner_end();
- if (raw.empty()) {
- LOG_ERROR("YOLOv8 detector inference failed");
- return {};
- }
- auto detections = decode_detections(raw, detector->config, input, image, params);
- LOG_INFO("ADetailer detected %zu object(s), taking %.2fs",
- detections.size(),
- (ggml_time_ms() - start) / 1000.f);
- for (size_t i = 0; i < detections.size(); ++i) {
- const auto& detection = detections[i];
- std::string object = "class_" + std::to_string(detection.class_id);
- if (detection.class_id >= 0 && static_cast<size_t>(detection.class_id) < class_names.size() &&
- !class_names[static_cast<size_t>(detection.class_id)].empty()) {
- object = class_names[static_cast<size_t>(detection.class_id)];
- }
- LOG_INFO("ADetailer detection %zu: object=%s, class_id=%d, confidence=%.3f, bbox=[x1=%.1f, y1=%.1f, x2=%.1f, y2=%.1f]",
- i + 1,
- object.c_str(),
- detection.class_id,
- detection.confidence,
- detection.x1,
- detection.y1,
- detection.x2,
- detection.y2);
- }
- return detections;
-}
-
-static bool parse_adetailer_params(const sd::ADetailConfig& public_params,
- const sd::ImageConfig& inpaint_params,
- ADetailerParams* params) {
- if (params == nullptr) {
- return false;
- }
- *params = {};
- params->prompt = public_params.prompt;
- params->negative_prompt = public_params.negative_prompt;
- params->inpaint_width = inpaint_params.width;
- params->inpaint_height = inpaint_params.height;
- params->denoising_strength = inpaint_params.strength;
- const char* extra_ad_args = public_params.extra_ad_args;
- if (extra_ad_args == nullptr || extra_ad_args[0] == '\0') {
- return validate_params(*params);
- }
-
- std::stringstream stream(extra_ad_args);
- std::string item;
- while (std::getline(stream, item, ',')) {
- item = trim_copy(item);
- if (item.empty()) {
- continue;
- }
- size_t equals = item.find('=');
- if (equals == std::string::npos) {
- LOG_ERROR("invalid --extra-ad-args item '%s'; expected key=value", item.c_str());
- return false;
- }
- std::string key = lower_copy(trim_copy(item.substr(0, equals)));
- std::string value = trim_copy(item.substr(equals + 1));
- bool ok = true;
- if (key == "input_size") {
- ok = parse_int(value, ¶ms->input_size);
- } else if (key == "confidence") {
- ok = parse_float(value, ¶ms->confidence);
- } else if (key == "nms" || key == "nms_threshold") {
- ok = parse_float(value, ¶ms->nms_threshold);
- } else if (key == "max_detections") {
- ok = parse_int(value, ¶ms->max_detections);
- } else if (key == "mask_k_largest") {
- ok = parse_int(value, ¶ms->mask_k_largest);
- } else if (key == "mask_filter_method") {
- const std::string method = lower_copy(value);
- ok = method == "area" || method == "confidence";
- params->mask_filter_by_confidence = method == "confidence";
- } else if (key == "mask_min_ratio") {
- ok = parse_float(value, ¶ms->mask_min_ratio);
- } else if (key == "mask_max_ratio") {
- ok = parse_float(value, ¶ms->mask_max_ratio);
- } else if (key == "dilate_erode") {
- ok = parse_int(value, ¶ms->dilate_erode);
- } else if (key == "x_offset") {
- ok = parse_int(value, ¶ms->x_offset);
- } else if (key == "y_offset") {
- ok = parse_int(value, ¶ms->y_offset);
- } else if (key == "merge_masks") {
- ok = parse_bool(value, ¶ms->merge_masks);
- } else if (key == "invert_mask") {
- ok = parse_bool(value, ¶ms->invert_mask);
- } else if (key == "mask_blur") {
- ok = parse_int(value, ¶ms->mask_blur);
- } else if (key == "inpaint_padding") {
- ok = parse_int(value, ¶ms->inpaint_padding);
- } else if (key == "inpaint_only_masked") {
- ok = parse_bool(value, ¶ms->inpaint_only_masked);
- } else if (key == "inpaint_width") {
- ok = parse_int(value, ¶ms->inpaint_width);
- } else if (key == "inpaint_height") {
- ok = parse_int(value, ¶ms->inpaint_height);
- } else if (key == "denoising_strength") {
- ok = parse_float(value, ¶ms->denoising_strength);
- } else if (key == "steps") {
- ok = parse_int(value, ¶ms->steps);
- } else if (key == "cfg_scale") {
- ok = parse_float(value, ¶ms->cfg_scale);
- } else if (key == "sample_method") {
- params->sample_method = str_to_sample_method(value.c_str());
- ok = params->sample_method != SAMPLE_METHOD_COUNT;
- } else if (key == "scheduler") {
- params->scheduler = str_to_scheduler(value.c_str());
- ok = params->scheduler != SCHEDULER_COUNT;
- } else if (key == "sort_by") {
- std::string sort = lower_copy(value);
- if (sort == "none") {
- params->sort_by = ADETAILER_SORT_NONE;
- } else if (sort == "left_to_right" || sort == "left-to-right") {
- params->sort_by = ADETAILER_SORT_LEFT_TO_RIGHT;
- } else if (sort == "center_to_edge" || sort == "center-to-edge") {
- params->sort_by = ADETAILER_SORT_CENTER_TO_EDGE;
- } else if (sort == "area") {
- params->sort_by = ADETAILER_SORT_AREA;
- } else {
- ok = false;
- }
- } else if (key == "mask_mode") {
- std::string mode = lower_copy(value);
- if (mode == "none") {
- params->merge_masks = false;
- params->invert_mask = false;
- } else if (mode == "merge") {
- params->merge_masks = true;
- params->invert_mask = false;
- } else if (mode == "merge_invert" || mode == "merge-invert") {
- params->merge_masks = true;
- params->invert_mask = true;
- } else {
- ok = false;
- }
- } else {
- LOG_ERROR("unknown --extra-ad-args key '%s'", key.c_str());
- return false;
- }
- if (!ok) {
- LOG_ERROR("invalid --extra-ad-args value '%s=%s'", key.c_str(), value.c_str());
- return false;
- }
- }
- if (!validate_params(*params)) {
- LOG_ERROR("invalid --extra-ad-args parameter range");
- return false;
- }
- return true;
-}
-
-sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::ADetailer, config);
- if (status != SD_STATUS_OK)
- return status;
- auto result = std::make_unique<sd_adetailer_t>();
- const auto& ctx = config.context;
- result->detailer = std::make_unique<ADetailerGGML>(ctx.n_threads, SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend));
- if (!result->detailer->load_from_file(ctx.model_path))
- return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load ADetailer");
- *out = result.release();
- return SD_STATUS_OK;
- },
- SD_STATUS_LOAD_FAILED);
-}
-
-void sd_adetailer_free(sd_adetailer_t* adetailer) {
- delete adetailer;
-}
-
-static bool run_adetail(sd_adetailer_t* context, sd_ctx_t* sd_ctx, sd::Image input_image, const sd::ADetailConfig* public_params, const sd::ImageConfig* inpaint_params, sd_result_t** result_out, sd_status_t& failure) {
- if (result_out)
- *result_out = nullptr;
- if (!result_out)
- return false;
- if (context == nullptr || context->detailer == nullptr || sd_ctx == nullptr || public_params == nullptr ||
- inpaint_params == nullptr || input_image.data == nullptr) {
- return false;
- }
-
- ADetailerParams params;
- if (!parse_adetailer_params(*public_params, *inpaint_params, ¶ms)) {
- failure = SD_STATUS_INVALID_ARGUMENT;
- return false;
- }
-
- auto resolved_inpaint = *inpaint_params;
- resolved_inpaint.seed = sd::pipeline::resolve_seed(resolved_inpaint.seed);
- const auto info = sd::adetailer_generation_info(*public_params, resolved_inpaint);
- auto detections = context->detailer->predict(input_image, params);
- filter_and_sort_detections(&detections, input_image.width, input_image.height, params);
- auto masks = make_masks(detections, input_image.width, input_image.height, params);
-
- sd::Image current = copy_for_detailing(input_image);
- if (current.data == nullptr) {
- failure = SD_STATUS_OUT_OF_MEMORY;
- return false;
- }
-
- std::unique_ptr<uint8_t, decltype(&free)> current_owner(current.data, &free);
- int applied = 0;
- for (size_t i = 0; i < masks.size(); ++i) {
- std::string prompt = resolve_prompt(params.prompt, inpaint_params->prompt, static_cast<int>(i));
- std::string negative_prompt = resolve_prompt(params.negative_prompt,
- inpaint_params->negative_prompt,
- static_cast<int>(i));
- if (prompt_is_skip(prompt)) {
- continue;
- }
-
- CropRegion crop;
- if (!mask_bbox(masks[i], &crop)) {
- continue;
- }
- crop = params.inpaint_only_masked ? expand_crop(crop,
- current.width,
- current.height,
- params.inpaint_padding,
- params.inpaint_width,
- params.inpaint_height)
- : CropRegion{0, 0, static_cast<int>(current.width), static_cast<int>(current.height)};
- sd::Image local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height);
- sd::Image local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height);
- if (local_image.data == nullptr || local_mask.data == nullptr) {
- failure = SD_STATUS_OUT_OF_MEMORY;
- free(local_image.data);
- free(local_mask.data);
- return false;
- }
-
- sd::ImageConfig generation = resolved_inpaint;
- generation.prompt = prompt.c_str();
- generation.negative_prompt = negative_prompt.c_str();
- generation.init_image = local_image;
- generation.mask_image = local_mask;
- generation.width = params.inpaint_width;
- generation.height = params.inpaint_height;
- generation.strength = params.denoising_strength;
- generation.seed = static_cast<int64_t>((static_cast<uint64_t>(resolved_inpaint.seed) + i) & INT64_MAX);
- generation.batch_count = 1;
- generation.ref_images = nullptr;
- generation.ref_images_count = 0;
- generation.control_image = {};
- generation.pm_params = {};
- generation.pulid_params = {};
- generation.hires.enabled = false;
- generation.image_preprocess = {};
-
- generation.sample_params.guidance.img_cfg = INFINITY;
- if (params.steps > 0) {
- generation.sample_params.sample_steps = params.steps;
- generation.sample_params.custom_sigmas = nullptr;
- generation.sample_params.custom_sigmas_count = 0;
- }
- if (params.cfg_scale >= 0.f) {
- generation.sample_params.guidance.txt_cfg = params.cfg_scale;
- }
- if (params.sample_method != SAMPLE_METHOD_COUNT) {
- generation.sample_params.sample_method = params.sample_method;
- }
- if (params.scheduler != SCHEDULER_COUNT) {
- generation.sample_params.scheduler = params.scheduler;
- }
-
- sd::ImageGenerationResult* generated = nullptr;
- failure = sd::generate_image(sd_ctx, generation, &generated);
- std::unique_ptr<sd::ImageGenerationResult, decltype(&sd::free_image_generation_result)> generated_owner(generated, sd::free_image_generation_result);
- free(local_image.data);
- free(local_mask.data);
- if (failure != SD_STATUS_OK)
- return false;
- if (!generated || generated->count <= 0 || !generated->items[0].image.data) {
- failure = sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "inpainting returned no image");
- return false;
- }
-
- Mask feather = gaussian_blur_mask(masks[i], params.mask_blur);
- composite_crop(¤t, generated->items[0].image, feather, crop);
- ++applied;
- }
-
- const std::vector<nlohmann::json> infos{info};
- auto* outputs = static_cast<sd::Image*>(calloc(1, sizeof(sd::Image)));
- if (!outputs) {
- failure = SD_STATUS_OUT_OF_MEMORY;
- return false;
- }
- outputs[0] = current;
- current_owner.release();
- if (!sd::make_image_result(outputs, 1, infos, result_out)) {
- failure = SD_STATUS_OUT_OF_MEMORY;
- return false;
- }
- LOG_INFO("ADetailer applied %d mask(s)", applied);
- return true;
-}
-
-sd_status_t sd_adetail(sd_adetailer_t* adetailer, sd_ctx_t* ctx, const sd_adetail_opt_t* opt, sd_result_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::ADetail, config);
- if (status != SD_STATUS_OK)
- return status;
- if (!adetailer || !ctx)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "adetailer and diffusion context are required");
- if (!sd_ctx_supports_image_generation(ctx))
- return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "inpainting requires an image-generation model");
- sd_status_t failure = SD_STATUS_GENERATION_FAILED;
- if (!run_adetail(adetailer, ctx, config.post.image, &config.post.adetail, &config.image, out, failure)) {
- if (failure == SD_STATUS_OK)
- failure = SD_STATUS_GENERATION_FAILED;
- if (*sd_get_last_error())
- return failure;
- return sd::api::error(failure, nullptr, "ADetailer failed; see log for details");
- }
- return SD_STATUS_OK;
- },
- SD_STATUS_GENERATION_FAILED);
-}
diff --git a/src/detailer.h b/src/detailer.h
deleted file mode 100644
index 67ede5b..0000000
--- a/src/detailer.h
+++ /dev/null
@@ -1,81 +0,0 @@
-#ifndef __SD_DETAILER_H__
-#define __SD_DETAILER_H__
-
-#include "core/media.h"
-
-#include "core/option_types.h"
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "core/ggml_extend_backend.h"
-#include "model/detector/yolov8.h"
-#include "model_manager.h"
-#include "stable-diffusion.h"
-
-struct ADetailerDetection {
- float x1 = 0.f;
- float y1 = 0.f;
- float x2 = 0.f;
- float y2 = 0.f;
- float confidence = 0.f;
- int class_id = 0;
-};
-
-enum ADetailerSort {
- ADETAILER_SORT_NONE,
- ADETAILER_SORT_LEFT_TO_RIGHT,
- ADETAILER_SORT_CENTER_TO_EDGE,
- ADETAILER_SORT_AREA,
-};
-
-struct ADetailerParams {
- const char* prompt = nullptr;
- const char* negative_prompt = nullptr;
- int input_size = 640;
- float confidence = 0.3f;
- float nms_threshold = 0.45f;
- int max_detections = 100;
- int mask_k_largest = 0;
- bool mask_filter_by_confidence = false;
- float mask_min_ratio = 0.f;
- float mask_max_ratio = 1.f;
- int dilate_erode = 4;
- int x_offset = 0;
- int y_offset = 0;
- bool merge_masks = false;
- bool invert_mask = false;
- int mask_blur = 4;
- int inpaint_padding = 32;
- bool inpaint_only_masked = true;
- int inpaint_width = 512;
- int inpaint_height = 512;
- float denoising_strength = 0.4f;
- int steps = 0;
- float cfg_scale = -1.f;
- sample_method_t sample_method = SAMPLE_METHOD_COUNT;
- scheduler_t scheduler = SCHEDULER_COUNT;
- ADetailerSort sort_by = ADETAILER_SORT_NONE;
-};
-
-struct ADetailerGGML {
- SDBackendManager backend_manager;
- std::shared_ptr<ModelManager> model_manager;
- std::shared_ptr<YOLOv8Runner> detector;
- std::vector<std::string> class_names;
- int n_threads = 1;
- std::string backend_spec;
- std::string params_backend_spec;
-
- ADetailerGGML(int n_threads,
- std::string backend_spec,
- std::string params_backend_spec);
- ~ADetailerGGML();
-
- bool load_from_file(const std::string& detector_path);
- std::vector<ADetailerDetection> predict(sd::Image image,
- const ADetailerParams& params);
-};
-
-#endif // __SD_DETAILER_H__
diff --git a/src/device_residency_manager.h b/src/device_residency_manager.h
index 7317410..3b1be4d 100644
--- a/src/device_residency_manager.h
+++ b/src/device_residency_manager.h
@@ -64,6 +64,7 @@ struct WeightResidencyBlock {
std::vector<ggml_tensor*> tensors;
size_t bytes = 0;
uint64_t last_use_epoch = 0;
+ size_t next_use = 0;
bool requires_demotion = false;
};
@@ -74,17 +75,22 @@ private:
public:
virtual ~DeviceResidencyManager() = default;
- std::shared_ptr<sd::ExecutionEngine> execution_engine();
+ virtual std::shared_ptr<sd::ExecutionEngine> execution_engine();
- virtual bool segmented_compute_enabled() const = 0;
- virtual bool prefetch_enabled() const = 0;
+ virtual bool segmented_compute_enabled() const = 0;
+ virtual bool prefetch_enabled() const = 0;
+ virtual bool retains_idle_weights() const { return false; }
+ virtual size_t context_vram_limit(ggml_backend_t) const { return 0; }
+ // Device bytes consume Windows commit without consuming physical host RAM.
+ virtual bool ensure_host_memory(size_t, const std::vector<ggml_tensor*>&, size_t = 0) { return true; }
virtual void register_runtime_owner(uintptr_t owner_id) = 0;
virtual void remove_runtime_owner(uintptr_t owner_id) = 0;
// Capacity requests select their backend's weights; protection spans all backends.
virtual bool fits_compute_backend_capacity(const DeviceMemoryRequest& request,
- const std::vector<ggml_tensor*>& required_params) const = 0;
+ const std::vector<ggml_tensor*>& required_params,
+ bool allow_reclamation) const = 0;
virtual bool assign_compute_backend(const std::vector<ggml_tensor*>& tensors,
- ggml_backend_t compute_backend) = 0;
+ ggml_backend_t compute_backend) = 0;
virtual ggml_tensor* resolve_param_tensor(ggml_tensor* tensor) const { return nullptr; }
virtual bool prepare_params(const std::vector<ggml_tensor*>& tensors) = 0;
virtual void release_compute_backend_params(const std::vector<ggml_tensor*>& tensors) = 0;
diff --git a/src/dgl/bindings/arguments.cpp b/src/dgl/bindings/arguments.cpp
new file mode 100644
index 0000000..5908d0b
--- /dev/null
+++ b/src/dgl/bindings/arguments.cpp
@@ -0,0 +1,100 @@
+#include "dgl/bindings/arguments.h"
+
+#include "dgl/bindings/values.h"
+#include "dgl/context.h"
+#include "dgl/ops/media.h"
+#include "dgl/ops/preview.h"
+#include "dgl/ops/sampling.h"
+
+namespace sd::dgl {
+ const sd::Tensor<float>& float_tensor(const Values& args, const std::string& name) {
+ return read_tensor<float>(args.at(name));
+ }
+
+ std::vector<sd::Tensor<float>> float_tensor_list(const Value& value) {
+ if (value.type().kind != Kind::List) {
+ throw std::runtime_error("expected a tensor list");
+ }
+ std::vector<sd::Tensor<float>> tensors;
+ for (const auto& item : value.items) {
+ tensors.push_back(read_tensor<float>(item));
+ }
+ return tensors;
+ }
+
+ void validate_float_shape(const std::vector<int64_t>& shape) {
+ size_t remaining = tensor_byte_limit / sizeof(float);
+ if (shape.empty() || shape.size() > 8) {
+ throw std::runtime_error("invalid tensor rank");
+ }
+ for (auto dim : shape) {
+ if (dim <= 0 || static_cast<uint64_t>(dim) > remaining) {
+ throw std::runtime_error("invalid tensor dimension or tensor exceeds 1 GiB");
+ }
+ remaining /= static_cast<size_t>(dim);
+ }
+ }
+
+ const sd::Tensor<float>& pixels(const Value& value, sd::Tensor<float>& decoded) {
+ if (value.type().kind != Kind::Tensor) {
+ throw std::runtime_error("expected an image or pixel tensor");
+ }
+ const auto& tensor = *value.as<sd_tensor_t>();
+ if (tensor.type == SD_TENSOR_F32) {
+ const auto& s = tensor.shape();
+ const auto channel_axis = s.size() == 5 ? 3 : 2;
+ if ((s.size() != 4 && s.size() != 5) || s[channel_axis] < 1 || s[channel_axis] > 4 || s.back() != 1) {
+ throw std::runtime_error("expected a float pixel tensor with width, height, optional time, channels and batch");
+ }
+ return read_tensor<float>(value);
+ }
+ decoded = ops::image_to_f32(tensor);
+ return decoded;
+ }
+
+ sd::Tensor<float> pixels(const Value& value) {
+ sd::Tensor<float> decoded;
+ const auto& input = pixels(value, decoded);
+ if (&input == &decoded)
+ return decoded;
+ return input;
+ }
+
+ Value image_to_u8(const sd::Tensor<float>& tensor) {
+ return Value::object(ops::image_to_u8(tensor));
+ }
+
+ const sd::Tensor<float>& sigma_tensor(const Values& args, const char* key) {
+ const auto tensor = object<sd_tensor_t>(args, key);
+ if (tensor->type != SD_TENSOR_F32 || tensor->shape().size() != 1)
+ throw std::runtime_error(std::string(key) + " must be a rank-one f32 tensor");
+ if (!tensor->bytes)
+ throw std::runtime_error(std::string(key) + " must be nonempty");
+ return tensor->as<float>();
+ }
+
+ ops::SamplingOptions sampling_options(const Values& args, const Context& context) {
+ ops::SamplingOptions options;
+ options.batched_cfg = context.batched_cfg;
+ options.method = get_string(args, "method");
+ options.eta = float(get_number(args, "eta"));
+ if (args.count("denoise_mask")) {
+ if (args.count("base"))
+ options.base = &float_tensor(args, "base");
+ options.denoise_mask = &float_tensor(args, "denoise_mask");
+ }
+ options.mask_input = get_bool(args, "mask_input");
+ options.extra_args = get_string(args, "extra_args");
+ if (args.count("finish"))
+ options.finish = get_bool(args, "finish");
+ if (args.count("cache"))
+ options.cache = cache_options(args.at("cache").fields);
+ if (args.count("observer"))
+ options.observer = object<ops::SamplingObserver>(args, "observer");
+ else {
+ static const auto observer = std::make_shared<ops::PreviewObserver>();
+ options.observer = observer;
+ }
+ return options;
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/arguments.h b/src/dgl/bindings/arguments.h
new file mode 100644
index 0000000..371df48
--- /dev/null
+++ b/src/dgl/bindings/arguments.h
@@ -0,0 +1,41 @@
+#ifndef __SD_DGL_BINDINGS_ARGUMENTS_H__
+#define __SD_DGL_BINDINGS_ARGUMENTS_H__
+
+#include "dgl/bindings/conversion.h"
+#include "dgl/models/fwd.h"
+
+namespace sd::dgl {
+ namespace ops {
+ struct SamplingOptions;
+ }
+ using models::AudioEncoder;
+ using models::AudioVAE;
+ using models::ControlNet;
+ using models::DiT;
+ using models::IPAdapter;
+ using models::Model;
+ using models::PhotoMaker;
+ using models::TextEncoder;
+ using models::VAE;
+ using models::VisionEncoder;
+
+ template <typename T>
+ const sd::Tensor<T>& read_tensor(const Value& value) {
+ const auto* tensor = value.get_if<sd_tensor_t>();
+ if (!tensor || tensor->type != tensor_dtype<T>() ||
+ tensor->bytes == 0 || tensor->shape().empty())
+ throw std::runtime_error("unexpected tensor type, scalar or empty tensor");
+ return tensor->as<T>();
+ }
+
+ std::vector<sd::Tensor<float>> float_tensor_list(const Value& value);
+ const sd::Tensor<float>& float_tensor(const Values& args, const std::string& name);
+ const sd::Tensor<float>& sigma_tensor(const Values& args, const char* key);
+ ops::SamplingOptions sampling_options(const Values& args, const Context& context);
+ const sd::Tensor<float>& pixels(const Value& value, sd::Tensor<float>& decoded);
+ sd::Tensor<float> pixels(const Value& value);
+ Value image_to_u8(const sd::Tensor<float>& tensor);
+ void validate_float_shape(const std::vector<int64_t>& shape);
+}
+
+#endif // __SD_DGL_BINDINGS_ARGUMENTS_H__
diff --git a/src/dgl/bindings/audio.cpp b/src/dgl/bindings/audio.cpp
new file mode 100644
index 0000000..f53369f
--- /dev/null
+++ b/src/dgl/bindings/audio.cpp
@@ -0,0 +1,44 @@
+#include "dgl/bindings/audio.h"
+
+#include <algorithm>
+#include <cctype>
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ OutputSignature audio_signature() {
+ return {{Kind::Dict}, {{"samples", {Kind::Tensor}}, {"sample_rate", {Kind::Integer}}}};
+ }
+
+ Value audio_result(ops::AudioReadResult audio) {
+ if (!audio.samples)
+ audio.samples = make_tensor(SD_TENSOR_F32, std::vector<int64_t>{1, 0});
+ return Value::dict({{"samples", Value::object(std::move(audio.samples))},
+ {"sample_rate", Value::literal(audio.sample_rate)}});
+ }
+
+ ops::AudioReadResult read_audio_dict(const Value& value, bool allow_empty) {
+ if (value.type().kind != Kind::Dict || !value.fields.count("samples") || !value.fields.count("sample_rate") ||
+ value.fields.at("samples").type().kind != Kind::Tensor || value.fields.at("sample_rate").type().kind != Kind::Integer)
+ throw std::runtime_error("audio requires a dict with samples (tensor) and sample_rate (integer)");
+ ops::AudioReadResult result{value.fields.at("samples").as<sd_tensor_t>(), get_int(value.fields, "sample_rate")};
+ if (allow_empty && result.sample_rate == 0 && result.samples->type == SD_TENSOR_F32 &&
+ result.samples->shape() == std::vector<int64_t>{1, 0})
+ return result;
+ audio_view(*result.samples, result.sample_rate);
+ return result;
+ }
+
+ Value execute_load_audio(Values& args, RunState&) {
+ return audio_result(read_wav(std::filesystem::u8path(get_string(args, "path"))));
+ }
+ Value execute_save_audio(Values& args, RunState&) {
+ auto path = std::filesystem::u8path(get_string(args, "path"));
+ auto extension = path.extension().u8string();
+ std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ if (extension != ".wav")
+ throw std::runtime_error("audio output supports .wav only");
+ const auto audio = read_audio_dict(args.at("audio"));
+ write_wav(*audio.samples, audio.sample_rate, path, get_string(args, "format") == "pcm16");
+ return Value::void_value();
+ }
+}
diff --git a/src/dgl/bindings/audio.h b/src/dgl/bindings/audio.h
new file mode 100644
index 0000000..0a1ae9a
--- /dev/null
+++ b/src/dgl/bindings/audio.h
@@ -0,0 +1,15 @@
+#ifndef __SD_DGL_BINDINGS_AUDIO_H__
+#define __SD_DGL_BINDINGS_AUDIO_H__
+
+#include "dgl/ops/audio.h"
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+ OutputSignature audio_signature();
+ Value audio_result(ops::AudioReadResult audio);
+ ops::AudioReadResult read_audio_dict(const Value& value, bool allow_empty = false);
+ Value execute_load_audio(Values& args, RunState&);
+ Value execute_save_audio(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_AUDIO_H__
diff --git a/src/dgl/bindings/component_loader.cpp b/src/dgl/bindings/component_loader.cpp
new file mode 100644
index 0000000..6561b93
--- /dev/null
+++ b/src/dgl/bindings/component_loader.cpp
@@ -0,0 +1,234 @@
+#include "dgl/bindings/component_loader.h"
+
+#include "core/util.h"
+#include "dgl/context.h"
+#include "dgl/models/animatediff.h"
+#include "dgl/models/audio_encoder.h"
+#include "dgl/models/audio_vae.h"
+#include "dgl/models/control.h"
+#include "dgl/models/diffusion_model.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/latent_upscaler.h"
+#include "dgl/models/photomaker.h"
+#include "dgl/models/pulid.h"
+#include "dgl/models/text_encoder.h"
+#include "dgl/models/vae.h"
+#include "dgl/models/vision_encoder.h"
+#include "model_definition.h"
+#include "tokenizers/tokenizer_config.h"
+
+namespace sd::dgl {
+ static const char* string_option(const Values& args, const char* name) {
+ auto found = args.find(name);
+ return found == args.end() ? "" : found->second.scalar.get_ref<const std::string&>().c_str();
+ }
+
+ static models::ModelLoadParams read_model_params(const Values& args) {
+ models::ModelLoadParams params;
+ const auto family = get_string(args, "family");
+ if (family != "auto") {
+ const auto* definition = find_model_definition(family);
+ if (!definition)
+ throw std::runtime_error("unknown component family '" + family + "'");
+ params.family = definition->version;
+ }
+ params.family_from = string_option(args, "family_from");
+ params.path = get_string(args, "path");
+ params.backend = get_string(args, "backend");
+ params.params_backend = get_string(args, "params_backend");
+ params.split_mode = get_string(args, "split_mode");
+ params.model_args = get_string(args, "model_args");
+ params.tensor_type_rules = get_string(args, "tensor_type_rules");
+ static const std::map<std::string, models::OperatorMode> modes{
+ {"auto", models::OperatorMode::Auto}, {"on", models::OperatorMode::On}, {"off", models::OperatorMode::Off}};
+ params.flash_attn_mode = modes.at(trim(get_string(args, "flash_attn_mode")));
+ params.conv_direct_mode = modes.at(trim(get_string(args, "conv_direct_mode")));
+ params.sage_attn = get_bool(args, "sage_attn");
+ params.conv_scale = float(get_number(args, "conv_scale"));
+ params.linear_scale = float(get_number(args, "linear_scale"));
+ params.attn_scale = float(get_number(args, "attn_scale"));
+ if (get_string(args, "wtype") != "auto") {
+ params.wtype = str_to_sd_type(string_option(args, "wtype"));
+ if (params.wtype == SD_TYPE_COUNT)
+ throw std::runtime_error("unknown weight type");
+ }
+ return params;
+ }
+
+ static nlohmann::json normalized_model_params(models::ModelLoadParams& params, PreparedResource& prepared) {
+ prepared.revision = nlohmann::json::object();
+ params.path = file_key(params.path, prepared.revision);
+ params.family_from = file_key(params.family_from, prepared.revision);
+ params.backend = trim(params.backend);
+ params.params_backend = trim(params.params_backend);
+ params.split_mode = trim(params.split_mode);
+ for (float* scale : {¶ms.conv_scale, ¶ms.linear_scale, ¶ms.attn_scale})
+ if (*scale == 0.f)
+ *scale = 0.f;
+ return {
+ {"path", params.path},
+ {"family", int(params.family)},
+ {"family_from", params.family_from},
+ {"backend", params.backend},
+ {"params_backend", params.params_backend},
+ {"split_mode", params.split_mode},
+ {"wtype", int(params.wtype)},
+ {"tensor_type_rules", params.tensor_type_rules},
+ {"model_args", params.model_args},
+ {"flash_attn_mode", int(params.flash_attn_mode)},
+ {"conv_direct_mode", int(params.conv_direct_mode)},
+ {"sage_attn", params.sage_attn},
+ {"conv_scale", params.conv_scale},
+ {"linear_scale", params.linear_scale},
+ {"attn_scale", params.attn_scale},
+ };
+ }
+
+ template <class T, class Params>
+ static std::shared_ptr<T> load_model(ResourceLoadRequest& request, Params params, const std::string& path) {
+ auto model = request.context.make_model<T>(request.weights_key);
+ if (!model->init(std::move(params)))
+ throw std::runtime_error("cannot load component from '" + path + "'");
+ return model;
+ }
+
+ template <class T, class Params>
+ static Value make_model(ResourceLoadRequest& request, Params params, const std::string& path) {
+ return Value::object(load_model<T>(request, std::move(params), path));
+ }
+
+ PreparedResource load_dit(Values& args) {
+ models::DiTLoadParams params{
+ read_model_params(args),
+ };
+ PreparedResource prepared;
+ prepared.key = normalized_model_params(params.model, prepared);
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ const auto path = params.model.path;
+ auto model = load_model<models::DiT>(request, std::move(params), path);
+ return Value::object(std::make_shared<models::DiffusionModel>(models::DiffusionModel{std::move(model)}));
+ };
+ return prepared;
+ }
+
+ PreparedResource load_text_encoder(Values& args) {
+ models::TextEncoderLoadParams params{
+ read_model_params(args),
+ TokenizerConfig(string_option(args, "tokenizer")),
+ string_option(args, "vision_path"),
+ string_option(args, "projection_path"),
+ string_option(args, "connectors_path"),
+ string_option(args, "t5_path"),
+ string_option(args, "clip_l_path"),
+ string_option(args, "clip_g_path"),
+ };
+ PreparedResource prepared;
+ prepared.key = normalized_model_params(params.model, prepared);
+ for (const auto& [name, member] : {std::pair{"vision_path", &models::TextEncoderLoadParams::vision_path},
+ {"projection_path", &models::TextEncoderLoadParams::projection_path},
+ {"connectors_path", &models::TextEncoderLoadParams::connectors_path},
+ {"t5_path", &models::TextEncoderLoadParams::t5_path},
+ {"clip_l_path", &models::TextEncoderLoadParams::clip_l_path},
+ {"clip_g_path", &models::TextEncoderLoadParams::clip_g_path}}) {
+ auto& path = params.*member;
+ path = file_key(path, prepared.revision);
+ prepared.key[name] = path;
+ }
+ auto paths = params.tokenizer.paths();
+ for (auto& path : paths)
+ path = file_key(path, prepared.revision);
+ prepared.key["tokenizer"] = paths;
+ params.tokenizer = TokenizerConfig(std::move(paths));
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ const auto path = params.model.path;
+ return make_model<models::TextEncoder>(request, std::move(params), path);
+ };
+ return prepared;
+ }
+
+ PreparedResource load_vae(Values& args) {
+ static const std::map<std::string, sd_vae_format_t> formats{
+ {"auto", SD_VAE_FORMAT_AUTO},
+ {"flux", SD_VAE_FORMAT_FLUX},
+ {"sd3", SD_VAE_FORMAT_SD3},
+ {"flux2", SD_VAE_FORMAT_FLUX2},
+ {"wan", SD_VAE_FORMAT_WAN},
+ };
+ models::VAELoadParams params{
+ read_model_params(args),
+ formats.at(get_string(args, "vae_format")),
+ };
+ PreparedResource prepared;
+ prepared.key = normalized_model_params(params.model, prepared);
+ prepared.key["format"] = int(params.format);
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ const auto path = params.model.path;
+ return make_model<models::VAE>(request, std::move(params), path);
+ };
+ return prepared;
+ }
+
+ PreparedResource load_tae(Values& args) {
+ auto params = read_model_params(args);
+ const bool decode_only = get_bool(args, "decode_only");
+ PreparedResource prepared;
+ prepared.key = normalized_model_params(params, prepared);
+ prepared.key["decode_only"] = decode_only;
+ prepared.load = [params = std::move(params), decode_only](ResourceLoadRequest& request) mutable {
+ auto model = request.context.make_model<models::VAE>(request.weights_key);
+ if (!model->init_tae(std::move(params), decode_only))
+ throw std::runtime_error("cannot load TAE");
+ return Value::object(std::move(model));
+ };
+ return prepared;
+ }
+
+ template <class T>
+ static PreparedResource prepare_model(Values& args) {
+ auto params = read_model_params(args);
+ PreparedResource prepared;
+ prepared.key = normalized_model_params(params, prepared);
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ const auto path = params.path;
+ return make_model<T>(request, std::move(params), path);
+ };
+ return prepared;
+ }
+
+ PreparedResource load_controlnet(Values& args) {
+ return prepare_model<models::ControlNet>(args);
+ }
+
+ PreparedResource load_vision_encoder(Values& args) {
+ return prepare_model<models::VisionEncoder>(args);
+ }
+
+ PreparedResource load_audio_encoder(Values& args) {
+ return prepare_model<models::AudioEncoder>(args);
+ }
+
+ PreparedResource load_audio_vae(Values& args) {
+ return prepare_model<models::AudioVAE>(args);
+ }
+
+ PreparedResource load_ip_adapter(Values& args) {
+ return prepare_model<models::IPAdapter>(args);
+ }
+
+ PreparedResource load_photomaker(Values& args) {
+ return prepare_model<models::PhotoMaker>(args);
+ }
+
+ PreparedResource load_pulid(Values& args) {
+ return prepare_model<models::PuLID>(args);
+ }
+
+ PreparedResource load_animatediff(Values& args) {
+ return prepare_model<models::AnimateDiff>(args);
+ }
+
+ PreparedResource load_latent_upscaler(Values& args) {
+ return prepare_model<models::LatentUpscaler>(args);
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/component_loader.h b/src/dgl/bindings/component_loader.h
new file mode 100644
index 0000000..997d44f
--- /dev/null
+++ b/src/dgl/bindings/component_loader.h
@@ -0,0 +1,24 @@
+#ifndef __SD_DGL_BINDINGS_COMPONENT_LOADER_H__
+#define __SD_DGL_BINDINGS_COMPONENT_LOADER_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct PreparedResource;
+
+ PreparedResource load_dit(Values& args);
+ PreparedResource load_text_encoder(Values& args);
+ PreparedResource load_vae(Values& args);
+ PreparedResource load_tae(Values& args);
+ PreparedResource load_controlnet(Values& args);
+ PreparedResource load_vision_encoder(Values& args);
+ PreparedResource load_audio_encoder(Values& args);
+ PreparedResource load_audio_vae(Values& args);
+ PreparedResource load_ip_adapter(Values& args);
+ PreparedResource load_photomaker(Values& args);
+ PreparedResource load_pulid(Values& args);
+ PreparedResource load_animatediff(Values& args);
+ PreparedResource load_latent_upscaler(Values& args);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_COMPONENT_LOADER_H__
diff --git a/src/dgl/bindings/components.cpp b/src/dgl/bindings/components.cpp
new file mode 100644
index 0000000..7d840d5
--- /dev/null
+++ b/src/dgl/bindings/components.cpp
@@ -0,0 +1,290 @@
+#include "dgl/bindings/components.h"
+
+#include <climits>
+#include <cmath>
+#include <type_traits>
+
+#include "conditioning/conditioner.hpp"
+#include "core/rng.hpp"
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/audio.h"
+#include "dgl/bindings/condition.h"
+#include "dgl/bindings/values.h"
+#include "dgl/bindings/video.h"
+#include "dgl/context.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/photomaker.h"
+#include "dgl/models/photomaker_prompt.h"
+#include "dgl/models/text_encoder.h"
+#include "dgl/models/vae.h"
+#include "dgl/ops/media.h"
+#include "dgl/ops/predictor.h"
+#include "dgl/ops/preview.h"
+#include "dgl/ops/prompt.h"
+#include "dgl/ops/sampling.h"
+#include "dgl/ops/vae.h"
+#include "extensions/pulid_embedding.h"
+#include "model/vae/vae.hpp"
+#include "model_definition.h"
+#include "model_loader.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl {
+ static ops::PromptMedia prompt_media(const Value& input) {
+ ops::PromptMedia result;
+ if (input.type().kind != Kind::Dict) {
+ result.media = pixels(input);
+ return result;
+ }
+ const Value* media = nullptr;
+ std::string kind;
+ for (const char* name : {"image", "video", "audio"}) {
+ if (const auto found = input.fields.find(name); found != input.fields.end()) {
+ if (media)
+ throw std::runtime_error("a media dict requires exactly one image, video or audio");
+ media = &found->second;
+ kind = name;
+ }
+ }
+ if (!media)
+ throw std::runtime_error("a media dict requires exactly one image, video or audio");
+ for (const auto& field : input.fields)
+ if (field.first != kind && field.first != "stride")
+ throw std::runtime_error("unknown media field '" + field.first + "'");
+ if (input.fields.count("stride")) {
+ if (kind != "video" || input.fields.at("stride").type().kind != Kind::Integer)
+ throw std::runtime_error("media stride requires a video and an integer value");
+ result.stride = get_int(input.fields, "stride");
+ if (result.stride < 1 || result.stride > 65536)
+ throw std::runtime_error("media stride must be in [1,65536]");
+ }
+ if (kind == "video") {
+ auto video = read_video_dict(*media);
+ result.media = std::move(video.frames);
+ result.fps_num = video.fps_num;
+ result.fps_den = video.fps_den;
+ } else if (kind == "audio") {
+ result.media = read_audio_dict(*media);
+ } else {
+ result.media = pixels(*media);
+ }
+ return result;
+ }
+
+ Value encode_prompt(Values& args, RunState&) {
+ ops::PromptEncodingOptions options;
+ options.text = get_string(args, "text");
+ options.width = int(get_int(args, "width"));
+ options.height = int(get_int(args, "height"));
+ options.clip_skip = int(get_int(args, "clip_skip"));
+ options.raw = get_bool(args, "raw");
+ if (args.count("system"))
+ options.system = get_string(args, "system");
+ if (options.raw && (args.count("media") || args.count("image_bindings")))
+ throw std::runtime_error("raw prompt encoding accepts text only; omit media and image_bindings");
+ if (args.count("media")) {
+ options.media.emplace();
+ for (const auto& input : args.at("media").items) {
+ try {
+ options.media->push_back(prompt_media(input));
+ } catch (const std::exception& error) {
+ throw std::runtime_error("media[" + std::to_string(options.media->size()) + "]: " + error.what());
+ }
+ }
+ }
+ if (args.count("image_bindings")) {
+ options.image_bindings = [&bindings = args.at("image_bindings").fields](const std::string& source) {
+ auto found = bindings.find(source);
+ if (found == bindings.end())
+ return sd::Tensor<float>{};
+ if (found->second.type().kind != Kind::Tensor)
+ throw std::runtime_error("image binding '" + source + "' must be a tensor");
+ auto image = pixels(found->second);
+ if (image.empty())
+ throw std::runtime_error("image binding '" + source + "' must not be empty");
+ return image;
+ };
+ }
+ return condition_value(ops::encode_prompt(*object<TextEncoder>(args, "encoder"), std::move(options)));
+ }
+
+ Value photomaker_tokens(Values& args, RunState&) {
+ auto model = object<PhotoMaker>(args, "photomaker");
+ Model::ExecutionScope scope(*model);
+ scope.require_ready();
+ std::vector<sd::Tensor<float>> images;
+ for (const auto& value : args.at("images").items)
+ images.push_back(pixels(value));
+ const auto* embedding = args.count("id_embedding") ? &float_tensor(args, "id_embedding") : nullptr;
+ auto tokens = model->project_images(images, embedding);
+ if (tokens.empty())
+ throw std::runtime_error("PhotoMaker image encoding failed");
+ return tensor_value(std::move(tokens));
+ }
+
+ Value encode_photomaker_prompt(Values& args, RunState&) {
+ auto encoder = object<TextEncoder>(args, "encoder");
+ ConditionerParams condition;
+ condition.text = get_string(args, "text");
+ condition.width = int(get_int(args, "width"));
+ condition.height = int(get_int(args, "height"));
+ condition.clip_skip = int(get_int(args, "clip_skip"));
+ auto output = encoder->encode_photomaker_prompt(std::move(condition), float_tensor(args, "tokens"));
+ return Value::dict({{"plain", condition_value(std::move(output.plain))},
+ {"condition", condition_value(std::move(output.condition))},
+ {"mask", Value::object(make_tensor(std::move(output.mask)))}});
+ }
+
+ Value apply_photomaker(Values& args, RunState&) {
+ auto fields = args.at("condition").fields;
+ if (!fields.count("crossattn"))
+ throw std::runtime_error("PhotoMaker requires a condition with crossattn");
+ auto model = object<PhotoMaker>(args, "photomaker");
+ Model::ExecutionScope scope(*model);
+ scope.require_ready();
+ auto fused = model->fuse_prompt(float_tensor(fields, "crossattn"), float_tensor(args, "tokens"),
+ read_tensor<uint8_t>(args.at("mask")));
+ if (fused.empty())
+ throw std::runtime_error("PhotoMaker condition fusion failed");
+ fields.insert_or_assign("crossattn", tensor_value(std::move(fused)));
+ return Value::dict(std::move(fields));
+ }
+
+ static ops::VAEOptions vae_options(const Values& args) {
+ ops::VAEOptions options;
+ options.circular_x = get_bool(args, "circular_x");
+ options.circular_y = get_bool(args, "circular_y");
+ if (args.count("tiling"))
+ options.tiling = tiling_options(args.at("tiling").fields);
+ return options;
+ }
+
+ Value vae_encode(Values& args, RunState&) {
+ const auto rng = object<RNG>(args, "rng");
+ auto& engine = *object<VAE>(args, "vae");
+ sd::Tensor<float> decoded;
+ const auto& input = pixels(args.at("pixels"), decoded);
+ auto output = ops::encode_vae(engine, input, rng, get_string(args, "posterior") == "mode", vae_options(args));
+ return tensor_value(std::move(output));
+ }
+
+ static Value convert_vae_latent(Values& args, bool to_dit) {
+ auto& engine = *object<VAE>(args, "vae");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ const auto& input = float_tensor(args, "latent");
+ engine.validate_vae_latent(input);
+ auto output = to_dit ? engine.vae_to_dit_latents(input) : engine.dit_to_vae_latents(input);
+ if (output.empty()) {
+ throw std::runtime_error("VAE latent conversion failed");
+ }
+ return tensor_value(std::move(output));
+ }
+
+ Value vae_decode(Values& args, RunState&) {
+ auto& engine = *object<VAE>(args, "vae");
+ auto output = ops::decode_vae(engine, float_tensor(args, "latent"), vae_options(args));
+ return get_string(args, "dtype") == "f32" ? tensor_value(std::move(output)) : image_to_u8(output);
+ }
+
+ Value latent_shape(Values& args, RunState&) {
+ auto& engine = *object<VAE>(args, "vae");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ return shape_value(engine.latent_shape(get_int(args, "width"), get_int(args, "height"),
+ args.count("frames") ? int(get_int(args, "frames")) : 0,
+ int(get_int(args, "batch"))));
+ }
+
+ static int schedule_sequence_length(const Values& args, const DiT& model) {
+ const auto sequence_length = args.count("sequence_length") ? int(get_int(args, "sequence_length")) : 0;
+ if (!args.count("latent"))
+ return sequence_length;
+ const auto& shape = object<sd_tensor_t>(args, "latent")->shape();
+ if (shape.size() != 4 && shape.size() != 5)
+ throw std::runtime_error("expected an image or video latent");
+ validate_float_shape(shape);
+ if (sequence_length > 0)
+ return sequence_length;
+ const auto patch = model.diffusion_model->get_patch_size();
+ auto length = ((shape[0] + patch - 1) / patch) * ((shape[1] + patch - 1) / patch);
+ if (shape.size() == 5 && (sd_version_is_ltxav(model.version()) || sd_version_is_minimax_h3(model.version())))
+ length *= shape[2];
+ if (length <= 0 || length > INT_MAX)
+ throw std::runtime_error("invalid diffusion sequence length");
+ return int(length);
+ }
+
+ Value t_to_sigma(Values& args, RunState&) {
+ const auto predict = object<ops::Predictor>(args, "predict");
+ const auto& leaf = ops::schedule_predictor(*predict);
+ auto& model = leaf.dit();
+ const auto sigma = leaf.resolved_parameterization().schedule_sigma(get_number(args, "t"), float(get_number(args, "flow_shift")),
+ schedule_sequence_length(args, model));
+ if (!std::isfinite(sigma) || sigma < 0.f)
+ throw std::runtime_error("model time mapping did not yield a finite nonnegative sigma");
+ return Value::literal(sigma);
+ }
+
+ Value denoiser_scalings(Values& args, RunState&) {
+ const auto predict = object<ops::Predictor>(args, "predict");
+ const auto scales = ops::scaling_parameterization(*predict).scalings(float(get_number(args, "sigma")));
+ return Value::dict({{"skip_scale", Value::literal(scales[0])},
+ {"output_scale", Value::literal(scales[1])},
+ {"input_scale", Value::literal(scales[2])}});
+ }
+
+ Value make_sigmas(Values& args, RunState&) {
+ const auto predict = object<ops::Predictor>(args, "predict");
+ const auto& leaf = ops::schedule_predictor(*predict);
+ auto& engine = leaf.dit();
+ const auto seq_len = schedule_sequence_length(args, engine);
+ const auto name = get_string(args, "scheduler");
+ const auto scheduler = str_to_scheduler(name.c_str());
+ if (scheduler == SCHEDULER_COUNT) {
+ throw std::runtime_error("unknown scheduler '" + name + "'");
+ }
+ const auto extra = get_string(args, "extra_args");
+ auto schedule = leaf.resolved_parameterization().schedule(float(get_number(args, "flow_shift")));
+ auto sigmas = schedule->get_sigmas(int(get_int(args, "steps")), seq_len, scheduler, engine.version(), extra.c_str());
+ return tensor_value(sd::Tensor<float>::from_vector(std::move(sigmas)));
+ }
+
+ Value sample_tensor(Values& args, RunState& state) {
+ const auto rng = object<RNG>(args, "rng");
+ const auto predictor = object<ops::Predictor>(args, "predict");
+ const auto& input = float_tensor(args, "start");
+ const auto& sigmas = ops::sampling_schedule(sigma_tensor(args, "sigmas"));
+ const auto options = sigmas.size() == 1 ? ops::SamplingOptions{} : sampling_options(args, state.context);
+ return tensor_value(ops::sample(input, sigmas, *predictor, rng, options, state.events));
+ }
+
+ Value execute_load_pulid_embedding(Values& args, RunState&) {
+ auto path = get_string(args, "path");
+ require_file_access(path, SD_DGL_FILE_READ);
+ auto result = load_pulid_id_embedding(path.c_str());
+ if (result.empty()) {
+ throw std::runtime_error("cannot load PuLID identity embedding");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value execute_vae_to_dit_latents(Values& args, RunState&) {
+ return convert_vae_latent(args, true);
+ }
+
+ Value execute_dit_to_vae_latents(Values& args, RunState&) {
+ return convert_vae_latent(args, false);
+ }
+
+ Value execute_add_noise(Values& args, RunState&) {
+ const auto predict = object<ops::Predictor>(args, "predict");
+ const auto& base = float_tensor(args, "base");
+ const auto& noise = float_tensor(args, "noise");
+ if (base.shape() != noise.shape()) {
+ throw std::runtime_error("base and noise shapes differ");
+ }
+ return tensor_value(ops::sampling_parameterization(*predict).add_noise(float(get_number(args, "sigma")), noise, base));
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/components.h b/src/dgl/bindings/components.h
new file mode 100644
index 0000000..4afb78a
--- /dev/null
+++ b/src/dgl/bindings/components.h
@@ -0,0 +1,26 @@
+#ifndef __SD_DGL_BINDINGS_COMPONENTS_H__
+#define __SD_DGL_BINDINGS_COMPONENTS_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value encode_prompt(Values& args, RunState&);
+ Value photomaker_tokens(Values& args, RunState&);
+ Value encode_photomaker_prompt(Values& args, RunState&);
+ Value apply_photomaker(Values& args, RunState&);
+ Value vae_encode(Values& args, RunState&);
+ Value vae_decode(Values& args, RunState&);
+ Value latent_shape(Values& args, RunState&);
+ Value t_to_sigma(Values& args, RunState&);
+ Value denoiser_scalings(Values& args, RunState&);
+ Value make_sigmas(Values& args, RunState&);
+ Value sample_tensor(Values& args, RunState& state);
+ Value execute_load_pulid_embedding(Values& args, RunState&);
+ Value execute_vae_to_dit_latents(Values& args, RunState&);
+ Value execute_dit_to_vae_latents(Values& args, RunState&);
+ Value execute_add_noise(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_COMPONENTS_H__
diff --git a/src/dgl/bindings/condition.cpp b/src/dgl/bindings/condition.cpp
new file mode 100644
index 0000000..7dc9ae5
--- /dev/null
+++ b/src/dgl/bindings/condition.cpp
@@ -0,0 +1,278 @@
+#include "dgl/bindings/condition.h"
+
+#include <algorithm>
+#include <iterator>
+
+#include "conditioning/conditioner.hpp"
+#include "dgl/bindings/arguments.h"
+#include "dgl/ops/tensor.h"
+
+namespace sd::dgl {
+
+ static bool tensor_field(const Value& value) {
+ if (value.type().kind == Kind::Tensor)
+ return true;
+ return value.type() == Type{Kind::List, Kind::Tensor};
+ }
+
+ static Value zero_field(const Value& value) {
+ if (value.type().kind == Kind::Tensor) {
+ const auto input = value.as<sd_tensor_t>();
+ return Value::object(ops::filled_tensor(input->shape(), input->type, int64_t(0)));
+ }
+ std::vector<Value> items;
+ items.reserve(value.items.size());
+ for (const auto& item : value.items)
+ items.push_back(zero_field(item));
+ return Value::list(std::move(items), Kind::Tensor);
+ }
+
+ Value zero_out_condition(Values& args, RunState&) {
+ auto fields = args.at("condition").fields;
+ const auto selected = args.find("fields");
+ if (selected == args.end()) {
+ for (auto& [name, value] : fields)
+ if (tensor_field(value))
+ value = zero_field(value);
+ } else {
+ for (const auto& item : selected->second.items) {
+ const auto name = item.scalar.get<std::string>();
+ const auto found = fields.find(name);
+ if (found == fields.end())
+ throw std::runtime_error("zero_out: condition has no field '" + name + "'");
+ if (!tensor_field(found->second))
+ throw std::runtime_error("zero_out: field '" + name + "' must be a tensor or tensor list");
+ found->second = zero_field(found->second);
+ }
+ }
+ return Value::dict(std::move(fields));
+ }
+
+ template <auto Member>
+ static std::optional<Value> condition_tensor(SDCondition& condition) {
+ auto& tensor = condition.*Member;
+ if (tensor.empty())
+ return std::nullopt;
+ return Value::object(make_tensor(std::move(tensor)));
+ }
+
+ template <auto Member>
+ static std::optional<Value> condition_tensors(SDCondition& condition) {
+ auto& tensors = condition.*Member;
+ if (tensors.empty())
+ return std::nullopt;
+ std::vector<Value> items;
+ items.reserve(tensors.size());
+ for (auto& tensor : tensors) {
+ if (tensor.empty())
+ throw std::runtime_error("conditioning contains an empty tensor list item");
+ items.push_back(Value::object(make_tensor(std::move(tensor))));
+ }
+ return Value::list(std::move(items), Kind::Tensor);
+ }
+
+ static std::optional<Value> image_embeds(SDCondition& condition) {
+ if (condition.c_image_embeds.empty())
+ return std::nullopt;
+ std::vector<Value> items;
+ for (auto& [index, tensor] : condition.c_image_embeds) {
+ if (tensor.empty())
+ throw std::runtime_error("conditioning contains an empty image embedding");
+ items.push_back(Value::object(make_tensor(std::move(tensor))));
+ }
+ return Value::list(std::move(items), Kind::Tensor);
+ }
+
+ static std::optional<Value> image_embed_indices(SDCondition& condition) {
+ if (condition.c_image_embeds.empty())
+ return std::nullopt;
+ std::vector<Value> items;
+ for (const auto& entry : condition.c_image_embeds)
+ items.push_back(Value::object(make_tensor(sd::Tensor<int32_t>::from_vector({entry.first}))));
+ return Value::list(std::move(items), Kind::Tensor);
+ }
+
+ static const std::string reference_kinds[] = {"image", "video", "audio", "video_audio"};
+
+ static std::optional<Value> reference_blocks(SDCondition& condition) {
+ if (condition.c_reference_blocks.empty())
+ return std::nullopt;
+ std::vector<Value> blocks;
+ blocks.reserve(condition.c_reference_blocks.size());
+ for (const auto& block : condition.c_reference_blocks) {
+ const auto kind = static_cast<size_t>(block.kind);
+ if (kind >= std::size(reference_kinds))
+ throw std::runtime_error("invalid conditioning reference block kind");
+ Values fields{{"kind", Value::literal(reference_kinds[kind])}};
+ if (block.kind != MiniMaxH3ReferenceKind::AUDIO)
+ fields.emplace("video", Value::literal(block.video_index));
+ if (block.kind == MiniMaxH3ReferenceKind::AUDIO || block.kind == MiniMaxH3ReferenceKind::VIDEO_AUDIO)
+ fields.emplace("audio", Value::literal(block.audio_index));
+ blocks.push_back(Value::dict(std::move(fields)));
+ }
+ return Value::list(std::move(blocks), Kind::Dict);
+ }
+
+ static std::vector<MiniMaxH3ReferenceBlock> read_reference_blocks(const Value& value) {
+ if (value.type().kind != Kind::List)
+ throw std::runtime_error("reference_blocks must be a list of dicts");
+ std::vector<MiniMaxH3ReferenceBlock> blocks;
+ blocks.reserve(value.items.size());
+ for (size_t i = 0; i < value.items.size(); ++i) {
+ const auto& item = value.items[i];
+ const auto label = "reference_blocks[" + std::to_string(i) + "]";
+ if (item.type().kind != Kind::Dict)
+ throw std::runtime_error(label + " must be a dict");
+ const auto& fields = item.fields;
+ const auto kind = fields.find("kind");
+ if (kind == fields.end() || kind->second.type().kind != Kind::String)
+ throw std::runtime_error(label + ".kind must be a string");
+ const auto name = kind->second.scalar.get<std::string>();
+ const auto found = std::find(std::begin(reference_kinds), std::end(reference_kinds), name);
+ if (found == std::end(reference_kinds))
+ throw std::runtime_error(label + ".kind must be image, video, audio or video_audio, got '" + name + "'");
+ MiniMaxH3ReferenceBlock block;
+ block.kind = MiniMaxH3ReferenceKind(found - std::begin(reference_kinds));
+ auto index = [&](const char* field, bool required) {
+ const auto entry = fields.find(field);
+ if (!required) {
+ if (entry != fields.end())
+ throw std::runtime_error(label + " of kind '" + name + "' does not accept '" + field + "'");
+ return int32_t(-1);
+ }
+ if (entry == fields.end())
+ throw std::runtime_error(label + " of kind '" + name + "' requires '" + field + "'");
+ if (entry->second.type().kind != Kind::Integer)
+ throw std::runtime_error(label + "." + field + " must be an integer");
+ const auto number = entry->second.scalar.get<int64_t>();
+ if (number < 0 || number > INT32_MAX)
+ throw std::runtime_error(label + "." + field + " must be between 0 and 2147483647");
+ return int32_t(number);
+ };
+ block.video_index = index("video", block.kind != MiniMaxH3ReferenceKind::AUDIO);
+ block.audio_index = index("audio", block.kind == MiniMaxH3ReferenceKind::AUDIO || block.kind == MiniMaxH3ReferenceKind::VIDEO_AUDIO);
+ for (const auto& [field, entry] : fields)
+ if (field != "kind" && field != "video" && field != "audio")
+ throw std::runtime_error(label + " has unknown field '" + field + "'");
+ blocks.push_back(block);
+ }
+ return blocks;
+ }
+
+ void validate_reference_blocks(const SDCondition& condition, size_t video_count) {
+ for (size_t i = 0; i < condition.c_reference_blocks.size(); ++i) {
+ const auto& block = condition.c_reference_blocks[i];
+ const auto label = "reference_blocks[" + std::to_string(i) + "]";
+ if (block.video_index >= 0 && size_t(block.video_index) >= video_count)
+ throw std::runtime_error(label + ".video index " + std::to_string(block.video_index) + " is out of range for " + std::to_string(video_count) + " reference latents");
+ if (block.audio_index >= 0 && size_t(block.audio_index) >= condition.c_ref_audios.size())
+ throw std::runtime_error(label + ".audio index " + std::to_string(block.audio_index) + " is out of range for " + std::to_string(condition.c_ref_audios.size()) + " reference audios");
+ }
+ }
+
+ struct ConditionField {
+ const char* name;
+ OutputSignature output;
+ std::optional<Value> (*extract)(SDCondition&);
+ void (*assign)(SDCondition&, const Value&) = nullptr;
+ };
+
+ template <typename T>
+ static void assign_condition_tensor(sd::Tensor<T>& target, const Value& value) {
+ target = read_tensor<T>(value);
+ }
+
+ template <auto Member>
+ static ConditionField tensor_field(const char* name) {
+ return {name, {Kind::Tensor}, condition_tensor<Member>, [](SDCondition& condition, const Value& value) {
+ assign_condition_tensor(condition.*Member, value);
+ }};
+ }
+
+ template <auto Member>
+ static ConditionField tensor_list_field(const char* name) {
+ return {name, {Kind::List, Kind::Tensor}, condition_tensors<Member>, [](SDCondition& condition, const Value& value) {
+ condition.*Member = float_tensor_list(value);
+ }};
+ }
+
+ static const ConditionField condition_fields[] = {
+ tensor_field<&SDCondition::c_crossattn>("crossattn"),
+ tensor_field<&SDCondition::c_vector>("vector"),
+ tensor_field<&SDCondition::c_concat>("concat"),
+ tensor_field<&SDCondition::c_t5_ids>("t5_ids"),
+ tensor_field<&SDCondition::c_t5_weights>("t5_weights"),
+ tensor_field<&SDCondition::c_input_ids>("input_ids"),
+ tensor_field<&SDCondition::c_position_ids>("position_ids"),
+ tensor_field<&SDCondition::c_token_types>("token_types"),
+ tensor_field<&SDCondition::c_vinput_mask>("vinput_mask"),
+ tensor_list_field<&SDCondition::extra_c_crossattns>("extra_crossattn"),
+ {"image_embeds", {Kind::List, Kind::Tensor}, image_embeds},
+ {"image_embed_indices", {Kind::List, Kind::Tensor}, image_embed_indices},
+ tensor_list_field<&SDCondition::c_ref_images>("ref_images"),
+ tensor_list_field<&SDCondition::c_ref_audios>("ref_audios"),
+ {"reference_blocks", {{Kind::List, Kind::Dict}, {{"kind", {Kind::String}}, {"video", {Kind::Integer}}, {"audio", {Kind::Integer}}}}, reference_blocks},
+ };
+
+ OutputSignature condition_signature() {
+ OutputSignature signature{Kind::Dict};
+ for (const auto& field : condition_fields)
+ signature.fields.emplace(field.name, field.output);
+ return signature;
+ }
+
+ Value condition_value(SDCondition condition) {
+ Values fields;
+ for (const auto& field : condition_fields)
+ if (auto value = field.extract(condition))
+ fields.emplace(field.name, std::move(*value));
+ return Value::dict(std::move(fields));
+ }
+
+ SDCondition read_condition(const Values& fields, const Values& overrides) {
+ SDCondition c;
+ for (const auto& field : condition_fields) {
+ auto value = fields.find(field.name);
+ if (field.assign && value != fields.end())
+ field.assign(c, value->second);
+ }
+ if (fields.count("image_embeds") && fields.count("image_embed_indices")) {
+ const auto embeds = float_tensor_list(fields.at("image_embeds"));
+ const auto& indices = fields.at("image_embed_indices").items;
+ if (embeds.size() != indices.size()) {
+ throw std::runtime_error("image embeddings and indices differ in length");
+ }
+ for (size_t i = 0; i < embeds.size(); ++i) {
+ const auto& index = read_tensor<int32_t>(indices[i]);
+ if (index.numel() != 1 || index[0] < 0) {
+ throw std::runtime_error("invalid image embedding index");
+ }
+ c.c_image_embeds.emplace_back(index[0], embeds[i]);
+ }
+ }
+ if (overrides.count("ref_audios")) {
+ c.c_ref_audios = float_tensor_list(overrides.at("ref_audios"));
+ }
+ if (overrides.count("keyframe_indices")) {
+ std::vector<int32_t> indices;
+ for (const auto& index : overrides.at("keyframe_indices").items) {
+ auto value = index.scalar.get<int64_t>();
+ if (value < 0 || value > INT32_MAX) {
+ throw std::runtime_error("invalid keyframe index");
+ }
+ indices.push_back(int32_t(value));
+ }
+ c.c_position_ids = sd::Tensor<int32_t>::from_vector(std::move(indices));
+ }
+ const auto* block = overrides.count("reference_blocks") ? &overrides.at("reference_blocks")
+ : fields.count("reference_blocks") ? &fields.at("reference_blocks")
+ : nullptr;
+ if (block)
+ c.c_reference_blocks = read_reference_blocks(*block);
+ if (overrides.count("concat")) {
+ c.c_concat = float_tensor(overrides, "concat");
+ }
+ return c;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/condition.h b/src/dgl/bindings/condition.h
new file mode 100644
index 0000000..5ed9e7c
--- /dev/null
+++ b/src/dgl/bindings/condition.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_BINDINGS_CONDITION_H__
+#define __SD_DGL_BINDINGS_CONDITION_H__
+
+#include "dgl/registry.h"
+
+struct SDCondition;
+
+namespace sd::dgl {
+ OutputSignature condition_signature();
+ Value zero_out_condition(Values& args, RunState& state);
+ Value condition_value(SDCondition condition);
+ SDCondition read_condition(const Values& fields, const Values& overrides = {});
+ void validate_reference_blocks(const SDCondition& condition, size_t video_count);
+}
+
+#endif // __SD_DGL_BINDINGS_CONDITION_H__
diff --git a/src/dgl/bindings/conversion.cpp b/src/dgl/bindings/conversion.cpp
new file mode 100644
index 0000000..696838d
--- /dev/null
+++ b/src/dgl/bindings/conversion.cpp
@@ -0,0 +1,39 @@
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ const sd_tensor_t& i64_vector(const Value& value) {
+ const auto& tensor = *value.as<sd_tensor_t>();
+ if (tensor.type != SD_TENSOR_I64 || tensor.shape().size() != 1)
+ throw std::runtime_error("expected a rank-one i64 tensor");
+ return tensor;
+ }
+
+ std::vector<int64_t> read_int_vector(const Value& value) {
+ if (value.type().kind == Kind::Tensor) {
+ const auto& tensor = i64_vector(value);
+ if (!tensor.bytes)
+ return {};
+ const auto* data = static_cast<const int64_t*>(tensor.data());
+ return {data, data + tensor.shape()[0]};
+ }
+ std::vector<int64_t> result;
+ for (const auto& item : value.items)
+ result.push_back(item.scalar.get<int64_t>());
+ return result;
+ }
+
+ Value shape_value(const std::vector<int64_t>& shape) {
+ return Value::object(make_tensor(SD_TENSOR_I64, std::vector<int64_t>{int64_t(shape.size())}, shape.data(), shape.size() * sizeof(int64_t)));
+ }
+
+ Value tensor_value(sd::Tensor<float> tensor) {
+ return Value::object(make_tensor(std::move(tensor)));
+ }
+
+ sd::Image image_view(const Value& value) {
+ if (value.type().kind != Kind::Tensor)
+ throw std::runtime_error("expected an image tensor");
+ return image_view(*value.as<sd_tensor_t>());
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/conversion.h b/src/dgl/bindings/conversion.h
new file mode 100644
index 0000000..43e2292
--- /dev/null
+++ b/src/dgl/bindings/conversion.h
@@ -0,0 +1,45 @@
+#ifndef __SD_DGL_BINDINGS_CONVERSION_H__
+#define __SD_DGL_BINDINGS_CONVERSION_H__
+
+#include "core/file_access.h"
+#include "dgl/executor.h"
+#include "dgl/ops/audio.h"
+#include "dgl/ops/image.h"
+#include "dgl/ops/tensor.h"
+#include "dgl/ops/video.h"
+#include "dgl/tensor.h"
+#include "stable-diffusion.h"
+
+#include <filesystem>
+
+namespace sd::dgl {
+ using ops::allocate_image;
+ using ops::audio_view;
+ using ops::copy_audio;
+ using ops::copy_image;
+ using ops::crop_tensor;
+ using ops::decode_base64;
+ using ops::decode_pcm_audio;
+ using ops::decode_video_jpeg;
+ using ops::encode_pcm_audio;
+ using ops::encode_video_jpeg;
+ using ops::image_view;
+ using ops::interpolate_tensor;
+ using ops::read_image_tensor;
+ using ops::read_wav;
+ using ops::write_wav;
+
+ const sd_tensor_t& i64_vector(const Value& value);
+ std::vector<int64_t> read_int_vector(const Value& value);
+ Value shape_value(const std::vector<int64_t>& shape);
+ Value tensor_value(sd::Tensor<float> tensor);
+ sd::Image image_view(const Value& value);
+
+ template <class T>
+ std::shared_ptr<T> object(const Values& args, const std::string& name) {
+ return args.at(name).as<T>();
+ }
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_CONVERSION_H__
diff --git a/src/dgl/bindings/detailer.cpp b/src/dgl/bindings/detailer.cpp
new file mode 100644
index 0000000..b577520
--- /dev/null
+++ b/src/dgl/bindings/detailer.cpp
@@ -0,0 +1,41 @@
+#include "dgl/bindings/detailer.h"
+
+#include "dgl/bindings/conversion.h"
+
+#include "dgl/bindings/values.h"
+
+#include "dgl/ops/detailer.h"
+#include "dgl/ops/media.h"
+
+namespace sd::dgl {
+
+ Value plan_mask_crops(Values& args, RunState&) {
+ const auto ratio = read_int_vector(args.at("aspect"));
+ if (ratio.size() != 2 || ratio[0] <= 0 || ratio[1] <= 0 || ratio[0] > 16384 || ratio[1] > 16384)
+ throw std::runtime_error("aspect requires two positive integers up to 16384");
+ const auto aspect = float(ratio[0]) / float(ratio[1]);
+ const auto padding = int(get_int(args, "padding"));
+ const auto only_masked = get_bool(args, "only_masked");
+ const auto& inputs = args.at("masks").items;
+ const auto canvas = inputs.empty() ? sd::Image{} : image_view(inputs.front());
+ std::vector<Value> regions;
+ regions.reserve(inputs.size());
+ for (const auto& input : inputs) {
+ const auto mask = image_view(input);
+ if (mask.channel != 1)
+ throw std::runtime_error("mask regions require grayscale tensors");
+ if (mask.width != canvas.width || mask.height != canvas.height)
+ throw std::runtime_error("masks must match the same canvas dimensions");
+ ops::DetailCrop crop;
+ if (!ops::detail_crop_box(mask, aspect, padding, only_masked, crop))
+ continue;
+ regions.push_back(Value::dict({
+ {"box", shape_value({crop.x1, crop.y1, crop.x2 - crop.x1, crop.y2 - crop.y1})},
+ {"mask", input},
+ {"index", Value::literal(int64_t(regions.size()))},
+ }));
+ }
+ return Value::list(std::move(regions), Kind::Dict);
+ }
+
+}
diff --git a/src/dgl/bindings/detailer.h b/src/dgl/bindings/detailer.h
new file mode 100644
index 0000000..e8b9b0f
--- /dev/null
+++ b/src/dgl/bindings/detailer.h
@@ -0,0 +1,11 @@
+#ifndef __SD_DGL_BINDINGS_DETAILER_H__
+#define __SD_DGL_BINDINGS_DETAILER_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+ Value plan_mask_crops(Values& args, RunState&);
+}
+
+#endif // __SD_DGL_BINDINGS_DETAILER_H__
diff --git a/src/dgl/bindings/detections.cpp b/src/dgl/bindings/detections.cpp
new file mode 100644
index 0000000..dea6c51
--- /dev/null
+++ b/src/dgl/bindings/detections.cpp
@@ -0,0 +1,151 @@
+#include "dgl/bindings/detections.h"
+
+#include <cmath>
+
+#include "dgl/bindings/conversion.h"
+#include "dgl/bindings/regions.h"
+#include "dgl/models/detail_detector.h"
+#include "dgl/ops/detailer.h"
+
+namespace sd::dgl {
+ struct DetectionView {
+ const sd::Tensor<float>& xyxy;
+ const sd::Tensor<float>& scores;
+ const sd::Tensor<int64_t>& class_ids;
+ };
+
+ OutputSignature detections_signature() {
+ return {{Kind::Dict}, {{"xyxy", {Kind::Tensor}}, {"scores", {Kind::Tensor}}, {"class_ids", {Kind::Tensor}}}};
+ }
+
+ static DetectionView read_detections(const Value& value) {
+ if (value.type().kind != Kind::Dict)
+ throw std::runtime_error("detections require a dict with xyxy, scores and class_ids");
+ const auto& fields = value.fields;
+ auto tensor = [&](const char* name, sd_tensor_type_t dtype) -> const sd_tensor_t& {
+ const auto found = fields.find(name);
+ if (found == fields.end())
+ throw std::runtime_error(std::string("detections are missing field '") + name + "'");
+ if (found->second.type().kind != Kind::Tensor)
+ throw std::runtime_error(std::string("detections.") + name + " must be a tensor");
+ const auto& result = *found->second.as<sd_tensor_t>();
+ if (result.type != dtype)
+ throw std::runtime_error(std::string("detections.") + name + (dtype == SD_TENSOR_I64 ? " requires I64" : " requires F32"));
+ return result;
+ };
+ const auto& xyxy = tensor("xyxy", SD_TENSOR_F32);
+ const auto& scores = tensor("scores", SD_TENSOR_F32);
+ const auto& class_ids = tensor("class_ids", SD_TENSOR_I64);
+ if (xyxy.shape().size() != 2 || xyxy.shape()[0] != 4 || xyxy.shape()[1] > 4096)
+ throw std::runtime_error("detections.xyxy requires F32 [4,N] with at most 4096 boxes");
+ if (scores.shape() != std::vector<int64_t>{xyxy.shape()[1]} || class_ids.shape() != scores.shape())
+ throw std::runtime_error("detections.scores and class_ids require shape [N] matching xyxy");
+ static const sd::Tensor<float> empty_xyxy({4, 0}), empty_scores({0});
+ static const sd::Tensor<int64_t> empty_ids({0});
+ const bool populated = xyxy.shape()[1] != 0;
+ DetectionView result{populated ? xyxy.as<float>() : empty_xyxy,
+ populated ? scores.as<float>() : empty_scores,
+ populated ? class_ids.as<int64_t>() : empty_ids};
+ for (int64_t i = 0; i < xyxy.shape()[1]; ++i) {
+ for (int j = 0; j < 4; ++j)
+ if (!std::isfinite(result.xyxy[i * 4 + j]))
+ throw std::runtime_error("detection " + std::to_string(i) + " coordinates must be finite");
+ if (!std::isfinite(result.scores[i]) || result.scores[i] < 0 || result.scores[i] > 1)
+ throw std::runtime_error("detection " + std::to_string(i) + " score must be finite and in [0, 1]");
+ if (result.class_ids[i] < 0)
+ throw std::runtime_error("detection " + std::to_string(i) + " class_id must be nonnegative");
+ }
+ return result;
+ }
+
+ static Value detection_value(std::vector<float> xyxy, std::vector<float> scores, std::vector<int64_t> class_ids) {
+ const auto count = int64_t(scores.size());
+ return Value::dict({
+ {"xyxy", Value::object(make_tensor(SD_TENSOR_F32, std::vector<int64_t>{4, count}, xyxy.data(), xyxy.size() * sizeof(float)))},
+ {"scores", Value::object(make_tensor(SD_TENSOR_F32, std::vector<int64_t>{count}, scores.data(), scores.size() * sizeof(float)))},
+ {"class_ids", Value::object(make_tensor(SD_TENSOR_I64, std::vector<int64_t>{count}, class_ids.data(), class_ids.size() * sizeof(int64_t)))},
+ });
+ }
+
+ Value detect_objects(Values& args, RunState&) {
+ const auto image = image_view(args.at("image"));
+ models::DetectionOptions options;
+ options.input_size = int(get_int(args, "input_size"));
+ options.confidence = float(get_number(args, "confidence"));
+ options.nms_threshold = float(get_number(args, "nms"));
+ options.max_detections = int(get_int(args, "max_detections"));
+ const auto found = object<models::DetailDetector>(args, "detector")->predict(image, options);
+ std::vector<float> xyxy, scores;
+ std::vector<int64_t> class_ids;
+ xyxy.reserve(found.size() * 4);
+ scores.reserve(found.size());
+ class_ids.reserve(found.size());
+ for (const auto& d : found) {
+ xyxy.insert(xyxy.end(), {d.x1, d.y1, d.x2, d.y2});
+ scores.push_back(d.confidence);
+ class_ids.push_back(d.class_id);
+ }
+ auto result = detection_value(std::move(xyxy), std::move(scores), std::move(class_ids));
+ read_detections(result);
+ return result;
+ }
+
+ static std::vector<models::DetailDetection> detection_rows(const DetectionView& view, const sd::Image& image) {
+ std::vector<models::DetailDetection> rows;
+ for (int64_t i = 0; i < view.scores.numel(); ++i) {
+ const auto* d = view.xyxy.data() + i * 4;
+ if (d[0] < 0 || d[1] < 0 || d[2] < d[0] || d[3] < d[1] || d[2] > image.width || d[3] > image.height)
+ throw std::runtime_error("detection " + std::to_string(i) + " lies outside the image or has reversed corners");
+ rows.push_back({d[0], d[1], d[2], d[3], view.scores[i]});
+ }
+ return rows;
+ }
+
+ Value filter_detections(Values& args, RunState&) {
+ using ops::DetectionSort;
+ static const std::map<std::string, DetectionSort> orders{
+ {"none", DetectionSort::None}, {"left_to_right", DetectionSort::LeftToRight}, {"center_to_edge", DetectionSort::CenterToEdge}, {"area", DetectionSort::Area}};
+ ops::DetectionFilter filter;
+ filter.top_k = int(get_int(args, "top_k"));
+ filter.top_k_by_confidence = get_string(args, "top_k_by") == "confidence";
+ const auto min_ratio = get_number(args, "min_area_ratio"), max_ratio = get_number(args, "max_area_ratio");
+ if (min_ratio > max_ratio)
+ throw std::runtime_error("min_area_ratio must not exceed max_area_ratio");
+ filter.min_area_ratio = float(min_ratio);
+ filter.max_area_ratio = float(max_ratio);
+ filter.sort_by = orders.at(get_string(args, "sort_by"));
+ const auto image = image_view(args.at("image"));
+ const auto view = read_detections(args.at("detections"));
+ auto rows = detection_rows(view, image);
+ const auto indices = ops::filter_detection_indices(rows, image.width, image.height, filter);
+ std::vector<float> xyxy, scores;
+ std::vector<int64_t> class_ids;
+ for (const auto index : indices) {
+ const auto* box = view.xyxy.data() + index * 4;
+ xyxy.insert(xyxy.end(), box, box + 4);
+ scores.push_back(view.scores[index]);
+ class_ids.push_back(view.class_ids[index]);
+ }
+ return detection_value(std::move(xyxy), std::move(scores), std::move(class_ids));
+ }
+
+ Value detection_box_masks(Values& args, RunState&) {
+ const auto image = image_view(args.at("image"));
+ const auto rows = detection_rows(read_detections(args.at("detections")), image);
+ if (uint64_t(image.width) * image.height * rows.size() > 256 * 1024 * 1024)
+ throw std::runtime_error("detection masks exceed 256 MiB");
+ std::vector<Value> masks;
+ for (const auto& row : rows) {
+ const auto mask = ops::box_mask(row, image.width, image.height);
+ masks.push_back(Value::object(copy_image({uint32_t(mask.width), uint32_t(mask.height), 1, const_cast<uint8_t*>(mask.data.data())})));
+ }
+ return Value::list(std::move(masks), Kind::Tensor);
+ }
+
+ Value detections_to_regions(Values& args, RunState&) {
+ const auto image = image_view(args.at("image"));
+ const auto detections = read_detections(args.at("detections"));
+ auto boxes = ops::boxes_from_detections(detections.xyxy, image.width, image.height);
+ return regions_value(image.width, image.height, make_tensor(std::move(boxes)));
+ }
+}
diff --git a/src/dgl/bindings/detections.h b/src/dgl/bindings/detections.h
new file mode 100644
index 0000000..2f0dd89
--- /dev/null
+++ b/src/dgl/bindings/detections.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_BINDINGS_DETECTIONS_H__
+#define __SD_DGL_BINDINGS_DETECTIONS_H__
+
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ OutputSignature detections_signature();
+ Value detect_objects(Values& args, RunState&);
+ Value filter_detections(Values& args, RunState&);
+ Value detection_box_masks(Values& args, RunState&);
+ Value detections_to_regions(Values& args, RunState&);
+}
+
+#endif // __SD_DGL_BINDINGS_DETECTIONS_H__
diff --git a/src/dgl/bindings/image.cpp b/src/dgl/bindings/image.cpp
new file mode 100644
index 0000000..7d4cc8c
--- /dev/null
+++ b/src/dgl/bindings/image.cpp
@@ -0,0 +1,43 @@
+#include "dgl/bindings/image.h"
+
+#include "core/image_tensor.h"
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ Value execute_image_to_u8(Values& args, RunState&) {
+ sd::Tensor<float> decoded;
+ return image_to_u8(pixels(args.at("pixels"), decoded));
+ }
+
+ Value execute_image_to_f32(Values& args, RunState&) {
+ const auto& image = args.at("image");
+ sd::Tensor<float> decoded;
+ const auto& input = pixels(image, decoded);
+ const auto channels = get_int(args, "channels");
+ const auto current = input.shape()[input.dim() == 5 ? 3 : 2];
+ if (!channels || channels == current) {
+ if (&input != &decoded)
+ return image;
+ return tensor_value(std::move(decoded));
+ }
+ if ((channels != 3 && channels != 4) || current == 2) {
+ throw std::runtime_error("unsupported pixel channel conversion");
+ }
+ return tensor_value(sd::ensure_image_tensor_channels(input, int(channels)));
+ }
+
+ Value execute_load_image(Values& args, RunState&) {
+ return Value::object(read_image_tensor(get_string(args, "path")));
+ }
+ Value execute_load_images(Values& args, RunState&) {
+ std::vector<Value> images;
+ for (auto image : ops::read_image_directory(std::filesystem::u8path(get_string(args, "path")), size_t(get_int(args, "max_images")), 0))
+ images.push_back(Value::object(std::move(image)));
+ return Value::list(std::move(images));
+ }
+ Value execute_save_image(Values& args, RunState& state) {
+ ops::save_image(*object<sd_tensor_t>(args, "image"), get_string(args, "path"), state.metadata);
+ return Value::void_value();
+ }
+}
diff --git a/src/dgl/bindings/image.h b/src/dgl/bindings/image.h
new file mode 100644
index 0000000..71629ad
--- /dev/null
+++ b/src/dgl/bindings/image.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_BINDINGS_IMAGE_H__
+#define __SD_DGL_BINDINGS_IMAGE_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value execute_image_to_f32(Values& args, RunState&);
+ Value execute_image_to_u8(Values& args, RunState&);
+ Value execute_load_image(Values& args, RunState&);
+ Value execute_load_images(Values& args, RunState&);
+ Value execute_save_image(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_IMAGE_H__
diff --git a/src/dgl/bindings/latents.cpp b/src/dgl/bindings/latents.cpp
new file mode 100644
index 0000000..12b3572
--- /dev/null
+++ b/src/dgl/bindings/latents.cpp
@@ -0,0 +1,133 @@
+#include "dgl/bindings/latents.h"
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/audio.h"
+#include "dgl/bindings/values.h"
+#include "dgl/models/audio_encoder.h"
+#include "dgl/models/audio_vae.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/latent_upscaler.h"
+#include "dgl/models/vision_encoder.h"
+#include "dgl/ops/media.h"
+#include "model/vae/audio_vae.hpp"
+
+namespace sd::dgl {
+
+ Value encode_vision(Values& args, RunState&) {
+ auto& engine = *object<VisionEncoder>(args, "encoder");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ sd::Tensor<float> decoded;
+ const auto& image = pixels(args.at("image"), decoded);
+ if (image.dim() != 4 || image.shape()[2] == 2) {
+ throw std::runtime_error("vision encoding requires an image");
+ }
+ const auto& rgb = image.shape()[2] == 3
+ ? image
+ : (decoded = sd::ensure_image_tensor_channels(image, 3));
+ const auto zero = get_bool(args, "zero");
+ const auto zero_input = get_bool(args, "zero_input");
+ if (zero && zero_input)
+ throw std::runtime_error("encode_vision accepts either zero or zero_input");
+ auto result = engine.get_clip_vision_output(rgb, get_bool(args, "pooled"),
+ int(get_int(args, "clip_skip")), zero, zero_input);
+ if (result.empty()) {
+ throw std::runtime_error("vision encoding failed");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value encode_audio(Values& args, RunState&) {
+ const auto audio = audio_view(*object<sd_tensor_t>(args, "samples"), get_int(args, "sample_rate"));
+ auto& engine = *object<AudioEncoder>(args, "encoder");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ auto result = engine.get_audio_embedding(audio);
+ if (result.empty()) {
+ throw std::runtime_error("audio encoding failed");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value encode_ip_adapter(Values& args, RunState&) {
+ auto& engine = *object<IPAdapter>(args, "adapter");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ auto result = engine.project_image(float_tensor(args, "embedding"));
+ if (result.empty()) {
+ throw std::runtime_error("IP-Adapter projection failed");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value upscale_latent(Values& args, RunState&) {
+ auto upscaler = object<models::LatentUpscaler>(args, "upscaler");
+ Model::ExecutionScope scope(*upscaler);
+ scope.require_ready();
+ const auto& latent = float_tensor(args, "latent");
+ const auto width = args.count("width") ? get_int(args, "width") : 0;
+ const auto height = args.count("height") ? get_int(args, "height") : 0;
+ auto result = upscaler->upscale(latent, width, height);
+ if (result.empty()) {
+ throw std::runtime_error("learned latent upscaling failed");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value audio_vae_encode(Values& args, RunState&) {
+ const auto audio = audio_view(*object<sd_tensor_t>(args, "samples"), get_int(args, "sample_rate"));
+ auto& engine = *object<AudioVAE>(args, "vae");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ auto waveform = sd::dgl::ops::prepare_minimax_h3_reference_waveform(audio, engine.audio_vae_model->input_sample_rate());
+ auto result = engine.audio_vae_model->encode(engine.n_threads, waveform);
+ if (result.empty()) {
+ throw std::runtime_error("audio VAE encoding failed");
+ }
+ return tensor_value(std::move(result));
+ }
+
+ Value audio_vae_decode(Values& args, RunState&) {
+ auto& engine = *object<AudioVAE>(args, "vae");
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ auto waveform = engine.decode_ltx_audio_latent(float_tensor(args, "latent"));
+ std::unique_ptr<sd::Audio, decltype(&sd::free_audio)> audio(sd::dgl::ops::waveform_to_sd_audio(&engine, waveform), sd::free_audio);
+ if (!audio) {
+ throw std::runtime_error("audio VAE decoding failed");
+ }
+ return audio_result(copy_audio(*audio));
+ }
+
+ Value execute_audio_latent_shape(Values& args, RunState&) {
+ const auto& vae = *object<AudioVAE>(args, "vae");
+ return shape_value(vae.latent_shape(int(get_int(args, "frames")), int(get_int(args, "fps"))));
+ }
+
+ Value execute_ltx_video_positions(Values& args, RunState&) {
+ const auto& s = object<sd_tensor_t>(args, "like")->shape();
+ const auto keys = get_int(args, "keyframe_latent_frames");
+ if (s.size() != 5 || s[0] <= 0 || s[1] <= 0 || s[2] <= 0 || s[0] > 16384 || s[1] > 16384 || s[2] > 65536) {
+ throw std::runtime_error("expected a video latent for positions");
+ }
+ validate_float_shape({2, 3, s[0] * s[1] * (s[2] + keys), 1});
+ return tensor_value(sd::dgl::ops::build_ltxv_video_positions(
+ s[0], s[1], s[2], keys, int(get_int(args, "keyframe_frame_index")), int(get_int(args, "keyframe_pixel_frames")),
+ int(get_int(args, "fps")), int(get_int(args, "spatial_scale")), int(get_int(args, "temporal_scale")), get_bool(args, "causal")));
+ }
+
+ Value execute_s2v_audio_window(Values& args, RunState&) {
+ const auto& embedding = float_tensor(args, "embedding");
+ if (embedding.dim() != 3) {
+ throw std::runtime_error("audio embedding must have feature, frame and layer axes");
+ }
+ const auto frames = 4 * get_int(args, "latent_frames");
+ validate_float_shape({embedding.shape()[0], frames, embedding.shape()[2]});
+ auto result = sd::dgl::ops::build_s2v_audio_window(embedding, frames);
+ if (result.empty()) {
+ throw std::runtime_error("could not build the S2V audio window");
+ }
+ return tensor_value(std::move(result));
+ }
+
+}
diff --git a/src/dgl/bindings/latents.h b/src/dgl/bindings/latents.h
new file mode 100644
index 0000000..20f555e
--- /dev/null
+++ b/src/dgl/bindings/latents.h
@@ -0,0 +1,20 @@
+#ifndef __SD_DGL_BINDINGS_LATENTS_H__
+#define __SD_DGL_BINDINGS_LATENTS_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value encode_vision(Values& args, RunState&);
+ Value encode_audio(Values& args, RunState&);
+ Value encode_ip_adapter(Values& args, RunState&);
+ Value upscale_latent(Values& args, RunState&);
+ Value audio_vae_encode(Values& args, RunState&);
+ Value audio_vae_decode(Values& args, RunState&);
+ Value execute_audio_latent_shape(Values& args, RunState&);
+ Value execute_ltx_video_positions(Values& args, RunState&);
+ Value execute_s2v_audio_window(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_LATENTS_H__
diff --git a/src/dgl/bindings/lora.cpp b/src/dgl/bindings/lora.cpp
new file mode 100644
index 0000000..6892b08
--- /dev/null
+++ b/src/dgl/bindings/lora.cpp
@@ -0,0 +1,46 @@
+#include "dgl/bindings/lora.h"
+
+#include <cmath>
+
+#include "core/file_access.h"
+#include "dgl/context.h"
+#include "dgl/executor.h"
+#include "dgl/models/lora.h"
+
+namespace sd::dgl {
+
+ static std::shared_ptr<models::LoraFile> lora_file(std::string path) {
+ require_file_access(path, SD_DGL_FILE_READ_MODEL);
+ if (path.empty())
+ throw std::runtime_error("LoRA path is empty");
+ return std::make_shared<models::LoraFile>(models::LoraFile{std::move(path)});
+ }
+
+ static float lora_weight(const Values& args) {
+ const auto weight = float(get_number(args, "weight"));
+ if (!std::isfinite(weight))
+ throw std::runtime_error("LoRA weight must be finite");
+ return weight;
+ }
+
+ PreparedResource load_lora(Values& args) {
+ PreparedResource prepared;
+ prepared.revision = nlohmann::json::object();
+ auto path = file_key(get_string(args, "path"), prepared.revision);
+ prepared.key = {{"path", path}};
+ prepared.load = [path = std::move(path)](ResourceLoadRequest&) {
+ return Value::object(lora_file(path));
+ };
+ return prepared;
+ }
+
+ Value apply_lora(Values& args, RunState& run) {
+ const auto& model = args.at("model");
+ auto config = model.loras ? std::make_shared<models::LoraConfig>(*model.loras)
+ : std::make_shared<models::LoraConfig>();
+ if (args.count("mode"))
+ config->mode = str_to_lora_apply_mode(get_string(args, "mode").c_str());
+ config->bindings.push_back({args.at("lora").as<models::LoraFile>(), lora_weight(args)});
+ return run.context.derive_model(model, std::move(config));
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/lora.h b/src/dgl/bindings/lora.h
new file mode 100644
index 0000000..9b6a1cb
--- /dev/null
+++ b/src/dgl/bindings/lora.h
@@ -0,0 +1,14 @@
+#ifndef __SD_DGL_BINDINGS_LORA_H__
+#define __SD_DGL_BINDINGS_LORA_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+ struct PreparedResource;
+
+ PreparedResource load_lora(Values& args);
+ Value apply_lora(Values& args, RunState& run);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_LORA_H__
diff --git a/src/dgl/bindings/media.cpp b/src/dgl/bindings/media.cpp
new file mode 100644
index 0000000..6bf6859
--- /dev/null
+++ b/src/dgl/bindings/media.cpp
@@ -0,0 +1,49 @@
+#include "dgl/bindings/media.h"
+
+#include "core/file_access.h"
+#include "dgl/bindings/audio.h"
+#include "dgl/bindings/conversion.h"
+#include "dgl/bindings/video.h"
+#include "dgl/ops/image.h"
+#include "dgl/ops/media_encoding.h"
+
+namespace sd::dgl {
+ OutputSignature media_signature() {
+ return {{Kind::Dict}, {{"data", {Kind::Bytes}}, {"mime_type", {Kind::String}}}};
+ }
+
+ static Value media_result(std::vector<uint8_t> bytes, const char* mime_type) {
+ return Value::dict({{"data", Value::object(std::make_shared<Bytes>(std::move(bytes)))},
+ {"mime_type", Value::literal(mime_type)}});
+ }
+
+ Value execute_encode_image(Values& args, RunState& state) {
+ return media_result(ops::encode_image(*object<sd_tensor_t>(args, "image"), state.metadata), "image/png");
+ }
+
+ Value execute_encode_video(Values& args, RunState& state) {
+ const auto video = read_video_dict(args.at("video"));
+ ops::VideoSaveOptions options;
+ options.fps_num = video.fps_num;
+ options.fps_den = video.fps_den;
+ options.metadata = state.metadata;
+ options.pcm16 = true;
+ if (args.count("audio"))
+ options.audio = read_audio_dict(args.at("audio"), true);
+ if (get_string(args, "format") == "avi")
+ return media_result(ops::encode_avi(*video.frames, options), "video/x-msvideo");
+ return media_result(ops::encode_webm(*video.frames, options), "video/webm");
+ }
+
+ Value execute_save_media(Values& args, RunState&) {
+ const auto& media = args.at("media");
+ const auto data = media.fields.find("data"), mime = media.fields.find("mime_type");
+ if (data == media.fields.end() || data->second.type().kind != Kind::Bytes ||
+ mime == media.fields.end() || mime->second.type().kind != Kind::String)
+ throw std::runtime_error("media::save requires a dict with data (bytes) and mime_type (string)");
+ const auto path = get_string(args, "path");
+ require_file_access(path, SD_DGL_FILE_WRITE);
+ ops::save_media(path, data->second.as<Bytes>()->data, mime->second.scalar.get<std::string>());
+ return Value::void_value();
+ }
+}
diff --git a/src/dgl/bindings/media.h b/src/dgl/bindings/media.h
new file mode 100644
index 0000000..4e52716
--- /dev/null
+++ b/src/dgl/bindings/media.h
@@ -0,0 +1,13 @@
+#ifndef __SD_DGL_BINDINGS_MEDIA_H__
+#define __SD_DGL_BINDINGS_MEDIA_H__
+
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+ OutputSignature media_signature();
+ Value execute_encode_image(Values& args, RunState& state);
+ Value execute_encode_video(Values& args, RunState& state);
+ Value execute_save_media(Values& args, RunState& state);
+}
+
+#endif
diff --git a/src/dgl/bindings/model.cpp b/src/dgl/bindings/model.cpp
new file mode 100644
index 0000000..e02b468
--- /dev/null
+++ b/src/dgl/bindings/model.cpp
@@ -0,0 +1,68 @@
+#include "dgl/bindings/model.h"
+
+#include "dgl/bindings/conversion.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+
+#include "dgl/context.h"
+#include "dgl/models/detail_detector.h"
+#include "dgl/models/upscaler.h"
+
+namespace sd::dgl {
+ PreparedResource execute_load_upscaler(Values& args) {
+ PreparedResource prepared;
+ prepared.revision = nlohmann::json::object();
+ models::ImageUpscalerParams params{
+ file_key(get_string(args, "path"), prepared.revision),
+ trim(get_string(args, "backend")),
+ trim(get_string(args, "params_backend")),
+ get_bool(args, "direct"),
+ int(get_int(args, "tile_size")),
+ };
+ prepared.key = {{"path", params.path}, {"backend", params.backend}, {"params_backend", params.params_backend}, {"direct", params.direct}, {"tile_size", params.tile_size}};
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ auto model = request.context.make_model<models::ImageUpscaler>(request.weights_key);
+ if (!model->init(std::move(params)))
+ throw std::runtime_error("upscaler loading failed");
+ return Value::object(std::move(model));
+ };
+ return prepared;
+ }
+
+ Value execute_upscale(Values& args, RunState&) {
+ auto image = image_view(args.at("image"));
+ auto model = object<models::ImageUpscaler>(args, "upscaler");
+ const int64_t factor = model->get_scale();
+ if (factor <= 0 || image.channel != 3 || int64_t(image.width) * factor > 16384 || int64_t(image.height) * factor > 16384 ||
+ int64_t(image.width) * image.height * factor * factor > 64 * 1024 * 1024)
+ throw Error(SD_STATUS_OUT_OF_RANGE, "upscale requires RGB and output at most 64 megapixels at the model's actual scale");
+ if (args.count("factor") && get_int(args, "factor") != factor)
+ LOG_WARN("upscale factor=%lld is ignored; the model's actual scale is %lld", (long long)get_int(args, "factor"), (long long)factor);
+ auto output = model->upscale(image);
+ std::unique_ptr<uint8_t, decltype(&free)> owner(output.data, free);
+ if (!output.data)
+ throw std::runtime_error("upscale failed");
+ return Value::object(copy_image(output));
+ }
+
+ PreparedResource execute_load_detector(Values& args) {
+ PreparedResource prepared;
+ prepared.revision = nlohmann::json::object();
+ models::DetailDetectorParams params{
+ file_key(get_string(args, "path"), prepared.revision),
+ trim(get_string(args, "backend")),
+ trim(get_string(args, "params_backend")),
+ };
+ prepared.key = {{"path", params.path}, {"backend", params.backend}, {"params_backend", params.params_backend}};
+ prepared.load = [params = std::move(params)](ResourceLoadRequest& request) mutable {
+ auto model = request.context.make_model<models::DetailDetector>(request.weights_key);
+ if (!model->init(std::move(params)))
+ throw std::runtime_error("detector loading failed");
+ return Value::object(std::move(model));
+ };
+ return prepared;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/model.h b/src/dgl/bindings/model.h
new file mode 100644
index 0000000..2deb80b
--- /dev/null
+++ b/src/dgl/bindings/model.h
@@ -0,0 +1,15 @@
+#ifndef __SD_DGL_BINDINGS_MODEL_H__
+#define __SD_DGL_BINDINGS_MODEL_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+ struct PreparedResource;
+
+ PreparedResource execute_load_upscaler(Values& args);
+ Value execute_upscale(Values& args, RunState&);
+ PreparedResource execute_load_detector(Values& args);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_MODEL_H__
diff --git a/src/dgl/bindings/postprocess.cpp b/src/dgl/bindings/postprocess.cpp
new file mode 100644
index 0000000..22fda06
--- /dev/null
+++ b/src/dgl/bindings/postprocess.cpp
@@ -0,0 +1,16 @@
+#include "dgl/bindings/postprocess.h"
+
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ Value execute_canny(Values& args, RunState&) {
+ auto source = image_view(args.at("image"));
+ if (source.channel != 3 || get_number(args, "low_threshold") > get_number(args, "high_threshold"))
+ throw std::runtime_error("image::canny requires RGB and low_threshold <= high_threshold");
+ auto result = copy_image(source);
+ if (!sd::preprocess_canny(image_view(*result), float(get_number(args, "high_threshold")), float(get_number(args, "low_threshold")), float(get_number(args, "weak")), float(get_number(args, "strong")), get_bool(args, "invert")))
+ throw std::runtime_error("image::canny preprocessing failed");
+ return Value::object(result);
+ }
+
+}
diff --git a/src/dgl/bindings/postprocess.h b/src/dgl/bindings/postprocess.h
new file mode 100644
index 0000000..ebd9c0c
--- /dev/null
+++ b/src/dgl/bindings/postprocess.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_BINDINGS_POSTPROCESS_H__
+#define __SD_DGL_BINDINGS_POSTPROCESS_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value execute_canny(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_POSTPROCESS_H__
diff --git a/src/dgl/bindings/predictor.cpp b/src/dgl/bindings/predictor.cpp
new file mode 100644
index 0000000..562b449
--- /dev/null
+++ b/src/dgl/bindings/predictor.cpp
@@ -0,0 +1,240 @@
+#include "dgl/bindings/predictor.h"
+
+#include <cmath>
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/condition.h"
+#include "dgl/context.h"
+#include "dgl/model_value.h"
+#include "dgl/models/control.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/pulid.h"
+#include "dgl/ops/predictor.h"
+
+namespace sd::dgl {
+ using ops::ApgPredictor;
+ using ops::CfgPredictor;
+ using ops::GuidanceBranches;
+ using ops::ModelPredictor;
+ using ops::Predictor;
+ using ops::SkipLayerPredictor;
+ using ops::SwitchPredictor;
+
+ template <class Node>
+ static Value predictor_value(Node node) {
+ return Value::object(std::make_shared<Predictor>(Predictor{std::move(node)}));
+ }
+
+ static void bind_predictor_options(ModelPredictor& predictor, const Values& args) {
+ predictor.circular_x = get_bool(args, "circular_x");
+ predictor.circular_y = get_bool(args, "circular_y");
+ auto& options = predictor.options;
+ for (const auto& [name, field] : {std::pair{"vace_context", &options.vace_context}, {"audio_embedding", &options.audio_embedding}, {"video_positions", &options.video_positions}}) {
+ if (args.count(name)) {
+ *field = float_tensor(args, name);
+ }
+ }
+ if (args.count("skip_layers")) {
+ for (const auto& layer : args.at("skip_layers").items) {
+ options.skip_layers.push_back(int(layer.scalar.get<int64_t>()));
+ }
+ }
+ options.fps = float(get_number(args, "fps"));
+ options.audio_length = int(get_int(args, "audio_length"));
+ options.video_sigma_shift = float(get_number(args, "video_sigma_shift"));
+ options.audio_sigma_shift = float(get_number(args, "audio_sigma_shift"));
+ options.vace_strength = float(get_number(args, "vace_strength"));
+ if (args.count("transform")) {
+ auto fn = object<ScriptFunction>(args, "transform");
+ auto captures = args.count("transform_args") ? args.at("transform_args") : Value::dict({});
+ options.transform = [fn = std::move(fn), captures = std::move(captures)](models::PredictionParameters step) {
+ const auto result = (*fn)(Value::dict({{"sigma", Value::literal(step.sigma)},
+ {"timestep", Value::literal(step.timestep)},
+ {"input_scale", Value::literal(step.input_scale)},
+ {"skip_scale", Value::literal(step.skip_scale)},
+ {"output_scale", Value::literal(step.output_scale)}}),
+ captures);
+ const auto update = [&](const char* name, float& target) {
+ if (!result.fields.count(name))
+ return;
+ target = float(get_number(result.fields, name));
+ if (!std::isfinite(target))
+ throw std::runtime_error(std::string("prediction transform returned a non-finite ") + name);
+ };
+ update("timestep", step.timestep);
+ update("input_scale", step.input_scale);
+ update("skip_scale", step.skip_scale);
+ update("output_scale", step.output_scale);
+ return step;
+ };
+ }
+ }
+
+ static ops::SigmaWindow sigma_window(const Values& args) {
+ const auto start = args.count("start_sigma") ? float(get_number(args, "start_sigma")) : INFINITY;
+ const auto end = args.count("end_sigma") ? float(get_number(args, "end_sigma")) : 0.f;
+ if (start < end)
+ throw std::runtime_error("start_sigma must be greater than or equal to end_sigma");
+ return {start, end};
+ }
+
+ Value apply_controlnet(Values& args, RunState&) {
+ auto fields = args.at("condition").fields;
+ const auto previous = fields.find("controls");
+ auto controls = previous == fields.end() ? std::make_shared<models::ControlBindings>()
+ : std::make_shared<models::ControlBindings>(*previous->second.as<models::ControlBindings>());
+ auto model = object<ControlNet>(args, "model");
+ auto hint = std::shared_ptr<const sd::Tensor<float>>(object<sd_tensor_t>(args, "hint"), &float_tensor(args, "hint"));
+ const auto type = str_to_sd_control_type(get_string(args, "type").c_str());
+ const auto [start, end] = sigma_window(args);
+ model->validate_hint(*hint, type);
+ controls->entries.push_back({std::move(model), std::move(hint), type, float(get_number(args, "strength")), start, end});
+ fields.insert_or_assign("controls", Value::object(std::move(controls)));
+ return Value::dict(std::move(fields));
+ }
+
+ Value apply_ip_adapter(Values& args, RunState&) {
+ auto fields = args.at("condition").fields;
+ const auto previous = fields.find("ip_adapters");
+ auto bindings = previous == fields.end() ? std::make_shared<ops::IPAdapterBindings>()
+ : std::make_shared<ops::IPAdapterBindings>(*previous->second.as<ops::IPAdapterBindings>());
+ auto adapter = object<IPAdapter>(args, "adapter");
+ auto tokens = std::shared_ptr<const sd::Tensor<float>>(object<sd_tensor_t>(args, "tokens"), &float_tensor(args, "tokens"));
+ const auto [start, end] = sigma_window(args);
+ adapter->validate_tokens(*tokens);
+ bindings->entries.push_back({std::move(adapter), std::move(tokens), float(get_number(args, "strength")), start, end});
+ fields.insert_or_assign("ip_adapters", Value::object(std::move(bindings)));
+ return Value::dict(std::move(fields));
+ }
+
+ Value apply_pulid(Values& args, RunState&) {
+ auto fields = args.at("condition").fields;
+ const auto previous = fields.find("pulid");
+ auto bindings = previous == fields.end() ? std::make_shared<ops::PuLIDBindings>()
+ : std::make_shared<ops::PuLIDBindings>(*previous->second.as<ops::PuLIDBindings>());
+ auto pulid = object<models::PuLID>(args, "pulid");
+ auto identity = std::shared_ptr<const sd::Tensor<float>>(object<sd_tensor_t>(args, "embedding"), &float_tensor(args, "embedding"));
+ const auto [start, end] = sigma_window(args);
+ pulid->validate_embedding(*identity);
+ bindings->entries.push_back({std::move(pulid), std::move(identity), float(get_number(args, "strength")), start, end});
+ fields.insert_or_assign("pulid", Value::object(std::move(bindings)));
+ return Value::dict(std::move(fields));
+ }
+
+ Value apply_animatediff(Values& args, RunState&) {
+ return with_animatediff(args.at("model"), object<models::AnimateDiff>(args, "motion"));
+ }
+
+ Value make_predictor(Values& args, RunState&) {
+ ModelPredictor output;
+ output.model = object<models::DiffusionModel>(args, "model");
+ auto& engine = output.dit();
+ output.parameterization = engine.make_parameterization(str_to_prediction(get_string(args, "prediction").c_str()));
+ const auto ref_index_mode = get_string(args, "ref_index_mode");
+ output.options.ref_index_mode = ref_index_mode == "increase" ? Rope::RefIndexMode::INCREASE
+ : ref_index_mode == "decrease" ? Rope::RefIndexMode::DECREASE
+ : Rope::RefIndexMode::FIXED;
+ output.options.force_ref_timestep_zero = get_bool(args, "force_ref_timestep_zero");
+ output.options.guidance = float(get_number(args, "guidance"));
+ bind_predictor_options(output, args);
+ const auto& condition = args.at("condition").fields;
+ if (const auto controls = condition.find("controls"); controls != condition.end()) {
+ output.controls = controls->second.as<models::ControlBindings>();
+ for (const auto& binding : output.controls->entries)
+ binding.model->validate_target(engine);
+ }
+ if (const auto adapters = condition.find("ip_adapters"); adapters != condition.end()) {
+ output.ip_adapters = adapters->second.as<ops::IPAdapterBindings>();
+ for (const auto& binding : output.ip_adapters->entries)
+ if (binding.strength != 0.f)
+ binding.model->validate_target(engine);
+ }
+ if (const auto bindings = condition.find("pulid"); bindings != condition.end()) {
+ output.pulid = bindings->second.as<ops::PuLIDBindings>();
+ for (const auto& binding : output.pulid->entries)
+ if (binding.strength != 0.f)
+ binding.model->validate_target(engine);
+ }
+ output.condition = read_condition(condition, args);
+ auto& c = output.condition;
+ if (args.count("ref_latents")) {
+ output.references = float_tensor_list(args.at("ref_latents"));
+ }
+ if (const auto lq = condition.find("lq_frames"); lq != condition.end()) {
+ if (engine.version() != VERSION_RELAY_VSR || !output.references.empty() || !c.c_ref_images.empty())
+ throw std::runtime_error("lq_frames requires a RelayVSR predictor with one low-quality frame tensor");
+ c.c_ref_images.push_back(pixels(lq->second));
+ }
+ validate_reference_blocks(c, output.references.empty() ? c.c_ref_images.size() : output.references.size());
+ return predictor_value(std::move(output));
+ }
+
+ static GuidanceBranches guidance_branches(const Values& args) {
+ GuidanceBranches output;
+ output.positive = object<Predictor>(args, "positive");
+ output.negative = object<Predictor>(args, "negative");
+ output.scale = float(get_number(args, "scale"));
+ output.image_scale = float(get_number(args, "image_scale"));
+ if (args.count("image_only"))
+ output.image_only = object<Predictor>(args, "image_only");
+ if (args.count("schedule")) {
+ for (const auto& item : args.at("schedule").items) {
+ const auto scale = item.scalar.get<float>();
+ if (!(scale >= 0.f && scale <= 1000.f))
+ throw std::runtime_error("guidance schedule values must be between zero and 1000");
+ output.scale_schedule.push_back(scale);
+ }
+ }
+ return output;
+ }
+
+ Value make_cfg(Values& args, RunState&) {
+ return predictor_value(CfgPredictor{guidance_branches(args)});
+ }
+
+ Value make_apg(Values& args, RunState&) {
+ auto branches = guidance_branches(args);
+ sd::guidance::AdaptiveProjectedGuidanceParams params{
+ float(get_number(args, "eta")),
+ float(get_number(args, "momentum")),
+ float(get_number(args, "norm_threshold")),
+ float(get_number(args, "norm_threshold_smoothing")),
+ };
+ if (!sd::guidance::is_adaptive_projected_guidance_enabled(params))
+ return predictor_value(CfgPredictor{std::move(branches)});
+
+ for (const auto* branch : {branches.positive.get(), branches.negative.get(), branches.image_only.get()})
+ if (branch && !ops::provides_model_output(*branch))
+ throw std::runtime_error("APG branches must provide raw model outputs");
+ return predictor_value(ApgPredictor{std::move(branches), params});
+ }
+
+ Value make_slg(Values& args, RunState&) {
+ std::optional<ops::StepWindow> steps;
+ if (args.count("start_step_fraction") || args.count("stop_step_fraction")) {
+ const auto start = args.count("start_step_fraction") ? float(get_number(args, "start_step_fraction")) : 0.f;
+ const auto stop = args.count("stop_step_fraction") ? float(get_number(args, "stop_step_fraction")) : 1.f;
+ if (stop < start)
+ throw std::runtime_error("SLG start_step_fraction must be less than or equal to stop_step_fraction");
+ steps = ops::StepWindow{start, stop};
+ }
+ return predictor_value(SkipLayerPredictor{
+ object<Predictor>(args, "base"),
+ object<Predictor>(args, "positive"),
+ object<Predictor>(args, "skipped"),
+ float(get_number(args, "scale")),
+ sigma_window(args),
+ steps,
+ });
+ }
+
+ Value switch_predictor(Values& args, RunState&) {
+ return predictor_value(SwitchPredictor{
+ object<Predictor>(args, "before"),
+ object<Predictor>(args, "after"),
+ int(get_int(args, "after_steps")),
+ });
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/predictor.h b/src/dgl/bindings/predictor.h
new file mode 100644
index 0000000..3cfc63f
--- /dev/null
+++ b/src/dgl/bindings/predictor.h
@@ -0,0 +1,20 @@
+#ifndef __SD_DGL_BINDINGS_PREDICTOR_H__
+#define __SD_DGL_BINDINGS_PREDICTOR_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value apply_controlnet(Values& args, RunState&);
+ Value apply_ip_adapter(Values& args, RunState&);
+ Value apply_pulid(Values& args, RunState&);
+ Value apply_animatediff(Values& args, RunState&);
+ Value make_predictor(Values& args, RunState&);
+ Value make_cfg(Values& args, RunState&);
+ Value make_apg(Values& args, RunState&);
+ Value make_slg(Values& args, RunState&);
+ Value switch_predictor(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_PREDICTOR_H__
diff --git a/src/dgl/bindings/preview.cpp b/src/dgl/bindings/preview.cpp
new file mode 100644
index 0000000..3db6e17
--- /dev/null
+++ b/src/dgl/bindings/preview.cpp
@@ -0,0 +1,14 @@
+#include "dgl/bindings/preview.h"
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/models/vae.h"
+#include "dgl/ops/preview.h"
+
+namespace sd::dgl {
+ Value make_preview(Values& args, RunState&) {
+ auto decoder = args.count("decoder") ? object<models::VAE>(args, "decoder") : std::shared_ptr<models::VAE>{};
+ std::shared_ptr<ops::SamplingObserver> observer = std::make_shared<ops::PreviewObserver>(
+ std::move(decoder), get_bool(args, "circular_x"), get_bool(args, "circular_y"));
+ return Value::object(std::move(observer));
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/preview.h b/src/dgl/bindings/preview.h
new file mode 100644
index 0000000..67c384e
--- /dev/null
+++ b/src/dgl/bindings/preview.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_BINDINGS_PREVIEW_H__
+#define __SD_DGL_BINDINGS_PREVIEW_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value make_preview(Values& args, RunState& state);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_PREVIEW_H__
diff --git a/src/dgl/bindings/regions.cpp b/src/dgl/bindings/regions.cpp
new file mode 100644
index 0000000..17efeb5
--- /dev/null
+++ b/src/dgl/bindings/regions.cpp
@@ -0,0 +1,83 @@
+#include "dgl/bindings/regions.h"
+
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ OutputSignature regions_signature() {
+ return {{Kind::Dict}, {{"width", {Kind::Integer}}, {"height", {Kind::Integer}}, {"boxes", {Kind::Tensor}}}};
+ }
+
+ static const sd::Tensor<int64_t>& region_box_tensor(const std::shared_ptr<sd_tensor_t>& boxes) {
+ if (!boxes || boxes->type != SD_TENSOR_I64)
+ throw std::runtime_error("region boxes require an I64 [4,N] tensor");
+ if (boxes->shape().size() != 2 || boxes->shape()[0] != 4 || boxes->shape()[1] > 10000)
+ throw std::runtime_error("region boxes require I64 [4,N] with at most 10000 boxes");
+ // Public empty tensor handles retain shape; their internal storage does not.
+ static const sd::Tensor<int64_t> empty({4, 0});
+ return boxes->bytes ? boxes->as<int64_t>() : empty;
+ }
+
+ Value regions_value(int64_t width, int64_t height, std::shared_ptr<sd_tensor_t> boxes) {
+ ops::validate_region_boxes(region_box_tensor(boxes), width, height);
+ return Value::dict({{"width", Value::literal(width)}, {"height", Value::literal(height)}, {"boxes", Value::object(std::move(boxes))}});
+ }
+
+ static std::shared_ptr<sd_tensor_t> read_regions(const Value& value, const sd::Image& canvas) {
+ if (value.type().kind != Kind::Dict)
+ throw std::runtime_error("regions require a dict with width, height and boxes");
+ const auto& fields = value.fields;
+ for (const auto* name : {"width", "height", "boxes"}) {
+ const auto found = fields.find(name);
+ if (found == fields.end())
+ throw std::runtime_error(std::string("regions are missing field '") + name + "'");
+ const auto expected = std::string(name) == "boxes" ? Kind::Tensor : Kind::Integer;
+ if (found->second.type().kind != expected)
+ throw std::runtime_error(std::string("regions field '") + name + "' expects " + type_name({expected}));
+ }
+ const auto width = get_int(fields, "width"), height = get_int(fields, "height");
+ auto boxes = fields.at("boxes").as<sd_tensor_t>();
+ ops::validate_region_boxes(region_box_tensor(boxes), width, height);
+ if (width != canvas.width || height != canvas.height)
+ throw std::runtime_error("region canvas dimensions do not match the image");
+ return boxes;
+ }
+
+ Value regions_from_boxes(Values& args, RunState&) {
+ const auto& input = args.at("boxes");
+ std::shared_ptr<sd_tensor_t> boxes;
+ if (input.type().kind == Kind::Tensor) {
+ boxes = input.as<sd_tensor_t>();
+ } else if (input.type().kind == Kind::List) {
+ if (input.items.size() % 4 || input.items.size() > 40000)
+ throw std::runtime_error("boxes must contain x,y,width,height groups (at most 10000)");
+ std::vector<int64_t> data;
+ data.reserve(input.items.size());
+ for (const auto& item : input.items) {
+ if (item.type().kind != Kind::Integer)
+ throw std::runtime_error("region box list elements must be integers");
+ data.push_back(item.scalar.get<int64_t>());
+ }
+ boxes = make_tensor(SD_TENSOR_I64, std::vector<int64_t>{4, int64_t(data.size() / 4)}, data.data(), data.size() * sizeof(int64_t));
+ } else
+ throw std::runtime_error("boxes require a flat list<int> or an I64 [4,N] tensor");
+ return regions_value(get_int(args, "width"), get_int(args, "height"), std::move(boxes));
+ }
+
+ Value crop_regions(Values& args, RunState&) {
+ const auto image = object<sd_tensor_t>(args, "image");
+ auto boxes = read_regions(args.at("regions"), image_view(*image));
+ std::vector<Value> crops;
+ for (auto crop : ops::crop_regions(*image, region_box_tensor(boxes)))
+ crops.push_back(Value::object(std::move(crop)));
+ return Value::list(std::move(crops), Kind::Tensor);
+ }
+
+ Value paste_regions(Values& args, RunState&) {
+ const auto background = object<sd_tensor_t>(args, "background");
+ auto boxes = read_regions(args.at("regions"), image_view(*background));
+ std::vector<std::shared_ptr<sd_tensor_t>> foregrounds;
+ for (const auto& item : args.at("foregrounds").items)
+ foregrounds.push_back(item.as<sd_tensor_t>());
+ return Value::object(ops::paste_regions(*background, region_box_tensor(boxes), foregrounds));
+ }
+}
diff --git a/src/dgl/bindings/regions.h b/src/dgl/bindings/regions.h
new file mode 100644
index 0000000..d092c97
--- /dev/null
+++ b/src/dgl/bindings/regions.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_BINDINGS_REGIONS_H__
+#define __SD_DGL_BINDINGS_REGIONS_H__
+
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ OutputSignature regions_signature();
+ Value regions_from_boxes(Values& args, RunState&);
+ Value regions_value(int64_t width, int64_t height, std::shared_ptr<sd_tensor_t> boxes);
+ Value crop_regions(Values& args, RunState&);
+ Value paste_regions(Values& args, RunState&);
+}
+
+#endif // __SD_DGL_BINDINGS_REGIONS_H__
diff --git a/src/dgl/bindings/registration.cpp b/src/dgl/bindings/registration.cpp
new file mode 100644
index 0000000..f06ebb2
--- /dev/null
+++ b/src/dgl/bindings/registration.cpp
@@ -0,0 +1,753 @@
+#include "dgl/bindings/registration.h"
+
+#include <climits>
+#include <limits>
+
+#include "core/util.h"
+#include "dgl/bindings/audio.h"
+#include "dgl/bindings/component_loader.h"
+#include "dgl/bindings/components.h"
+#include "dgl/bindings/condition.h"
+#include "dgl/bindings/detailer.h"
+#include "dgl/bindings/detections.h"
+#include "dgl/bindings/image.h"
+#include "dgl/bindings/latents.h"
+#include "dgl/bindings/lora.h"
+#include "dgl/bindings/media.h"
+#include "dgl/bindings/model.h"
+#include "dgl/bindings/postprocess.h"
+#include "dgl/bindings/predictor.h"
+#include "dgl/bindings/preview.h"
+#include "dgl/bindings/regions.h"
+#include "dgl/bindings/relay_vsr.h"
+#include "dgl/bindings/rng.h"
+#include "dgl/bindings/seedvr2.h"
+#include "dgl/bindings/sefi.h"
+#include "dgl/bindings/tensor_ops.h"
+#include "dgl/bindings/values.h"
+#include "dgl/bindings/video.h"
+#include "dgl/bindings/video_windows.h"
+
+#include "dgl/bindings/conversion.h"
+#include "model_definition.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl {
+
+ static Argument backend_option(const char* name) {
+ auto argument = optional(name, Kind::String, "");
+ argument.validate = [name](const Value& value) {
+ const auto& spec = value.scalar.get_ref<const std::string&>();
+ if (spec.find_first_of("=,") != std::string::npos)
+ throw std::runtime_error(std::string(name) + " expects a value for this model");
+ };
+ return argument;
+ }
+
+ static Argument operator_mode(const char* name) {
+ auto argument = optional(name, Kind::String, "auto");
+ argument.validate = [name](const Value& value) {
+ const auto mode = trim(value.scalar.get_ref<const std::string&>());
+ if (mode != "auto" && mode != "on" && mode != "off")
+ throw std::runtime_error(std::string(name) + " expects auto, on, or off");
+ };
+ return argument;
+ }
+
+ static Operation configuration(std::vector<Argument> arguments) {
+ OutputSignature output{Kind::Dict};
+ for (const auto& argument : arguments)
+ output.fields.emplace(argument.name, argument.type);
+ return make_operation(std::move(arguments), std::move(output), execute_configuration);
+ }
+
+ static Argument int_vector_argument(const std::string& name) {
+ return {name, {Kind::IntVector}, true, std::nullopt, [](const Value& value) {
+ if (value.type().kind == Kind::Tensor)
+ i64_vector(value);
+ }};
+ }
+
+ static void register_value_operations(Registry& registry) {
+ registry["list::replace"] = make_operation(
+ {list_argument("items", Kind::Any), integer("index", 0, INT64_MAX), required("value", Kind::Any)},
+ {Kind::List}, execute_list_replace);
+ registry["math::product"] = make_operation(
+ {int_vector_argument("items")}, {Kind::Integer}, execute_math_product);
+ registry["math::ceil_div"] = make_operation(
+ {integer("numerator", 0, INT64_MAX), integer("denominator", 1, INT64_MAX)},
+ {Kind::Integer}, execute_math_ceil_div);
+ const std::vector<Argument> integers{required("a", Kind::Integer), required("b", Kind::Integer)};
+ registry["math::min"] = make_operation(
+ integers, {Kind::Integer}, execute_math_min);
+ const std::vector<Argument> numbers{required("a", Kind::Number), required("b", Kind::Number)};
+ registry["math::add"] = make_operation(
+ numbers, {Kind::Number}, execute_math_add);
+ registry["math::sub"] = make_operation(
+ numbers, {Kind::Number}, execute_math_sub);
+ registry["math::mul"] = make_operation(
+ {required("a", Kind::Number), required("b", Kind::Number)}, {Kind::Number}, execute_math_mul);
+ registry["math::div"] = make_operation(
+ {required("a", Kind::Number), required("b", Kind::Number)}, {Kind::Number}, execute_math_div);
+ const std::vector<Argument> rounding_args{required("value", Kind::Number)};
+ registry["math::floor"] = make_operation(rounding_args, {Kind::Integer}, execute_math_floor);
+ registry["math::ceil"] = make_operation(rounding_args, {Kind::Integer}, execute_math_ceil);
+ registry["math::round"] = make_operation(rounding_args, {Kind::Integer}, execute_math_round);
+ registry["tiling"] = configuration({
+ optional("enabled", Kind::Boolean, true),
+ optional("temporal_tiling", Kind::Boolean, false),
+ integer("tile_size_w", 0, 16384, 0),
+ integer("tile_size_h", 0, 16384, 0),
+ number("target_overlap", 0, 0.99, 0.5),
+ number("rel_size_w", 0, 16384, 0),
+ number("rel_size_h", 0, 16384, 0),
+ optional("extra_args", Kind::String, ""),
+ });
+ std::vector<Argument> cache_args{choice("mode", {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}, "disabled")};
+ auto cache_arg = [&](Argument argument) { argument.required=false; cache_args.push_back(std::move(argument)); };
+ cache_arg(number("reuse_threshold", 0, 100000));
+ cache_arg(number("start_percent", 0, 1));
+ cache_arg(number("end_percent", 0, 1));
+ cache_arg(number("error_decay_rate", 0, 100000));
+ cache_arg(optional("use_relative_threshold", Kind::Boolean));
+ cache_arg(optional("reset_error_on_compute", Kind::Boolean));
+ cache_arg(number("residual_diff_threshold", 0, 100000));
+ cache_arg(integer("max_warmup_steps", 0, 100000));
+ cache_arg(integer("max_cached_steps", INT32_MIN, INT32_MAX));
+ cache_arg(integer("max_continuous_cached_steps", INT32_MIN, INT32_MAX));
+ cache_arg(integer("taylorseer_n_derivatives", 0, 100000));
+ cache_arg(integer("taylorseer_skip_interval", 0, 100000));
+ cache_arg(optional("scm_mask", Kind::String));
+ cache_arg(number("spectrum_w", 0, 100000));
+ cache_arg(integer("spectrum_m", 0, 100000));
+ cache_arg(number("spectrum_lam", 0, 100000));
+ cache_arg(integer("spectrum_window_size", 0, 100000));
+ cache_arg(number("spectrum_flex_window", 0, 100000));
+ cache_arg(integer("spectrum_warmup_steps", 0, 100000));
+ cache_arg(number("spectrum_stop_percent", 0, 1));
+ registry["cache"] = configuration(std::move(cache_args));
+ }
+
+ static void register_rng_operations(Registry& registry) {
+ registry["make_rng"] = make_operation(
+ {choice("algorithm", {"cuda", "cpu", "std", "std_default"}, "cuda"), integer("seed", 0, INT64_MAX, 0)}, {Kind::Rng},
+ execute_make_rng);
+ }
+
+ static void register_tensor_operations(Registry& registry) {
+ const auto tensor = required("input", Kind::Tensor);
+ const auto shape = int_vector_argument("shape");
+ const auto axis = integer("axis", 0, 7);
+ const auto dtype = choice("dtype", {"u8", "i32", "i64", "f32", "f64"});
+ const auto default_dtype = choice("dtype", {"u8", "i32", "i64", "f32", "f64"}, "f32");
+ const auto start = integer("start", 0, INT64_MAX);
+ const auto scalar = [](const char* name) { return required(name, Kind::Number); };
+ registry["tensor::load"] = make_operation(
+ {required("path", Kind::String)}, {Kind::Tensor}, execute_load_tensor);
+ registry["tensor::save"] = make_operation(
+ {required("path", Kind::String), required("tensor", Kind::Tensor)}, {Kind::Void}, execute_save_tensor);
+ registry["tensor::literal"] = make_operation(
+ {dtype, shape, required("data", Kind::String)}, {Kind::Tensor}, execute_tensor_literal);
+ registry["tensor::shape"] = make_operation(
+ {tensor}, {Kind::Tensor}, execute_tensor_shape);
+ registry["tensor::index"] = make_operation(
+ {tensor, list_argument("indices", Kind::Dict)}, {Kind::Any}, execute_tensor_index);
+ registry["tensor::vector"] = make_operation(
+ {list_argument("values", Kind::Number), default_dtype}, {Kind::Tensor}, execute_tensor_vector);
+ registry["tensor::to_list"] = make_operation(
+ {tensor, choice("element", {"auto", "int", "number"}, "auto")}, {Kind::List}, execute_tensor_to_list);
+ registry["tensor::reshape"] = make_operation(
+ {tensor, shape}, {Kind::Tensor}, execute_tensor_reshape);
+ registry["tensor::unsqueeze"] = make_operation(
+ {tensor, integer("axis", -8, 7)}, {Kind::Tensor}, execute_tensor_unsqueeze);
+ registry["tensor::pad"] = make_operation(
+ {tensor, int_vector_argument("pad"), choice("mode", {"constant"}, "constant"), optional("value", Kind::Number, 0)},
+ {Kind::Tensor}, execute_tensor_pad);
+ registry["tensor::permute"] = make_operation(
+ {tensor, int_vector_argument("axes")}, {Kind::Tensor}, execute_tensor_permute);
+ registry["tensor::crop"] = make_operation(
+ {tensor, int_vector_argument("axes"), int_vector_argument("start"), int_vector_argument("size")},
+ {Kind::Tensor}, execute_tensor_crop);
+ registry["tensor::numel"] = make_operation(
+ {tensor}, {Kind::Integer}, execute_tensor_numel);
+ registry["tensor::count_nonzero"] = make_operation(
+ {tensor}, {Kind::Integer}, execute_tensor_count_nonzero);
+ registry["tensor::replace_slice"] = make_operation(
+ {tensor, required("value", Kind::Tensor), axis, start}, {Kind::Tensor},
+ replace_slice);
+ registry["tensor::concat"] = make_operation(
+ {list_argument("items", Kind::Tensor), axis}, {Kind::Tensor},
+ concat_tensors);
+ registry["tensor::cast"] = make_operation(
+ {tensor, dtype}, {Kind::Tensor}, cast_tensor);
+ registry["tensor::broadcast_to"] = make_operation(
+ {tensor, shape}, {Kind::Tensor}, execute_tensor_broadcast_to);
+ registry["tensor::zeros"] = make_operation(
+ {shape, default_dtype}, {Kind::Tensor}, execute_tensor_zeros);
+ registry["tensor::full"] = make_operation(
+ {shape, scalar("value"), default_dtype}, {Kind::Tensor}, execute_tensor_full);
+ registry["tensor::zeros_like"] = make_operation(
+ {required("like", Kind::Tensor)}, {Kind::Tensor}, execute_tensor_zeros_like);
+ registry["tensor::arange"] = make_operation(
+ {integer("start", INT64_MIN, INT64_MAX), integer("end", INT64_MIN, INT64_MAX), integer("step", INT64_MIN, INT64_MAX, 1)},
+ {Kind::Tensor}, arange_tensor);
+ registry["tensor::index_select"] = make_operation(
+ {tensor, axis, int_vector_argument("indices")}, {Kind::Tensor}, index_select_tensor);
+ registry["tensor::randn_like"] = make_operation(
+ {required("like", Kind::Tensor), required("rng", Kind::Rng)},
+ {Kind::Tensor}, execute_tensor_randn_like);
+ const std::vector<Argument> binary_args{required("a", Kind::Any), required("b", Kind::Any)};
+ registry["tensor::add"] = make_operation(binary_args, {Kind::Tensor}, execute_tensor_add);
+ registry["tensor::mul"] = make_operation(binary_args, {Kind::Tensor}, execute_tensor_mul);
+ registry["tensor::div"] = make_operation(binary_args, {Kind::Tensor}, execute_tensor_div);
+ registry["tensor::lerp"] = make_operation(
+ {required("a", Kind::Any), required("b", Kind::Any), required("weight", Kind::Any)}, {Kind::Tensor},
+ execute_tensor_lerp);
+ const auto numeric_limit = std::numeric_limits<double>::max();
+ registry["tensor::affine"] = make_operation(
+ {tensor, number("scale", -numeric_limit, numeric_limit, 1), number("offset", -numeric_limit, numeric_limit, 0)},
+ {Kind::Tensor}, execute_tensor_affine);
+ registry["tensor::clamp"] = make_operation(
+ {tensor, scalar("min"), scalar("max")}, {Kind::Tensor}, execute_tensor_clamp);
+ registry["tensor::translate"] = make_operation(
+ {tensor, int_vector_argument("axes"), int_vector_argument("offset"), number("value", -numeric_limit, numeric_limit, 0)},
+ {Kind::Tensor}, translate_tensor);
+ registry["tensor::index_fill"] = make_operation(
+ {tensor, axis, int_vector_argument("indices"), scalar("value")}, {Kind::Tensor}, execute_tensor_index_fill);
+ registry["tensor::grayscale"] = make_operation(
+ {tensor, integer("axis", 0, 7, 0)}, {Kind::Tensor}, grayscale_tensor);
+ registry["tensor::threshold"] = make_operation(
+ {tensor, scalar("threshold"), number("low", -numeric_limit, numeric_limit, 0), number("high", -numeric_limit, numeric_limit, 1)},
+ {Kind::Tensor}, threshold_tensor);
+ registry["tensor::maximum"] = make_operation(
+ {required("a", Kind::Tensor), required("b", Kind::Tensor)}, {Kind::Tensor}, maximum_tensor);
+ const auto optional_vector = [](const char* name) {
+ auto argument = int_vector_argument(name);
+ argument.required = false;
+ return argument;
+ };
+ const std::vector<Argument> pool_args{
+ tensor,
+ int_vector_argument("kernel"),
+ optional_vector("axes"),
+ optional_vector("stride"),
+ optional_vector("padding_before"),
+ optional_vector("padding_after"),
+ optional("pad_value", Kind::Number),
+ };
+ registry["tensor::max_pool2d"] = make_operation(
+ pool_args, {Kind::Tensor}, execute_tensor_max_pool2d);
+ registry["tensor::min_pool2d"] = make_operation(
+ pool_args, {Kind::Tensor}, execute_tensor_min_pool2d);
+ registry["tensor::gaussian_blur"] = make_operation(
+ {tensor, number("sigma", 0, numeric_limit), integer("radius", 0, 16384), optional_vector("axes"),
+ choice("precision", {"auto", "f32", "f64"}, "auto")},
+ {Kind::Tensor}, blur_tensor);
+ auto axes = int_vector_argument("axes");
+ axes.required = false;
+ registry["tensor::interpolate"] = make_operation(
+ {tensor, int_vector_argument("size"), std::move(axes),
+ choice("mode", {"nearest", "nearest_exact", "nearest_max", "bilinear", "bicubic", "lanczos"}, "bilinear"),
+ optional("antialias", Kind::Boolean, false)},
+ {Kind::Tensor}, execute_tensor_interpolate);
+ }
+
+ static void register_image_operations(Registry& registry) {
+ registry["image::to_f32"] = make_operation(
+ {required("image", Kind::Tensor), integer("channels", 0, 4, 0)}, {Kind::Tensor}, execute_image_to_f32);
+ registry["image::to_u8"] = make_operation(
+ {required("pixels", Kind::Tensor)}, {Kind::Tensor}, execute_image_to_u8);
+ registry["image::load"] = make_operation(
+ {required("path", Kind::String)}, {Kind::Tensor}, execute_load_image);
+ registry["image::load_dir"] = make_operation(
+ {required("path", Kind::String), integer("max_images", 0, 100000, 0)},
+ {Kind::List, Kind::Tensor}, execute_load_images);
+ registry["image::save"] = make_operation(
+ {required("path", Kind::String), required("image", Kind::Tensor)}, {Kind::Void}, execute_save_image);
+ registry["image::encode"] = make_operation(
+ {required("image", Kind::Tensor), choice("format", {"png"}, "png")}, media_signature(), execute_encode_image);
+ registry["image::canny"] = make_operation(
+ {
+ required("image", Kind::Tensor),
+ number("high_threshold", 0, 1, 0.08),
+ number("low_threshold", 0, 1, 0.04),
+ number("weak", 0, 1, 0.5),
+ number("strong", 0, 1, 1),
+ optional("invert", Kind::Boolean, false),
+ },
+ {Kind::Tensor},
+ execute_canny);
+ }
+
+ static void register_region_operations(Registry& registry) {
+ registry["regions::from_boxes"] = make_operation(
+ {integer("width", 1, 16384), integer("height", 1, 16384), required("boxes", Kind::Any)},
+ regions_signature(), regions_from_boxes);
+ registry["regions::crop"] = make_operation(
+ {required("image", Kind::Tensor), required("regions", Kind::Dict)},
+ {Kind::List, Kind::Tensor}, crop_regions);
+ registry["regions::paste"] = make_operation(
+ {required("background", Kind::Tensor), list_argument("foregrounds", Kind::Tensor), required("regions", Kind::Dict)},
+ {Kind::Tensor}, paste_regions);
+ }
+
+ static void register_video_operations(Registry& registry) {
+ registry["video::encode"] = make_operation(
+ {required("video", Kind::Dict), optional("audio", Kind::Dict), choice("format", {"webm", "avi"}, "webm")},
+ media_signature(), execute_encode_video);
+ registry["video::load"] = make_operation(
+ {
+ required("path", Kind::String),
+ integer("max_frames", 0, 100000, 0),
+ integer("fps_num", 0, 1000000, 0),
+ integer("fps_den", 1, 1000000, 1),
+ optional("audio", Kind::Boolean, true),
+ },
+ {{Kind::Dict}, {{"video", video_signature()}, {"audio", audio_signature()}}},
+ execute_load_video);
+ registry["video::save"] = make_operation(
+ {
+ required("path", Kind::String),
+ required("video", Kind::Dict),
+ optional("audio", Kind::Dict),
+ optional("trim_audio", Kind::Boolean, false),
+ choice("codec", {"auto", "mjpeg", "rgb"}, "auto"),
+ integer("quality", 1, 90, 90),
+ choice("audio_format", {"f32", "pcm16"}, "f32"),
+ },
+ {Kind::Void},
+ execute_save_video);
+ registry["video::split_windows"] = make_operation(
+ {required("frames", Kind::Tensor), integer("size", 1, 4097), integer("overlap", 0, 4096, 0)},
+ {Kind::List, Kind::Tensor}, split_video_frames);
+ registry["video::join_windows"] = make_operation(
+ {list_argument("windows", Kind::Tensor), integer("overlap", 0, 4096, 0)},
+ {Kind::Tensor}, join_video_windows);
+ }
+
+ static void register_audio_operations(Registry& registry) {
+ registry["audio::load"] = make_operation(
+ {
+ required("path", Kind::String),
+ },
+ audio_signature(),
+ execute_load_audio);
+ registry["audio::save"] = make_operation(
+ {
+ required("path", Kind::String),
+ required("audio", Kind::Dict),
+ choice("format", {"f32", "pcm16"}, "f32"),
+ },
+ {Kind::Void},
+ execute_save_audio);
+ }
+
+ static void register_model_operations(Registry& registry) {
+ registry["load_upscaler"] = make_resource_operation(
+ {
+ model_path("path"),
+ integer("tile_size", 0, 16384, 128),
+ backend_option("backend"),
+ backend_option("params_backend"),
+ optional("direct", Kind::Boolean, false),
+ },
+ {Kind::Upscaler},
+ execute_load_upscaler);
+ auto scale_hint = integer("factor", 1, 16);
+ scale_hint.required = false;
+ registry["upscale"] = make_operation(
+ {
+ required("upscaler", Kind::Upscaler),
+ required("image", Kind::Tensor),
+ std::move(scale_hint),
+ },
+ {Kind::Tensor},
+ execute_upscale);
+ registry["load_detector"] = make_resource_operation(
+ {
+ model_path("path"),
+ backend_option("backend"),
+ backend_option("params_backend"),
+ },
+ {Kind::Detector},
+ execute_load_detector);
+ }
+
+ template <class T, class Load>
+ static Operation component_loader(const Argument& family, Load load, std::vector<Argument> options = {}, bool require_path = true) {
+ std::vector<Argument> args{
+ model_path("path", require_path),
+ family,
+ optional("wtype", Kind::String, "auto"),
+ number("conv_scale", 0, 100000, 0),
+ number("linear_scale", 0, 100000, 0),
+ number("attn_scale", 0, 100000, 0),
+ };
+ for (const char* name : {"backend", "params_backend", "split_mode"})
+ args.push_back(backend_option(name));
+ for (const char* name : {"model_args", "tensor_type_rules"})
+ args.push_back(optional(name, Kind::String, ""));
+ args.push_back(operator_mode("flash_attn_mode"));
+ args.push_back(operator_mode("conv_direct_mode"));
+ args.push_back(optional("sage_attn", Kind::Boolean, false));
+ for (auto& option : options)
+ args.push_back(std::move(option));
+ return make_resource_operation(std::move(args), {ObjectKind<T>::value}, load);
+ }
+
+ static void register_lora_operations(Registry& registry) {
+ registry["load_lora"] = make_resource_operation({model_path("path")}, {Kind::Lora}, load_lora);
+ OutputSignature output{Kind::Model};
+ output.same_type_as = "model";
+ auto mode = choice("mode", {"auto", "immediately", "at_runtime"});
+ mode.required = false;
+ registry["apply_lora"] = make_operation(
+ {required("model", Kind::Model), required("lora", Kind::Lora), optional("weight", Kind::Number, 1.0), std::move(mode)},
+ std::move(output), apply_lora);
+ }
+
+ static void register_component_operations(Registry& registry) {
+ std::vector<std::string> family_ids{"auto"};
+ for (int version = 0; version < VERSION_COUNT; ++version)
+ family_ids.emplace_back(find_model_definition(static_cast<SDVersion>(version))->id);
+ const auto family = choice("family", std::move(family_ids), "auto");
+ const auto family_from = model_path("family_from", false);
+ register_lora_operations(registry);
+
+ registry["load_dit"] = component_loader<models::DiffusionModel>(family, load_dit);
+ registry.alias("load_unet", "load_dit");
+
+ auto tokenizer = optional("tokenizer", Kind::String, "");
+ registry["load_text_encoder"] = component_loader<models::TextEncoder>(
+ family,
+ load_text_encoder,
+ {
+ family_from,
+ tokenizer,
+ model_path("vision_path", false),
+ model_path("projection_path", false),
+ model_path("connectors_path", false),
+ model_path("t5_path", false),
+ model_path("clip_l_path", false),
+ model_path("clip_g_path", false),
+ });
+ registry["load_vae"] = component_loader<models::VAE>(
+ family,
+ load_vae,
+ {
+ family_from,
+ choice("vae_format", {"auto", "flux", "sd3", "flux2", "wan"}, "auto"),
+ });
+ registry["load_tae"] = component_loader<models::VAE>(
+ family, load_tae, {family_from, optional("decode_only", Kind::Boolean, false)});
+ registry["controlnet::load"] = component_loader<models::ControlNet>(family, load_controlnet);
+ registry["load_vision_encoder"] = component_loader<models::VisionEncoder>(family, load_vision_encoder);
+ registry["load_audio_encoder"] = component_loader<models::AudioEncoder>(family, load_audio_encoder);
+ registry["load_audio_vae"] = component_loader<models::AudioVAE>(family, load_audio_vae, {family_from});
+ registry["ip_adapter::load"] = component_loader<models::IPAdapter>(family, load_ip_adapter);
+ registry["photomaker::load"] = component_loader<models::PhotoMaker>(family, load_photomaker);
+ registry["pulid::load"] = component_loader<models::PuLID>(family, load_pulid);
+ registry["animatediff::load"] = component_loader<models::AnimateDiff>(family, load_animatediff);
+ registry["load_latent_upscaler"] = component_loader<models::LatentUpscaler>(
+ choice("family", {"auto", "ltxav", "minimax_h3"}, "auto"), load_latent_upscaler);
+ registry["pulid::load_embedding"] = make_operation(
+ {required("path", Kind::String)}, {Kind::Tensor}, execute_load_pulid_embedding);
+ registry["photomaker::encode_images"] = make_operation(
+ {required("photomaker", Kind::PhotoMaker),
+ list_argument("images", Kind::Tensor),
+ optional("id_embedding", Kind::Tensor)},
+ {Kind::Tensor},
+ photomaker_tokens);
+ registry["photomaker::encode_prompt"] = make_operation(
+ {required("encoder", Kind::TextEncoder), required("text", Kind::String), required("tokens", Kind::Tensor),
+ integer("width", 1, 16384, 1024), integer("height", 1, 16384, 1024), integer("clip_skip", -1, 1024, -1)},
+ {{Kind::Dict}, {{"plain", condition_signature()}, {"condition", condition_signature()}, {"mask", {Kind::Tensor}}}}, encode_photomaker_prompt);
+ OutputSignature photomaker_condition{{Kind::Dict}, {{"crossattn", {Kind::Tensor}}}};
+ photomaker_condition.same_type_as = "condition";
+ registry["photomaker::apply"] = make_operation(
+ {required("condition", Kind::Dict), required("photomaker", Kind::PhotoMaker),
+ required("tokens", Kind::Tensor), required("mask", Kind::Tensor)},
+ std::move(photomaker_condition), apply_photomaker);
+ registry["encode_prompt"] = make_operation(
+ {required("encoder", Kind::TextEncoder), required("text", Kind::String),
+ integer("width", 1, 16384, 512), integer("height", 1, 16384, 512),
+ integer("clip_skip", INT32_MIN, INT32_MAX, -1), optional("system", Kind::String),
+ optional("media", Kind::List), optional("image_bindings", Kind::Dict), optional("raw", Kind::Boolean, false)},
+ condition_signature(),
+ encode_prompt);
+ OutputSignature zeroed_condition{Kind::Dict};
+ zeroed_condition.same_type_as = "condition";
+ auto zero_fields = list_argument("fields", Kind::String);
+ zero_fields.required = false;
+ registry["zero_out"] = make_operation(
+ {required("condition", Kind::Dict), std::move(zero_fields)},
+ std::move(zeroed_condition), zero_out_condition);
+ registry["vae_encode"] = make_operation(
+ {required("vae", Kind::VAE), required("pixels", Kind::Tensor), required("rng", Kind::Rng),
+ optional("tiling", Kind::Dict), optional("circular_x", Kind::Boolean, false), optional("circular_y", Kind::Boolean, false),
+ choice("posterior", {"sample", "mode"}, "sample")},
+ {Kind::Tensor}, vae_encode);
+ registry["vae_decode"] = make_operation(
+ {required("vae", Kind::VAE), required("latent", Kind::Tensor), optional("tiling", Kind::Dict),
+ optional("circular_x", Kind::Boolean, false), optional("circular_y", Kind::Boolean, false),
+ choice("dtype", {"u8", "f32"}, "u8")},
+ {Kind::Tensor}, vae_decode);
+ const auto conversion_args = {required("vae", Kind::VAE), required("latent", Kind::Tensor)};
+ registry["vae_to_dit_latents"] = make_operation(
+ conversion_args, {Kind::Tensor}, execute_vae_to_dit_latents);
+ registry["dit_to_vae_latents"] = make_operation(
+ conversion_args, {Kind::Tensor}, execute_dit_to_vae_latents);
+ registry.alias("vae_to_unet_latents", "vae_to_dit_latents");
+ registry.alias("unet_to_vae_latents", "dit_to_vae_latents");
+ registry["latent_shape"] = make_operation(
+ {required("vae", Kind::VAE), integer("width", 1, 16384), integer("height", 1, 16384),
+ integer("batch", 1, 65536, 1), integer("frames", 1, 65536)},
+ {Kind::Tensor}, latent_shape);
+ registry["latent_shape"].arguments.back().required = false;
+ auto sequence_length = integer("sequence_length", 1, INT_MAX);
+ sequence_length.required = false;
+ std::vector<std::string> schedulers;
+ for (int i = 0; i < SCHEDULER_COUNT; ++i)
+ schedulers.emplace_back(sd_scheduler_name(static_cast<scheduler_t>(i)));
+ registry["make_sigmas"] = make_operation(
+ {required("predict", Kind::Predictor), required("latent", Kind::Tensor), choice("scheduler", std::move(schedulers), "discrete"),
+ integer("steps", 1, 10000), number("flow_shift", 0, 10000, 0), sequence_length,
+ optional("extra_args", Kind::String, "")},
+ {Kind::Tensor}, make_sigmas);
+ registry["t_to_sigma"] = make_operation(
+ {required("predict", Kind::Predictor), number("t", 0, 1), number("flow_shift", 0, 10000, 0),
+ optional("latent", Kind::Tensor), std::move(sequence_length)},
+ {Kind::Number}, t_to_sigma);
+ const auto sigma_limit = std::numeric_limits<float>::max();
+ const std::map<std::string, Type> scaling_fields{
+ {"input_scale", {Kind::Number}}, {"skip_scale", {Kind::Number}}, {"output_scale", {Kind::Number}}};
+ OutputSignature scaling_output{Kind::Dict};
+ for (const auto& [name, type] : scaling_fields)
+ scaling_output.fields.emplace(name, type);
+ registry["denoiser_scalings"] = make_operation(
+ {required("predict", Kind::Predictor), number("sigma", 0, sigma_limit)},
+ std::move(scaling_output), denoiser_scalings);
+ auto start_sigma = number("start_sigma", 0, sigma_limit);
+ auto end_sigma = number("end_sigma", 0, sigma_limit);
+ start_sigma.required = end_sigma.required = false;
+ registry["add_noise"] = make_operation(
+ {required("predict", Kind::Predictor), required("base", Kind::Tensor), required("noise", Kind::Tensor),
+ number("sigma", 0, 100000)},
+ {Kind::Tensor}, execute_add_noise);
+ auto optional_list = [](const char* name, Kind type) {
+ auto arg = list_argument(name, type);
+ arg.required = false;
+ return arg;
+ };
+ registry["sefi::make_sigmas"] = make_operation(
+ {integer("steps", 1, 10000), number("alpha", 0.001, 100, 1), number("delta", 0, 1, 0.1)},
+ {{Kind::Dict}, {{"sigmas", {Kind::Tensor}}, {"semantic_sigmas", {Kind::Tensor}}}}, sefi_sigmas);
+ OutputSignature controlled_condition{{Kind::Dict}, {{"controls", {Kind::ControlBindings}}}};
+ controlled_condition.same_type_as = "condition";
+ registry["controlnet::apply"] = make_operation(
+ {required("condition", Kind::Dict), required("model", Kind::ControlNet), required("hint", Kind::Tensor),
+ number("strength", 0, 100, 1), start_sigma, end_sigma,
+ optional("type", Kind::String, "none")},
+ std::move(controlled_condition), apply_controlnet);
+ OutputSignature adapted_condition{{Kind::Dict}, {{"ip_adapters", {Kind::IPAdapterBindings}}}};
+ adapted_condition.same_type_as = "condition";
+ registry["ip_adapter::apply"] = make_operation(
+ {required("condition", Kind::Dict), required("adapter", Kind::IPAdapter), required("tokens", Kind::Tensor),
+ number("strength", 0, 100, 1), start_sigma, end_sigma},
+ std::move(adapted_condition), apply_ip_adapter);
+ OutputSignature pulid_condition{{Kind::Dict}, {{"pulid", {Kind::PuLIDBindings}}}};
+ pulid_condition.same_type_as = "condition";
+ registry["pulid::apply"] = make_operation(
+ {required("condition", Kind::Dict), required("pulid", Kind::PuLID), required("embedding", Kind::Tensor),
+ number("strength", 0, 100, 1), start_sigma, end_sigma},
+ std::move(pulid_condition), apply_pulid);
+ registry["animatediff::apply"] = make_operation(
+ {required("model", Kind::Denoiser), required("motion", Kind::AnimateDiff)},
+ {Kind::Denoiser}, apply_animatediff);
+ auto transform_fields = scaling_fields;
+ transform_fields.emplace("timestep", Type{Kind::Number});
+ auto step_fields = transform_fields;
+ step_fields.emplace("sigma", Type{Kind::Number});
+ auto transform = optional("transform", Kind::Function);
+ transform.callback = CallbackSignature{"transform_args", std::move(step_fields), std::move(transform_fields)};
+ std::vector<std::string> prediction_ids{"auto"};
+ for (int i = 0; i < PREDICTION_COUNT; ++i)
+ prediction_ids.emplace_back(sd_prediction_name(static_cast<prediction_t>(i)));
+ registry["predictor"] = make_operation(
+ {required("model", Kind::Denoiser), required("condition", Kind::Dict), choice("prediction", std::move(prediction_ids), "auto"), optional("concat", Kind::Tensor),
+ optional_list("ref_latents", Kind::Tensor), optional_list("ref_audios", Kind::Tensor),
+ optional_list("reference_blocks", Kind::Dict), optional_list("keyframe_indices", Kind::Integer),
+ choice("ref_index_mode", {"fixed", "increase", "decrease"}, "fixed"), optional("force_ref_timestep_zero", Kind::Boolean, false),
+ number("guidance", 0, 1000, 3.5), optional_list("skip_layers", Kind::Integer),
+ optional("vace_context", Kind::Tensor), number("vace_strength", 0, 100, 1),
+ optional("audio_embedding", Kind::Tensor), optional("video_positions", Kind::Tensor),
+ integer("audio_length", 0, INT_MAX, 0),
+ number("fps", 0.01, 100000, 24), number("video_sigma_shift", 0.001, 1000, 12), number("audio_sigma_shift", 0.001, 1000, 3),
+ optional("circular_x", Kind::Boolean, false), optional("circular_y", Kind::Boolean, false),
+ std::move(transform), optional("transform_args", Kind::Dict)},
+ {Kind::Predictor}, make_predictor);
+ const std::vector<Argument> guidance_branches{
+ required("positive", Kind::Predictor), required("negative", Kind::Predictor), number("scale", 0, 1000),
+ optional("image_only", Kind::Predictor), number("image_scale", 0, 1000, 1), optional_list("schedule", Kind::Number)};
+ registry["cfg"] = make_operation(guidance_branches, {Kind::Predictor}, make_cfg);
+ auto apg = guidance_branches;
+ apg.insert(apg.end(),
+ {number("eta", -sigma_limit, sigma_limit, 1), number("momentum", -sigma_limit, sigma_limit, 0),
+ number("norm_threshold", 0, sigma_limit, 0), number("norm_threshold_smoothing", 0, sigma_limit, 0)});
+ registry["apg"] = make_operation(std::move(apg), {Kind::Predictor}, make_apg);
+ registry["switch_predictor"] = make_operation(
+ {required("before", Kind::Predictor), required("after", Kind::Predictor), integer("after_steps", 0, 10000)},
+ {Kind::Predictor}, switch_predictor);
+ auto start_step_fraction = number("start_step_fraction", 0, 1);
+ auto stop_step_fraction = number("stop_step_fraction", 0, 1);
+ start_step_fraction.required = stop_step_fraction.required = false;
+ registry["slg"] = make_operation(
+ {required("base", Kind::Predictor), required("positive", Kind::Predictor), required("skipped", Kind::Predictor),
+ number("scale", 0, 1000), start_sigma, end_sigma, std::move(start_step_fraction), std::move(stop_step_fraction)},
+ {Kind::Predictor}, make_slg);
+ registry["preview"] = make_operation(
+ {optional("decoder", Kind::VAE), optional("circular_x", Kind::Boolean, false), optional("circular_y", Kind::Boolean, false)},
+ {Kind::SamplingObserver}, make_preview);
+ std::vector<std::string> sample_methods;
+ for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i)
+ sample_methods.emplace_back(sd_sample_method_name(static_cast<sample_method_t>(i)));
+ std::vector<Argument> sampling_arguments{
+ required("start", Kind::Tensor), required("sigmas", Kind::Tensor), required("predict", Kind::Predictor),
+ required("rng", Kind::Rng), choice("method", std::move(sample_methods), "euler"), number("eta", 0, 100, 1),
+ optional("base", Kind::Tensor), optional("denoise_mask", Kind::Tensor), optional("mask_input", Kind::Boolean, false),
+ optional("extra_args", Kind::String, ""),
+ optional("finish", Kind::Boolean),
+ optional("cache", Kind::Dict), optional("observer", Kind::SamplingObserver)};
+ registry["sample"] = make_operation(sampling_arguments, {Kind::Tensor}, sample_tensor);
+ registry["sample"].sampling = true;
+ sampling_arguments.insert(sampling_arguments.begin() + 2, required("semantic_sigmas", Kind::Tensor));
+ registry["sefi::sample"] = make_operation(std::move(sampling_arguments), {Kind::Tensor}, sample_sefi_tensor);
+ registry["sefi::sample"].sampling = true;
+ }
+
+ static void register_latent_operations(Registry& registry) {
+ registry["audio_latent_shape"] = make_operation(
+ {required("vae", Kind::AudioVAE), integer("frames", 1, 65536), integer("fps", 1, 1000)},
+ {Kind::Tensor}, execute_audio_latent_shape);
+ registry["ltx::video_positions"] = make_operation(
+ {required("like", Kind::Tensor), integer("keyframe_latent_frames", 1, 65536),
+ integer("keyframe_frame_index", 0, 65536), integer("keyframe_pixel_frames", 1, 65536),
+ integer("fps", 1, 1000, 24), integer("spatial_scale", 1, 128, 32),
+ integer("temporal_scale", 1, 128, 8), optional("causal", Kind::Boolean, true)},
+ {Kind::Tensor}, execute_ltx_video_positions);
+ registry["wan::s2v_audio_window"] = make_operation(
+ {required("embedding", Kind::Tensor), integer("latent_frames", 1, 65536)}, {Kind::Tensor}, execute_s2v_audio_window);
+ registry["encode_vision"] = make_operation(
+ {required("encoder", Kind::VisionEncoder), required("image", Kind::Tensor), optional("pooled", Kind::Boolean, false),
+ integer("clip_skip", -16, 1024, -2), optional("zero", Kind::Boolean, false),
+ optional("zero_input", Kind::Boolean, false)},
+ {Kind::Tensor}, encode_vision);
+ registry["encode_audio"] = make_operation(
+ {required("encoder", Kind::AudioEncoder), required("samples", Kind::Tensor), integer("sample_rate", 1000, 384000)}, {Kind::Tensor}, encode_audio);
+ registry["ip_adapter::project"] = make_operation(
+ {required("adapter", Kind::IPAdapter), required("embedding", Kind::Tensor)}, {Kind::Tensor}, encode_ip_adapter);
+ auto width = integer("width", 1, 16384), height = integer("height", 1, 16384);
+ width.required = height.required = false;
+ registry["upscale_latent"] = make_operation(
+ {required("upscaler", Kind::LatentUpscaler), required("latent", Kind::Tensor), std::move(width), std::move(height)},
+ {Kind::Tensor}, upscale_latent);
+ registry["audio_vae_encode"] = make_operation(
+ {required("vae", Kind::AudioVAE), required("samples", Kind::Tensor), integer("sample_rate", 1000, 384000)}, {Kind::Tensor}, audio_vae_encode);
+ registry["audio_vae_decode"] = make_operation(
+ {required("vae", Kind::AudioVAE), required("latent", Kind::Tensor)}, audio_signature(), audio_vae_decode);
+ }
+
+ static void register_video_restoration_operations(Registry& registry) {
+ registry["seedvr2::positive_embedding"] = make_operation(
+ {required("dit", Kind::Denoiser)}, {Kind::Tensor}, seedvr2_positive_embedding);
+ registry["relayvsr::positive_embedding"] = make_operation(
+ {required("dit", Kind::Denoiser)}, {Kind::Tensor}, positive_embedding);
+ registry["relayvsr::latent_shape"] = make_operation(
+ {required("dit", Kind::Denoiser), required("keyframes", Kind::Tensor)}, {Kind::Tensor}, relay_latent_shape);
+ registry["relayvsr::decode"] = make_operation(
+ {required("flash", Kind::VAE), required("latent", Kind::Tensor), required("frames", Kind::Tensor),
+ required("keyframe_indices", Kind::Tensor), number("detail", 0, 1, 1)},
+ {Kind::Tensor}, relay_decode);
+ }
+
+ static void register_detailer_operations(Registry& registry) {
+ auto input_size = optional("input_size", Kind::Integer, 640);
+ input_size.validate = [](const Value& value) {
+ auto size = value.scalar.get<int64_t>();
+ if (size < 1 || size > 4096)
+ throw std::runtime_error("input_size must be between 1 and 4096");
+ };
+ registry["detections::detect"] = make_operation(
+ {
+ required("detector", Kind::Detector),
+ required("image", Kind::Tensor),
+ std::move(input_size),
+ number("confidence", 0, 1, 0.3),
+ number("nms", 0, 1, 0.45),
+ integer("max_detections", 1, 4096, 100),
+ },
+ detections_signature(),
+ detect_objects);
+ registry["detections::filter"] = make_operation(
+ {
+ required("image", Kind::Tensor),
+ required("detections", Kind::Dict),
+ integer("top_k", 0, INT_MAX, 0),
+ choice("top_k_by", {"area", "confidence"}, "area"),
+ number("min_area_ratio", 0, 1, 0),
+ number("max_area_ratio", 0, 1, 1),
+ choice("sort_by", {"none", "left_to_right", "center_to_edge", "area"}, "none"),
+ },
+ detections_signature(), filter_detections);
+ registry["detections::box_masks"] = make_operation(
+ {required("image", Kind::Tensor), required("detections", Kind::Dict)},
+ {Kind::List, Kind::Tensor}, detection_box_masks);
+ registry["detections::to_regions"] = make_operation(
+ {required("image", Kind::Tensor), required("detections", Kind::Dict)},
+ regions_signature(), detections_to_regions);
+ auto aspect = int_vector_argument("aspect");
+ aspect.required = false;
+ aspect.default_value = Value::list({Value::literal(1), Value::literal(1)});
+ registry["regions::plan_mask_crops"] = make_operation(
+ {
+ list_argument("masks", Kind::Tensor),
+ std::move(aspect),
+ integer("padding", 0, 16384, 32),
+ optional("only_masked", Kind::Boolean, true),
+ },
+ {
+ {Kind::List, Kind::Dict},
+ {
+ {"box", {Kind::Tensor}},
+ {"mask", {Kind::Tensor}},
+ {"index", {Kind::Integer}},
+ },
+ },
+ plan_mask_crops);
+ }
+
+ void register_builtin_operations(Registry& registry) {
+ registry["media::save"] = make_operation(
+ {required("path", Kind::String), required("media", Kind::Dict)}, {Kind::Void}, execute_save_media);
+ register_value_operations(registry);
+ register_rng_operations(registry);
+ register_tensor_operations(registry);
+ register_image_operations(registry);
+ register_region_operations(registry);
+ register_video_operations(registry);
+ register_audio_operations(registry);
+ register_model_operations(registry);
+ register_component_operations(registry);
+ register_latent_operations(registry);
+ register_video_restoration_operations(registry);
+ register_detailer_operations(registry);
+ for (const auto* name : {"latent_shape", "audio_latent_shape", "make_sigmas", "t_to_sigma", "denoiser_scalings", "relayvsr::latent_shape",
+ "add_noise", "vae_to_dit_latents", "dit_to_vae_latents", "list::replace", "zero_out"})
+ if (auto found = registry.find(name); found != registry.end())
+ found->second.model_use = Operation::ModelUse::Metadata;
+ for (const auto* name : {"controlnet::apply", "ip_adapter::apply", "pulid::apply"})
+ registry.at(name).model_use = Operation::ModelUse::Capture;
+ registry.at("preview").model_use = Operation::ModelUse::Preview;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/registration.h b/src/dgl/bindings/registration.h
new file mode 100644
index 0000000..ebf0871
--- /dev/null
+++ b/src/dgl/bindings/registration.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_BINDINGS_REGISTRATION_H__
+#define __SD_DGL_BINDINGS_REGISTRATION_H__
+
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+
+ void register_builtin_operations(Registry& registry);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_REGISTRATION_H__
diff --git a/src/dgl/bindings/relay_vsr.cpp b/src/dgl/bindings/relay_vsr.cpp
new file mode 100644
index 0000000..ace4e7c
--- /dev/null
+++ b/src/dgl/bindings/relay_vsr.cpp
@@ -0,0 +1,51 @@
+#include "dgl/bindings/relay_vsr.h"
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/values.h"
+#include "dgl/models/diffusion_model.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/vae.h"
+#include "dgl/ops/relay_vsr.h"
+
+namespace sd::dgl {
+ template <typename T>
+ static T& relay_component(T& engine) {
+ if (engine.version() != VERSION_RELAY_VSR)
+ throw std::runtime_error("expected a RelayVSR component");
+ return engine;
+ }
+
+ Value positive_embedding(Values& args, RunState&) {
+ auto& engine = relay_component(*object<models::DiffusionModel>(args, "dit")->dit);
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ return tensor_value(sd::dgl::ops::relay_vsr_positive_embedding(*engine.diffusion_model, *engine.model_weights));
+ }
+
+ Value relay_latent_shape(Values& args, RunState&) {
+ auto& engine = relay_component(*object<models::DiffusionModel>(args, "dit")->dit);
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ const auto frames = object<sd_tensor_t>(args, "keyframes");
+ const auto& shape = frames->shape();
+ if (frames->type != SD_TENSOR_U8 || shape.size() != 4 || shape[0] != 3 || !frames->bytes || shape[1] > 16384 || shape[2] > 16384)
+ throw std::runtime_error("RelayVSR keyframes require nonempty U8 [3, width, height, time]");
+ return shape_value(sd::dgl::ops::relay_vsr_latent_shape(*engine.diffusion_model, shape[1], shape[2], shape[3]));
+ }
+
+ Value relay_decode(Values& args, RunState&) {
+ auto& engine = relay_component(*object<VAE>(args, "flash"));
+ Model::ExecutionScope scope(engine);
+ scope.require_ready();
+ const auto& frames = args.at("frames");
+ const auto* raw = frames.get_if<sd_tensor_t>();
+ if (!raw || raw->shape().size() != 4 || !raw->bytes || raw->shape()[1] > 16384 || raw->shape()[2] > 16384)
+ throw std::runtime_error("RelayVSR frames require nonempty U8 [3, width, height, time]");
+ auto output = sd::dgl::ops::relay_vsr_decode(
+ *engine.first_stage_model, engine.n_threads, float_tensor(args, "latent"),
+ read_tensor<uint8_t>(frames), read_tensor<int64_t>(args.at("keyframe_indices")),
+ float(get_number(args, "detail")));
+ return Value::object(make_tensor(std::move(output)));
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/relay_vsr.h b/src/dgl/bindings/relay_vsr.h
new file mode 100644
index 0000000..9af0f7a
--- /dev/null
+++ b/src/dgl/bindings/relay_vsr.h
@@ -0,0 +1,14 @@
+#ifndef __SD_DGL_BINDINGS_RELAY_VSR_H__
+#define __SD_DGL_BINDINGS_RELAY_VSR_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value positive_embedding(Values& args, RunState&);
+ Value relay_latent_shape(Values& args, RunState&);
+ Value relay_decode(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_RELAY_VSR_H__
diff --git a/src/dgl/bindings/rng.cpp b/src/dgl/bindings/rng.cpp
new file mode 100644
index 0000000..a2758ad
--- /dev/null
+++ b/src/dgl/bindings/rng.cpp
@@ -0,0 +1,27 @@
+#include "dgl/bindings/rng.h"
+
+#include <utility>
+
+#include "core/option_types.h"
+#include "core/rng.hpp"
+#include "core/rng_mt19937.hpp"
+#include "core/rng_philox.hpp"
+
+namespace sd::dgl {
+
+ static std::shared_ptr<RNG> make_rng(rng_type_t type) {
+ if (type == STD_DEFAULT_RNG)
+ return std::make_shared<STDDefaultRNG>();
+ if (type == CPU_RNG)
+ return std::make_shared<MT19937RNG>();
+ return std::make_shared<PhiloxRNG>();
+ }
+
+ Value execute_make_rng(Values& args, RunState&) {
+ const auto algorithm = get_string(args, "algorithm");
+ auto rng = make_rng(algorithm == "std" ? STD_DEFAULT_RNG : str_to_rng_type(algorithm.c_str()));
+ rng->manual_seed(get_int(args, "seed"));
+ return Value::object(std::move(rng));
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/rng.h b/src/dgl/bindings/rng.h
new file mode 100644
index 0000000..fd5b3e8
--- /dev/null
+++ b/src/dgl/bindings/rng.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_BINDINGS_RNG_H__
+#define __SD_DGL_BINDINGS_RNG_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value execute_make_rng(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_RNG_H__
diff --git a/src/dgl/bindings/seedvr2.cpp b/src/dgl/bindings/seedvr2.cpp
new file mode 100644
index 0000000..b81fb4f
--- /dev/null
+++ b/src/dgl/bindings/seedvr2.cpp
@@ -0,0 +1,16 @@
+#include "dgl/bindings/seedvr2.h"
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/values.h"
+#include "dgl/models/diffusion_model.h"
+#include "dgl/models/dit.h"
+#include "dgl/ops/seedvr2.h"
+
+namespace sd::dgl {
+ Value seedvr2_positive_embedding(Values& args, RunState&) {
+ auto& dit = *object<models::DiffusionModel>(args, "dit")->dit;
+ Model::ExecutionScope scope(dit);
+ scope.require_ready();
+ return tensor_value(ops::seedvr2_positive_embedding(dit));
+ }
+}
diff --git a/src/dgl/bindings/seedvr2.h b/src/dgl/bindings/seedvr2.h
new file mode 100644
index 0000000..1d75277
--- /dev/null
+++ b/src/dgl/bindings/seedvr2.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_BINDINGS_SEEDVR2_H__
+#define __SD_DGL_BINDINGS_SEEDVR2_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value seedvr2_positive_embedding(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_SEEDVR2_H__
diff --git a/src/dgl/bindings/sefi.cpp b/src/dgl/bindings/sefi.cpp
new file mode 100644
index 0000000..127f794
--- /dev/null
+++ b/src/dgl/bindings/sefi.cpp
@@ -0,0 +1,26 @@
+#include "dgl/bindings/sefi.h"
+
+#include "dgl/bindings/arguments.h"
+#include "dgl/bindings/values.h"
+
+#include "dgl/ops/predictor.h"
+#include "dgl/ops/sefi_sampling.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl {
+ Value sefi_sigmas(Values& args, RunState&) {
+ SefiFlowDenoiser denoiser;
+ denoiser.timestep_shift_alpha = float(get_number(args, "alpha"));
+ denoiser.delta_t = float(get_number(args, "delta"));
+ auto sigmas = denoiser.get_sigmas(int(get_int(args, "steps")), 0, DISCRETE_SCHEDULER, VERSION_SEFI_IMAGE);
+ return Value::dict({{"sigmas", tensor_value(sd::Tensor<float>::from_vector(std::move(sigmas)))},
+ {"semantic_sigmas", tensor_value(sd::Tensor<float>::from_vector(std::move(denoiser.sem_sigmas)))}});
+ }
+
+ Value sample_sefi_tensor(Values& args, RunState& state) {
+ const auto predictor = object<ops::Predictor>(args, "predict");
+ const auto options = sampling_options(args, state.context);
+ return tensor_value(ops::sefi_sample(float_tensor(args, "start"), sigma_tensor(args, "sigmas"),
+ sigma_tensor(args, "semantic_sigmas"), *predictor, options, state.events));
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/sefi.h b/src/dgl/bindings/sefi.h
new file mode 100644
index 0000000..f6d8db6
--- /dev/null
+++ b/src/dgl/bindings/sefi.h
@@ -0,0 +1,13 @@
+#ifndef __SD_DGL_BINDINGS_SEFI_H__
+#define __SD_DGL_BINDINGS_SEFI_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value sefi_sigmas(Values& args, RunState&);
+ Value sample_sefi_tensor(Values& args, RunState& state);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_SEFI_H__
diff --git a/src/dgl/bindings/tensor_ops.cpp b/src/dgl/bindings/tensor_ops.cpp
new file mode 100644
index 0000000..e0d2b3e
--- /dev/null
+++ b/src/dgl/bindings/tensor_ops.cpp
@@ -0,0 +1,377 @@
+#include "dgl/bindings/tensor_ops.h"
+
+#include "dgl/bindings/conversion.h"
+
+#include <algorithm>
+#include <cmath>
+#include <type_traits>
+
+#include "core/rng.hpp"
+#include "core/tensor.hpp"
+#include "dgl/indexing.h"
+#include "dgl/tensor_io.h"
+
+namespace sd::dgl {
+ using ops::cast_number;
+ using ops::copy_tensor_box;
+ using ops::element_count;
+ using ops::element_size;
+ using ops::rounded_byte;
+ using ops::tensor_axis;
+ using ops::with_float_dtype;
+
+ static const std::map<std::string, sd_tensor_type_t> dtypes{
+ {"u8", SD_TENSOR_U8},
+ {"i32", SD_TENSOR_I32},
+ {"i64", SD_TENSOR_I64},
+ {"f32", SD_TENSOR_F32},
+ {"f64", SD_TENSOR_F64},
+ };
+
+ static std::vector<int64_t> int_list(const Values& args, const char* name) {
+ return read_int_vector(args.at(name));
+ }
+
+ static Value tensor_result(sd_tensor_type_t dtype, std::vector<int64_t> shape, const void* data, size_t bytes) {
+ return Value::object(make_tensor(dtype, std::move(shape), data, bytes));
+ }
+
+ template <typename T>
+ static Value tensor_result(sd::Tensor<T> tensor, bool scalar = false) {
+ return Value::object(make_tensor(std::move(tensor), scalar));
+ }
+
+ template <typename T>
+ static Value tensor_result(std::vector<int64_t> shape, std::vector<T> values) {
+ if (values.empty())
+ return tensor_result(tensor_dtype<T>(), std::move(shape), nullptr, 0);
+ const bool scalar = shape.empty();
+ if (scalar)
+ shape.push_back(1);
+ return tensor_result(sd::Tensor<T>(std::move(shape), std::move(values)), scalar);
+ }
+
+ static bool little_endian() {
+ const uint16_t endian = 1;
+ return *reinterpret_cast<const uint8_t*>(&endian) != 0;
+ }
+
+ static void swap_endian(uint8_t* bytes, size_t size, size_t stride) {
+ for (size_t i = 0; i < size; i += stride)
+ std::reverse(bytes + i, bytes + i + stride);
+ }
+
+ template <typename T>
+ static T scalar_number(const Value& value) {
+ if (value.type().kind == Kind::Integer)
+ return cast_number<T>(value.scalar.get<int64_t>());
+ if (value.type().kind == Kind::Number)
+ return cast_number<T>(value.scalar.get<double>());
+ throw std::runtime_error("expected a number or tensor");
+ }
+
+ static ops::TensorNumber number_value(const Value& value) {
+ if (value.type().kind == Kind::Integer)
+ return value.scalar.get<int64_t>();
+ if (value.type().kind == Kind::Number)
+ return value.scalar.get<double>();
+ throw std::runtime_error("expected a scalar number");
+ }
+
+ static Value filled_tensor(std::vector<int64_t> shape, sd_tensor_type_t dtype, const Value& value) {
+ return Value::object(ops::filled_tensor(std::move(shape), dtype, number_value(value)));
+ }
+
+ template <typename T>
+ static const sd::Tensor<T>& numeric_operand(const Value& value, sd::Tensor<T>& scalar) {
+ if (const auto* tensor = value.get_if<sd_tensor_t>())
+ return tensor->as<T>();
+ scalar = sd::Tensor<T>({1}, {scalar_number<T>(value)});
+ return scalar;
+ }
+
+ template <class F>
+ static Value numeric_operation(std::initializer_list<const Value*> operands, F compute) {
+ sd_tensor_type_t dtype = 0;
+ std::vector<int64_t> shape;
+ for (const auto* operand : operands) {
+ if (operand->type().kind == Kind::Tensor) {
+ const auto& tensor = *operand->as<sd_tensor_t>();
+ if (dtype && dtype != tensor.type)
+ throw std::runtime_error("numeric tensor operands must have matching dtypes");
+ dtype = tensor.type;
+ shape = sd::tensor_broadcast_shape(shape, tensor.shape());
+ } else if (operand->type().kind != Kind::Number && operand->type().kind != Kind::Integer) {
+ throw std::runtime_error("expected a number or tensor");
+ }
+ }
+ if (!dtype)
+ dtype = SD_TENSOR_F32;
+ const auto count = element_count(shape, dtype);
+ return with_float_dtype(dtype, [&](auto tag) {
+ if (!count)
+ return tensor_result(dtype, std::move(shape), nullptr, 0);
+ auto output = compute(tag);
+ return tensor_result(std::move(output), shape.empty());
+ });
+ }
+
+ static const sd_tensor_t* tensor_operand(const Value& value) {
+ return value.get_if<sd_tensor_t>();
+ }
+
+ static std::vector<int64_t> spatial_axes(const Values& args) {
+ return args.count("axes") ? int_list(args, "axes") : std::vector<int64_t>{0, 1};
+ }
+
+ static Value pool_tensor(const Values& args, bool maximum) {
+ std::optional<ops::TensorNumber> pad;
+ if (args.count("pad_value")) {
+ const auto& value = args.at("pad_value");
+ if (value.type().kind == Kind::Integer)
+ pad = value.scalar.get<int64_t>();
+ else
+ pad = value.scalar.get<double>();
+ }
+ auto before = args.count("padding_before") ? int_list(args, "padding_before") : std::vector<int64_t>{0, 0};
+ return Value::object(ops::pool_tensor(*object<sd_tensor_t>(args, "input"), spatial_axes(args), int_list(args, "kernel"),
+ args.count("stride") ? int_list(args, "stride") : std::vector<int64_t>{1, 1}, before,
+ args.count("padding_after") ? int_list(args, "padding_after") : before, pad, maximum));
+ }
+
+ Value blur_tensor(const Values& args, RunState&) {
+ return Value::object(ops::blur_tensor(object<sd_tensor_t>(args, "input"), spatial_axes(args), get_int(args, "radius"), get_number(args, "sigma"), get_string(args, "precision")));
+ }
+
+ Value execute_load_tensor(Values& args, RunState&) {
+ return Value::object(load_npy(get_string(args, "path"), tensor_byte_limit));
+ }
+
+ Value execute_save_tensor(Values& args, RunState&) {
+ save_npy_atomic(*object<sd_tensor_t>(args, "tensor"), get_string(args, "path"));
+ return Value::void_value();
+ }
+
+ Value execute_tensor_literal(Values& args, RunState&) {
+ const auto type = dtypes.at(get_string(args, "dtype"));
+ auto dims = int_list(args, "shape");
+ element_count(dims, type);
+ auto result = make_tensor(type, std::move(dims));
+ auto* data = static_cast<uint8_t*>(result->mutable_data());
+ decode_base64(get_string(args, "data"), data, result->bytes);
+ if (!little_endian())
+ swap_endian(data, result->bytes, element_size(type));
+ return Value::object(std::move(result));
+ }
+
+ Value execute_tensor_shape(Values& args, RunState&) {
+ return shape_value(object<sd_tensor_t>(args, "input")->shape());
+ }
+
+ Value execute_tensor_index(Values& args, RunState&) {
+ return index_value(args.at("input"), args.at("indices"));
+ }
+ Value execute_tensor_vector(Values& args, RunState&) {
+ const auto type = dtypes.at(get_string(args, "dtype"));
+ const auto& items = args.at("values").items;
+ std::vector<int64_t> dims{int64_t(items.size())};
+ element_count(dims, type);
+ return with_dtype(type, [&](auto tag) {
+ using T = decltype(tag);
+ std::vector<T> data;
+ data.reserve(items.size());
+ for (const auto& item : items)
+ data.push_back(scalar_number<T>(item));
+ return tensor_result(std::move(dims), std::move(data));
+ });
+ }
+
+ Value execute_tensor_to_list(Values& args, RunState&) {
+ const auto input = object<sd_tensor_t>(args, "input");
+ if (input->shape().size() != 1)
+ throw std::runtime_error("tensor::to_list requires a rank-one tensor");
+ const auto count = input->shape()[0];
+ if (count > 10000)
+ throw std::runtime_error("tensor::to_list supports at most 10000 elements");
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ constexpr auto kind = std::is_integral_v<T> ? Kind::Integer : Kind::Number;
+ const auto element = get_string(args, "element");
+ if (element != "auto" && element != (kind == Kind::Integer ? "int" : "number"))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "tensor::to_list element does not match input dtype; use tensor::cast to convert it");
+ std::vector<Value> items;
+ items.reserve(size_t(count));
+ const auto* data = input->as<T>().data();
+ for (int64_t i = 0; i < count; ++i) {
+ if constexpr (std::is_integral_v<T>)
+ items.push_back(Value::literal(int64_t(data[i])));
+ else
+ items.push_back(Value::literal(double(data[i])));
+ }
+ return Value::list(std::move(items), kind);
+ });
+ }
+
+ Value execute_tensor_reshape(Values& args, RunState&) {
+ const auto input = object<sd_tensor_t>(args, "input");
+ auto dims = int_list(args, "shape");
+ if (element_count(dims, input->type) * element_size(input->type) != input->bytes)
+ throw std::runtime_error("reshape must preserve element count");
+ if (dims == input->shape())
+ return args.at("input");
+ return tensor_result(input->type, std::move(dims), input->data(), input->bytes);
+ }
+
+ Value execute_tensor_unsqueeze(Values& args, RunState&) {
+ return Value::object(ops::unsqueeze_tensor(object<sd_tensor_t>(args, "input"), get_int(args, "axis")));
+ }
+
+ Value execute_tensor_pad(Values& args, RunState&) {
+ return Value::object(ops::pad_tensor(object<sd_tensor_t>(args, "input"), int_list(args, "pad"), number_value(args.at("value"))));
+ }
+
+ Value execute_tensor_crop(Values& args, RunState&) {
+ return Value::object(crop_tensor(*object<sd_tensor_t>(args, "input"), int_list(args, "axes"),
+ int_list(args, "start"), int_list(args, "size")));
+ }
+
+ Value execute_tensor_numel(Values& args, RunState&) {
+ const auto input = object<sd_tensor_t>(args, "input");
+ return Value::literal(int64_t(input->bytes / element_size(input->type)));
+ }
+
+ Value execute_tensor_zeros(Values& args, RunState&) {
+ return filled_tensor(int_list(args, "shape"), dtypes.at(get_string(args, "dtype")), Value::literal(0));
+ }
+
+ Value execute_tensor_full(Values& args, RunState&) {
+ return filled_tensor(int_list(args, "shape"), dtypes.at(get_string(args, "dtype")), args.at("value"));
+ }
+
+ Value execute_tensor_zeros_like(Values& args, RunState&) {
+ const auto like = object<sd_tensor_t>(args, "like");
+ return filled_tensor(like->shape(), like->type, Value::literal(0));
+ }
+
+ Value execute_tensor_lerp(Values& args, RunState&) {
+ const auto& a = args.at("a");
+ const auto& b = args.at("b");
+ const auto& w = args.at("weight");
+ const auto* left = tensor_operand(a);
+ const auto* right = tensor_operand(b);
+ if ((left && left->type == SD_TENSOR_U8) || (right && right->type == SD_TENSOR_U8)) {
+ if (!left || !right)
+ throw std::runtime_error("u8 lerp requires two u8 endpoints");
+ return Value::object(ops::lerp_bytes(*left, *right, tensor_operand(w), tensor_operand(w) ? 0 : scalar_number<double>(w)));
+ }
+ return numeric_operation({&a, &b, &w}, [&](auto tag) {
+ using T = decltype(tag);
+ sd::Tensor<T> scalar_a, scalar_b, scalar_weight;
+ const auto& weight = numeric_operand(w, scalar_weight);
+ return numeric_operand(a, scalar_a) * (T(1) - weight) + numeric_operand(b, scalar_b) * weight;
+ });
+ }
+
+ Value execute_tensor_affine(Values& args, RunState&) {
+ const auto& input = args.at("input");
+ return numeric_operation({&input}, [&](auto tag) {
+ using T = decltype(tag);
+ sd::Tensor<T> literal;
+ return numeric_operand(input, literal) * scalar_number<T>(args.at("scale")) + scalar_number<T>(args.at("offset"));
+ });
+ }
+
+ Value execute_tensor_clamp(Values& args, RunState&) {
+ const auto& input = args.at("input");
+ if (get_number(args, "min") > get_number(args, "max"))
+ throw std::runtime_error("clamp min exceeds max");
+ return numeric_operation({&input}, [&](auto tag) {
+ using T = decltype(tag);
+ sd::Tensor<T> literal;
+ return sd::ops::clamp(numeric_operand(input, literal), scalar_number<T>(args.at("min")), scalar_number<T>(args.at("max")));
+ });
+ }
+
+ Value execute_tensor_max_pool2d(Values& args, RunState&) {
+ return pool_tensor(args, true);
+ }
+
+ Value execute_tensor_min_pool2d(Values& args, RunState&) {
+ return pool_tensor(args, false);
+ }
+
+ Value execute_tensor_interpolate(Values& args, RunState&) {
+ const auto axes = args.count("axes") ? int_list(args, "axes") : std::vector<int64_t>{0, 1};
+ return Value::object(interpolate_tensor(*object<sd_tensor_t>(args, "input"), int_list(args, "size"),
+ axes, get_string(args, "mode"), get_bool(args, "antialias")));
+ }
+
+ template <typename F>
+ static Value binary_tensor(const Values& args, F compute) {
+ const auto& a = args.at("a");
+ const auto& b = args.at("b");
+ return numeric_operation({&a, &b}, [&](auto tag) {
+ using T = decltype(tag);
+ sd::Tensor<T> scalar_a, scalar_b;
+ return compute(numeric_operand(a, scalar_a), numeric_operand(b, scalar_b));
+ });
+ }
+
+ Value execute_tensor_add(Values& args, RunState&) {
+ return binary_tensor(args, [](const auto& a, const auto& b) { return a + b; });
+ }
+
+ Value execute_tensor_mul(Values& args, RunState&) {
+ return binary_tensor(args, [](const auto& a, const auto& b) { return a * b; });
+ }
+
+ Value execute_tensor_div(Values& args, RunState&) {
+ return binary_tensor(args, [](const auto& a, const auto& b) { return a / b; });
+ }
+
+ Value cast_tensor(const Values& args, RunState&) {
+ return Value::object(ops::cast_tensor(object<sd_tensor_t>(args, "input"), dtypes.at(get_string(args, "dtype"))));
+ }
+ Value translate_tensor(const Values& args, RunState&) {
+ return Value::object(ops::translate_tensor(object<sd_tensor_t>(args, "input"), int_list(args, "axes"), int_list(args, "offset"), number_value(args.at("value"))));
+ }
+ Value arange_tensor(const Values& args, RunState&) {
+ return Value::object(ops::arange_tensor(get_int(args, "start"), get_int(args, "end"), get_int(args, "step")));
+ }
+ Value index_select_tensor(const Values& args, RunState&) {
+ return Value::object(ops::index_select_tensor(object<sd_tensor_t>(args, "input"), get_int(args, "axis"), int_list(args, "indices")));
+ }
+ Value replace_slice(const Values& args, RunState&) {
+ return Value::object(ops::replace_slice(object<sd_tensor_t>(args, "input"), object<sd_tensor_t>(args, "value"), get_int(args, "axis"), get_int(args, "start")));
+ }
+ Value concat_tensors(const Values& args, RunState&) {
+ std::vector<std::shared_ptr<sd_tensor_t>> items;
+ for (const auto& item : args.at("items").items)
+ items.push_back(item.as<sd_tensor_t>());
+ return Value::object(ops::concat_tensors(items, get_int(args, "axis")));
+ }
+ Value grayscale_tensor(const Values& args, RunState&) {
+ return Value::object(ops::grayscale_tensor(object<sd_tensor_t>(args, "input"), get_int(args, "axis")));
+ }
+ Value threshold_tensor(const Values& args, RunState&) {
+ return Value::object(ops::threshold_tensor(object<sd_tensor_t>(args, "input"), number_value(args.at("threshold")), number_value(args.at("low")), number_value(args.at("high"))));
+ }
+ Value maximum_tensor(const Values& args, RunState&) {
+ return Value::object(ops::maximum_tensor(object<sd_tensor_t>(args, "a"), object<sd_tensor_t>(args, "b")));
+ }
+ Value execute_tensor_permute(Values& args, RunState&) {
+ return Value::object(ops::permute(object<sd_tensor_t>(args, "input"), int_list(args, "axes")));
+ }
+ Value execute_tensor_broadcast_to(Values& args, RunState&) {
+ return Value::object(ops::broadcast_to(object<sd_tensor_t>(args, "input"), int_list(args, "shape")));
+ }
+ Value execute_tensor_index_fill(Values& args, RunState&) {
+ return Value::object(ops::index_fill(object<sd_tensor_t>(args, "input"), get_int(args, "axis"), int_list(args, "indices"), number_value(args.at("value"))));
+ }
+ Value execute_tensor_randn_like(Values& args, RunState&) {
+ return Value::object(ops::randn_like(object<sd_tensor_t>(args, "like"), object<RNG>(args, "rng")));
+ }
+ Value execute_tensor_count_nonzero(Values& args, RunState&) {
+ return Value::literal(ops::count_nonzero(object<sd_tensor_t>(args, "input")));
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/tensor_ops.h b/src/dgl/bindings/tensor_ops.h
new file mode 100644
index 0000000..92e1984
--- /dev/null
+++ b/src/dgl/bindings/tensor_ops.h
@@ -0,0 +1,50 @@
+#ifndef __SD_DGL_BINDINGS_TENSOR_OPS_H__
+#define __SD_DGL_BINDINGS_TENSOR_OPS_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value blur_tensor(const Values& args, RunState&);
+ Value execute_load_tensor(Values& args, RunState&);
+ Value execute_save_tensor(Values& args, RunState&);
+ Value execute_tensor_literal(Values& args, RunState&);
+ Value execute_tensor_shape(Values& args, RunState&);
+ Value execute_tensor_index(Values& args, RunState&);
+ Value execute_tensor_vector(Values& args, RunState&);
+ Value execute_tensor_to_list(Values& args, RunState&);
+ Value execute_tensor_reshape(Values& args, RunState&);
+ Value execute_tensor_unsqueeze(Values& args, RunState&);
+ Value execute_tensor_pad(Values& args, RunState&);
+ Value execute_tensor_crop(Values& args, RunState&);
+ Value execute_tensor_numel(Values& args, RunState&);
+ Value execute_tensor_zeros(Values& args, RunState&);
+ Value execute_tensor_full(Values& args, RunState&);
+ Value execute_tensor_zeros_like(Values& args, RunState&);
+ Value execute_tensor_lerp(Values& args, RunState&);
+ Value execute_tensor_affine(Values& args, RunState&);
+ Value execute_tensor_clamp(Values& args, RunState&);
+ Value execute_tensor_max_pool2d(Values& args, RunState&);
+ Value execute_tensor_min_pool2d(Values& args, RunState&);
+ Value execute_tensor_interpolate(Values& args, RunState&);
+ Value execute_tensor_add(Values& args, RunState&);
+ Value execute_tensor_mul(Values& args, RunState&);
+ Value execute_tensor_div(Values& args, RunState&);
+ Value cast_tensor(const Values& args, RunState&);
+ Value translate_tensor(const Values& args, RunState&);
+ Value arange_tensor(const Values& args, RunState&);
+ Value index_select_tensor(const Values& args, RunState&);
+ Value replace_slice(const Values& args, RunState&);
+ Value concat_tensors(const Values& args, RunState&);
+ Value grayscale_tensor(const Values& args, RunState&);
+ Value threshold_tensor(const Values& args, RunState&);
+ Value maximum_tensor(const Values& args, RunState&);
+ Value execute_tensor_permute(Values& args, RunState&);
+ Value execute_tensor_broadcast_to(Values& args, RunState&);
+ Value execute_tensor_index_fill(Values& args, RunState&);
+ Value execute_tensor_randn_like(Values& args, RunState&);
+ Value execute_tensor_count_nonzero(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_TENSOR_OPS_H__
diff --git a/src/dgl/bindings/values.cpp b/src/dgl/bindings/values.cpp
new file mode 100644
index 0000000..5e23149
--- /dev/null
+++ b/src/dgl/bindings/values.cpp
@@ -0,0 +1,209 @@
+#include "dgl/bindings/values.h"
+
+#include "dgl/bindings/conversion.h"
+
+#include <algorithm>
+#include <cmath>
+
+namespace sd::dgl {
+ static void validate_options(const Values& settings, const std::string& operation) {
+ const auto& definition = operations().at(operation);
+ for (const auto& setting : settings) {
+ auto argument = std::find_if(definition.arguments.begin(), definition.arguments.end(), [&](const Argument& arg) { return arg.name == setting.first; });
+ if (argument == definition.arguments.end())
+ throw std::runtime_error("unknown " + operation + " option '" + setting.first + "'");
+ const auto& value = setting.second;
+ auto accepts = [](Kind expected, Kind actual) { return expected == actual || (expected == Kind::Number && actual == Kind::Integer); };
+ if (!accepts(argument->type.kind, value.type().kind))
+ throw std::runtime_error("invalid type for " + operation + "." + setting.first);
+ if (value.type().kind == Kind::List)
+ for (const auto& item : value.items)
+ if (!accepts(argument->type.element, item.type().kind))
+ throw std::runtime_error("invalid list element for " + operation + "." + setting.first);
+ if (argument->validate)
+ argument->validate(value);
+ }
+ }
+
+ sd::CacheConfig cache_options(const Values& settings) {
+ sd::CacheConfig params;
+ validate_options(settings, "cache");
+ for (const auto& option : settings) {
+ const auto& key = option.first;
+ const auto& value = option.second.scalar;
+ if (key == "mode") {
+ static const std::map<std::string, sd_cache_mode_t> modes{{"disabled", SD_CACHE_DISABLED}, {"easycache", SD_CACHE_EASYCACHE}, {"ucache", SD_CACHE_UCACHE}, {"dbcache", SD_CACHE_DBCACHE}, {"taylorseer", SD_CACHE_TAYLORSEER}, {"cache_dit", SD_CACHE_CACHE_DIT}, {"spectrum", SD_CACHE_SPECTRUM}};
+ auto found = modes.find(value.get<std::string>());
+ if (found == modes.end())
+ throw std::runtime_error("unknown cache mode");
+ params.mode = found->second;
+ } else if (key == "reuse_threshold")
+ params.reuse_threshold = value.get<float>();
+ else if (key == "start_percent")
+ params.start_percent = value.get<float>();
+ else if (key == "end_percent")
+ params.end_percent = value.get<float>();
+ else if (key == "error_decay_rate")
+ params.error_decay_rate = value.get<float>();
+ else if (key == "use_relative_threshold")
+ params.use_relative_threshold = value.get<bool>();
+ else if (key == "reset_error_on_compute")
+ params.reset_error_on_compute = value.get<bool>();
+ else if (key == "residual_diff_threshold")
+ params.residual_diff_threshold = value.get<float>();
+ else if (key == "max_warmup_steps")
+ params.max_warmup_steps = value.get<int>();
+ else if (key == "max_cached_steps")
+ params.max_cached_steps = value.get<int>();
+ else if (key == "max_continuous_cached_steps")
+ params.max_continuous_cached_steps = value.get<int>();
+ else if (key == "taylorseer_n_derivatives")
+ params.taylorseer_n_derivatives = value.get<int>();
+ else if (key == "taylorseer_skip_interval")
+ params.taylorseer_skip_interval = value.get<int>();
+ else if (key == "scm_mask")
+ params.scm_mask = value.get_ref<const std::string&>().c_str();
+ else if (key == "spectrum_w")
+ params.spectrum_w = value.get<float>();
+ else if (key == "spectrum_m")
+ params.spectrum_m = value.get<int>();
+ else if (key == "spectrum_lam")
+ params.spectrum_lam = value.get<float>();
+ else if (key == "spectrum_window_size")
+ params.spectrum_window_size = value.get<int>();
+ else if (key == "spectrum_flex_window")
+ params.spectrum_flex_window = value.get<float>();
+ else if (key == "spectrum_warmup_steps")
+ params.spectrum_warmup_steps = value.get<int>();
+ else if (key == "spectrum_stop_percent")
+ params.spectrum_stop_percent = value.get<float>();
+ else
+ throw std::runtime_error("unknown cache option '" + key + "'");
+ }
+ if (params.start_percent > params.end_percent)
+ throw std::runtime_error("cache start_percent exceeds end_percent");
+ return params;
+ }
+
+ sd::TilingConfig tiling_options(const Values& settings) {
+ sd::TilingConfig params{};
+ params.enabled = true;
+ params.target_overlap = 0.5f;
+ validate_options(settings, "tiling");
+ for (const auto& option : settings) {
+ const auto& key = option.first;
+ const auto& value = option.second.scalar;
+ if (key == "enabled")
+ params.enabled = value.get<bool>();
+ else if (key == "temporal_tiling")
+ params.temporal_tiling = value.get<bool>();
+ else if (key == "tile_size_w")
+ params.tile_size_w = value.get<int>();
+ else if (key == "tile_size_h")
+ params.tile_size_h = value.get<int>();
+ else if (key == "target_overlap")
+ params.target_overlap = value.get<float>();
+ else if (key == "rel_size_w")
+ params.rel_size_w = value.get<float>();
+ else if (key == "rel_size_h")
+ params.rel_size_h = value.get<float>();
+ else if (key == "extra_args")
+ params.extra_tiling_args = value.get_ref<const std::string&>().c_str();
+ }
+ return params;
+ }
+
+ Value execute_list_replace(Values& args, RunState&) {
+ auto list = args.at("items");
+ const auto index = uint64_t(get_int(args, "index"));
+ const auto& value = args.at("value");
+ if (index >= list.items.size())
+ throw std::runtime_error("list index out of range");
+ if (!accepts_list_element(list.type().element, value.type().kind))
+ throw std::runtime_error("replacement must match the list element type");
+ list.items[size_t(index)] = value;
+ return list;
+ }
+
+ Value execute_math_product(Values& args, RunState&) {
+ int64_t product = 1;
+ for (const auto n : read_int_vector(args.at("items"))) {
+ if (n < 0 || (n && product > INT64_MAX / n))
+ throw std::runtime_error("product requires nonnegative integers and an int64 result");
+ product *= n;
+ }
+ return Value::literal(product);
+ }
+
+ Value execute_math_ceil_div(Values& args, RunState&) {
+ const auto a = get_int(args, "numerator"), b = get_int(args, "denominator");
+ return Value::literal(a / b + int64_t(a % b != 0));
+ }
+
+ Value execute_math_min(Values& args, RunState&) {
+ return Value::literal(std::min(get_int(args, "a"), get_int(args, "b")));
+ }
+
+ Value execute_math_add(Values& args, RunState&) {
+ if (args.at("a").type().kind != Kind::Integer || args.at("b").type().kind != Kind::Integer)
+ return Value::literal(get_number(args, "a") + get_number(args, "b"));
+ const auto a = get_int(args, "a"), b = get_int(args, "b");
+ if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b))
+ throw std::runtime_error("addition exceeds int64 range");
+ return Value::literal(a + b);
+ }
+
+ Value execute_math_sub(Values& args, RunState&) {
+ if (args.at("a").type().kind != Kind::Integer || args.at("b").type().kind != Kind::Integer)
+ return Value::literal(get_number(args, "a") - get_number(args, "b"));
+ const auto a = get_int(args, "a"), b = get_int(args, "b");
+ if ((b > 0 && a < INT64_MIN + b) || (b < 0 && a > INT64_MAX + b))
+ throw std::runtime_error("subtraction exceeds int64 range");
+ return Value::literal(a - b);
+ }
+
+ Value execute_math_mul(Values& args, RunState&) {
+ if (args.at("a").type().kind != Kind::Integer || args.at("b").type().kind != Kind::Integer)
+ return Value::literal(get_number(args, "a") * get_number(args, "b"));
+ const auto a = get_int(args, "a"), b = get_int(args, "b");
+ if ((a > 0 && ((b > 0 && a > INT64_MAX / b) || (b < 0 && b < INT64_MIN / a))) ||
+ (a < 0 && ((b > 0 && a < INT64_MIN / b) || (b < 0 && a < INT64_MAX / b))))
+ throw std::runtime_error("multiplication exceeds int64 range");
+ return Value::literal(a * b);
+ }
+
+ Value execute_math_div(Values& args, RunState&) {
+ const auto divisor = get_number(args, "b");
+ if (divisor == 0.0)
+ throw std::runtime_error("division by zero");
+ return Value::literal(get_number(args, "a") / divisor);
+ }
+
+ Value execute_configuration(Values& args, RunState&) {
+ return Value::dict(args);
+ }
+
+ static Value round_math(const Values& args, double (*round)(double)) {
+ const auto& value = args.at("value");
+ if (value.type().kind == Kind::Integer)
+ return value;
+ const auto result = round(get_number(args, "value"));
+ const auto limit = std::ldexp(1.0, 63);
+ if (!(result >= -limit && result < limit))
+ throw std::runtime_error("rounded value exceeds int64 range");
+ return Value::literal(int64_t(result));
+ }
+
+ Value execute_math_floor(Values& args, RunState&) {
+ return round_math(args, std::floor);
+ }
+
+ Value execute_math_round(Values& args, RunState&) {
+ return round_math(args, std::round);
+ }
+
+ Value execute_math_ceil(Values& args, RunState&) {
+ return round_math(args, std::ceil);
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/bindings/values.h b/src/dgl/bindings/values.h
new file mode 100644
index 0000000..fed829a
--- /dev/null
+++ b/src/dgl/bindings/values.h
@@ -0,0 +1,27 @@
+#ifndef __SD_DGL_BINDINGS_VALUES_H__
+#define __SD_DGL_BINDINGS_VALUES_H__
+
+#include "config.h"
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ sd::CacheConfig cache_options(const Values& settings);
+ sd::TilingConfig tiling_options(const Values& settings);
+
+ Value execute_list_replace(Values& args, RunState&);
+ Value execute_math_product(Values& args, RunState&);
+ Value execute_math_ceil_div(Values& args, RunState&);
+ Value execute_math_min(Values& args, RunState&);
+ Value execute_math_add(Values& args, RunState&);
+ Value execute_math_sub(Values& args, RunState&);
+ Value execute_math_mul(Values& args, RunState&);
+ Value execute_math_div(Values& args, RunState&);
+ Value execute_configuration(Values& args, RunState&);
+ Value execute_math_floor(Values& args, RunState&);
+ Value execute_math_round(Values& args, RunState&);
+ Value execute_math_ceil(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_VALUES_H__
diff --git a/src/dgl/bindings/video.cpp b/src/dgl/bindings/video.cpp
new file mode 100644
index 0000000..86b2bd9
--- /dev/null
+++ b/src/dgl/bindings/video.cpp
@@ -0,0 +1,58 @@
+#include "dgl/bindings/video.h"
+
+#include "dgl/bindings/audio.h"
+#include "dgl/bindings/conversion.h"
+
+namespace sd::dgl {
+ OutputSignature video_signature() {
+ return {{Kind::Dict}, {{"frames", {Kind::Tensor}}, {"fps_num", {Kind::Integer}}, {"fps_den", {Kind::Integer}}}};
+ }
+
+ Value video_result(std::shared_ptr<sd_tensor_t> frames, int64_t fps_num, int64_t fps_den) {
+ return Value::dict({{"frames", Value::object(std::move(frames))},
+ {"fps_num", Value::literal(fps_num)},
+ {"fps_den", Value::literal(fps_den)}});
+ }
+
+ ops::VideoReadResult read_video_dict(const Value& value) {
+ if (value.type().kind != Kind::Dict || !value.fields.count("frames") || !value.fields.count("fps_num") || !value.fields.count("fps_den") ||
+ value.fields.at("frames").type().kind != Kind::Tensor || value.fields.at("fps_num").type().kind != Kind::Integer ||
+ value.fields.at("fps_den").type().kind != Kind::Integer)
+ throw std::runtime_error("video requires a dict with frames (tensor), fps_num and fps_den (integers)");
+ ops::VideoReadResult result;
+ result.frames = value.fields.at("frames").as<sd_tensor_t>();
+ result.fps_num = get_int(value.fields, "fps_num");
+ result.fps_den = get_int(value.fields, "fps_den");
+ if (result.fps_num < 1 || result.fps_num > 1000000 || result.fps_den < 1 || result.fps_den > 1000000)
+ throw std::runtime_error("video frame rate numerator and denominator must be in [1,1000000]");
+ return result;
+ }
+
+ Value execute_load_video(Values& args, RunState&) {
+ const auto source = get_string(args, "path");
+ require_file_access(source, SD_DGL_FILE_READ);
+ const auto path = std::filesystem::u8path(source);
+ const auto max_frames = size_t(get_int(args, "max_frames"));
+ const auto fps_num = get_int(args, "fps_num"), fps_den = get_int(args, "fps_den");
+ auto result = std::filesystem::is_directory(path) ? ops::load_frame_directory(path, max_frames, fps_num, fps_den)
+ : ops::load_video(path, max_frames, fps_num, fps_den, get_bool(args, "audio"));
+ return Value::dict({{"video", video_result(std::move(result.frames), result.fps_num, result.fps_den)},
+ {"audio", audio_result(std::move(result.audio))}});
+ }
+
+ Value execute_save_video(Values& args, RunState& state) {
+ ops::VideoSaveOptions options;
+ options.metadata = state.metadata;
+ const auto video = read_video_dict(args.at("video"));
+ options.fps_num = video.fps_num;
+ options.fps_den = video.fps_den;
+ if (args.count("audio"))
+ options.audio = read_audio_dict(args.at("audio"), true);
+ options.trim_audio = get_bool(args, "trim_audio");
+ options.jpeg = get_string(args, "codec") != "rgb";
+ options.quality = int(get_int(args, "quality"));
+ options.pcm16 = get_string(args, "audio_format") == "pcm16";
+ ops::save_video(*video.frames, std::filesystem::u8path(get_string(args, "path")), options);
+ return Value::void_value();
+ }
+}
diff --git a/src/dgl/bindings/video.h b/src/dgl/bindings/video.h
new file mode 100644
index 0000000..b5dde7e
--- /dev/null
+++ b/src/dgl/bindings/video.h
@@ -0,0 +1,15 @@
+#ifndef __SD_DGL_BINDINGS_VIDEO_H__
+#define __SD_DGL_BINDINGS_VIDEO_H__
+
+#include "dgl/ops/video.h"
+#include "dgl/registry.h"
+
+namespace sd::dgl {
+ OutputSignature video_signature();
+ Value video_result(std::shared_ptr<sd_tensor_t> frames, int64_t fps_num, int64_t fps_den);
+ ops::VideoReadResult read_video_dict(const Value& value);
+ Value execute_load_video(Values& args, RunState&);
+ Value execute_save_video(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_VIDEO_H__
diff --git a/src/dgl/bindings/video_windows.cpp b/src/dgl/bindings/video_windows.cpp
new file mode 100644
index 0000000..feeefa6
--- /dev/null
+++ b/src/dgl/bindings/video_windows.cpp
@@ -0,0 +1,20 @@
+#include "dgl/bindings/video_windows.h"
+
+#include "dgl/bindings/conversion.h"
+#include "dgl/ops/video_windows.h"
+
+namespace sd::dgl {
+ Value split_video_frames(Values& args, RunState&) {
+ std::vector<Value> windows;
+ for (auto window : ops::split_video_frames(object<sd_tensor_t>(args, "frames"), size_t(get_int(args, "size")), size_t(get_int(args, "overlap"))))
+ windows.push_back(Value::object(std::move(window)));
+ return Value::list(std::move(windows), Kind::Tensor);
+ }
+
+ Value join_video_windows(Values& args, RunState&) {
+ std::vector<std::shared_ptr<sd_tensor_t>> windows;
+ for (const auto& window : args.at("windows").items)
+ windows.push_back(window.as<sd_tensor_t>());
+ return Value::object(ops::join_video_windows(windows, size_t(get_int(args, "overlap"))));
+ }
+}
diff --git a/src/dgl/bindings/video_windows.h b/src/dgl/bindings/video_windows.h
new file mode 100644
index 0000000..5ef2d02
--- /dev/null
+++ b/src/dgl/bindings/video_windows.h
@@ -0,0 +1,13 @@
+#ifndef __SD_DGL_BINDINGS_VIDEO_WINDOWS_H__
+#define __SD_DGL_BINDINGS_VIDEO_WINDOWS_H__
+
+#include "dgl/value.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ Value split_video_frames(Values& args, RunState&);
+ Value join_video_windows(Values& args, RunState&);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_BINDINGS_VIDEO_WINDOWS_H__
diff --git a/src/dgl/bytes.h b/src/dgl/bytes.h
new file mode 100644
index 0000000..2f48ed0
--- /dev/null
+++ b/src/dgl/bytes.h
@@ -0,0 +1,17 @@
+#ifndef __SD_DGL_BYTES_H__
+#define __SD_DGL_BYTES_H__
+
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+namespace sd::dgl {
+ struct Bytes {
+ const std::vector<uint8_t> data;
+
+ explicit Bytes(std::vector<uint8_t> value)
+ : data(std::move(value)) {}
+ };
+}
+
+#endif
diff --git a/src/dgl/context.cpp b/src/dgl/context.cpp
new file mode 100644
index 0000000..33da3c2
--- /dev/null
+++ b/src/dgl/context.cpp
@@ -0,0 +1,254 @@
+#include "context.h"
+
+#include <algorithm>
+#include <charconv>
+#include <cmath>
+#include <filesystem>
+#include <iterator>
+#include <limits>
+
+#include "core/ggml_graph_cut.h"
+#include "core/probe/backend_probe.h"
+#include "core/util.h"
+#include "dgl/model_value.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/lora.h"
+
+namespace sd::dgl {
+
+ Context::Context(bool keep_resources)
+ : keep_resources(keep_resources),
+ model_execution_{std::make_shared<sd::ParallelExecutor>(sd_get_num_physical_cores())},
+ conditioning_cache_size_(4) {
+ model_execution_.backend_probes = std::make_shared<BackendProbeCache>();
+ model_execution_.family_probes = std::make_shared<models::FamilyProbeCache>();
+ model_execution_.defer_initialization = true;
+ models->set_retain_weights(true);
+ models->set_idle_resource_eviction([this] { return evict_idle_resource(); });
+ }
+
+ void Context::prune_expired_resources() {
+ for (auto it = resources.begin(); it != resources.end();) {
+ if (it->second.value.expired())
+ it = resources.erase(it);
+ else
+ ++it;
+ }
+ }
+
+ bool Context::evict_idle_resource() {
+ auto oldest = resources.end();
+ for (auto it = resources.begin(); it != resources.end(); ++it) {
+ if (it->second.value.idle() &&
+ (oldest == resources.end() || it->second.last_use < oldest->second.last_use))
+ oldest = it;
+ }
+ if (oldest == resources.end())
+ return false;
+ LOG_VERBOSE("DGL unloaded idle %s", type_name(oldest->second.value.type()).c_str());
+ resources.erase(oldest);
+ return true;
+ }
+
+ static size_t parse_max_ram(const std::string& value) {
+ constexpr long double gib_bytes = 1024.0L * 1024 * 1024;
+ size_t end = 0;
+ long double gib = 0;
+ try {
+ gib = std::stold(value, &end);
+ } catch (const std::exception&) {
+ throw std::runtime_error("max_ram expects a nonnegative number of GiB");
+ }
+ const long double bytes = gib * gib_bytes;
+ if (value.find_first_not_of(" \t\r\n", end) != std::string::npos ||
+ !std::isfinite(gib) || gib < 0 || (gib > 0 && bytes < 1) ||
+ bytes >= std::ldexp(1.0L, std::numeric_limits<size_t>::digits))
+ throw std::runtime_error("max_ram expects a nonnegative number of GiB");
+ return static_cast<size_t>(bytes);
+ }
+
+ static int parse_context_int(const std::string& key, const std::string& value, int minimum, int maximum) {
+ const auto text = trim(value);
+ const auto* end = text.data() + text.size();
+ int number = 0;
+ const auto parsed = std::from_chars(text.data(), end, number);
+ if (parsed.ec != std::errc{} || parsed.ptr != end || number < minimum || number > maximum)
+ throw std::runtime_error(key + " expects an integer from " + std::to_string(minimum) + " to " + std::to_string(maximum));
+ return number;
+ }
+
+ void Context::set_model_execution_option(const std::string& key, const std::string& value) {
+ if (key == "n_threads") {
+ const auto count = parse_context_int(key, value, 1, 4096);
+ if (count == model_execution_.tensor_executor->num_threads())
+ return;
+ model_execution_.tensor_executor = std::make_shared<sd::ParallelExecutor>(count);
+ } else {
+ static constexpr std::pair<const char*, bool models::ModelExecutionOptions::*> flags[] = {
+ {"enable_mmap", &models::ModelExecutionOptions::enable_mmap},
+ {"eager_load", &models::ModelExecutionOptions::eager_load},
+ {"disable_prefetch", &models::ModelExecutionOptions::disable_prefetch},
+ {"disable_segmented_compute", &models::ModelExecutionOptions::disable_segmented_compute},
+ };
+ auto flag = std::find_if(std::begin(flags), std::end(flags), [&](const auto& item) { return key == item.first; });
+ if (flag == std::end(flags))
+ throw std::runtime_error("unknown DGL context option '" + key + "'");
+ bool enabled;
+ if (!parse_strict_bool(value, enabled))
+ throw std::runtime_error(key + " expects a boolean");
+ auto& current = model_execution_.*(flag->second);
+ if (current == enabled)
+ return;
+ current = enabled;
+ }
+ // Runner sessions and weight storage take their settings from the model at initialization.
+ for (auto it = resources.begin(); it != resources.end();) {
+ if (is_model_kind(it->second.value.type().kind))
+ it = resources.erase(it);
+ else
+ ++it;
+ }
+ }
+
+ void Context::set(const std::string& key, const std::string& value) {
+ if (key == "embed_metadata") {
+ bool enabled;
+ if (!parse_strict_bool(value, enabled))
+ throw std::runtime_error("embed_metadata expects a boolean");
+ embed_metadata = enabled;
+ } else if (key == "auto_fit") {
+ if (!parse_strict_bool(value, auto_fit_))
+ throw std::runtime_error("auto_fit expects a boolean");
+ } else if (key == "batched_cfg") {
+ bool enabled;
+ if (!parse_strict_bool(value, enabled))
+ throw std::runtime_error("batched_cfg expects a boolean");
+ batched_cfg = enabled;
+ } else if (key == "max_vram") {
+ ggml_graph_cut::MaxVramAssignment limit;
+ std::string error;
+ if (!limit.parse(value, &error) || !limit.canonicalize_backend_keys(&error))
+ throw std::runtime_error(error);
+ models->set_max_vram(std::move(limit));
+ } else if (key == "max_ram") {
+ if (!models->set_max_ram(parse_max_ram(value)))
+ throw std::runtime_error(models->host_memory_error());
+ } else if (key == "conditioning_cache_size") {
+ conditioning_cache_size_ = static_cast<size_t>(parse_context_int(key, value, 0, 4096));
+ for (const auto& [_, resource] : resources) {
+ if (auto cached = resource.value.lock())
+ set_conditioning_cache_size(*cached, conditioning_cache_size_);
+ }
+ } else {
+ set_model_execution_option(key, value);
+ }
+ }
+
+ std::string file_key(const std::string& path, nlohmann::json& revision) {
+ if (path.empty())
+ return path;
+ require_file_access(path, SD_DGL_FILE_READ_MODEL);
+ std::error_code error;
+ auto file = std::filesystem::weakly_canonical(std::filesystem::u8path(path), error);
+ if (error)
+ return path;
+ const auto canonical = file.u8string();
+ const auto timestamp = std::filesystem::last_write_time(file, error);
+ if (!error) {
+ const auto bytes = std::filesystem::file_size(file, error);
+ revision[canonical] = {timestamp.time_since_epoch().count(), error ? 0 : bytes};
+ }
+ return canonical;
+ }
+
+ std::optional<Value> Context::find_resource(const ResourceKey& identity, const std::string& revision, const std::string& lora_revision, uint64_t last_use) {
+ auto existing = resources.find(identity);
+ if (existing == resources.end())
+ return std::nullopt;
+ bool changed = false;
+ if (existing->second.revision == revision && existing->second.lora_revision == lora_revision &&
+ ModelLoader::dependencies_changed(existing->second.dependencies, changed) && !changed) {
+ if (auto value = existing->second.value.lock()) {
+ existing->second.last_use = last_use;
+ return value;
+ }
+ }
+ resources.erase(existing);
+ return std::nullopt;
+ }
+
+ Value Context::cache_resource(ResourceKey identity, std::string revision, std::string lora_revision, ModelLoader::Dependencies dependencies, Value value, uint64_t last_use) {
+ bool changed = false;
+ if (!ModelLoader::dependencies_changed(dependencies, changed) || changed)
+ throw std::runtime_error("model sources changed while loading");
+ set_conditioning_cache_size(value, conditioning_cache_size_);
+ resources.insert_or_assign(std::move(identity), Resource{std::move(revision), std::move(lora_revision),
+ Value::Cached(value, keep_resources), std::move(dependencies), last_use});
+ return value;
+ }
+
+ Value Context::load(const Registry::Entry& registration, Values& args, bool prepare_only) {
+ const auto& [name, operation] = registration;
+ const auto kind = operation.output.type.kind;
+ nlohmann::json key = {{"operation", name}};
+ auto prepared = operation.prepare(args);
+ key["params"] = std::move(prepared.key);
+ auto revision = std::move(prepared.revision);
+ ResourceKey identity{key.dump(), {}};
+ auto argument_revision = revision.dump();
+ const auto last_use = ++resource_epoch;
+ if (auto value = find_resource(identity, argument_revision, {}, last_use)) {
+ if (!prepare_only)
+ finish_resource_loading(*value);
+ return std::move(*value);
+ }
+ const bool model = is_model_kind(kind);
+ ResourceLoadRequest request{*this, model ? identity.first : ""};
+ auto value = prepared.load(request);
+ auto dependencies = model ? model_from_value(value).dependencies() : ModelLoader::Dependencies{};
+ for (const auto& dependency : dependencies)
+ revision[dependency.path] = {dependency.exists, dependency.modified.time_since_epoch().count(), dependency.size};
+ if (model)
+ value.model_source = std::make_shared<ModelSource>(ModelSource{identity.first, revision.dump()});
+ if (!prepare_only)
+ finish_resource_loading(value);
+ return cache_resource(std::move(identity), std::move(argument_revision), {}, std::move(dependencies), std::move(value), last_use);
+ }
+
+ void Context::finish_resource_loading(const Value& value) const {
+ if (model_execution_.eager_load && is_model_kind(value.type().kind) && !model_from_value(value).finish_initialization())
+ throw std::runtime_error("cannot initialize model weights");
+ }
+
+ Value Context::derive_model(const Value& base, std::shared_ptr<const models::LoraConfig> loras) {
+ if (!base.model_source)
+ throw std::runtime_error("model has no weight source");
+ const auto& source = *base.model_source;
+ nlohmann::json files = nlohmann::json::object();
+ nlohmann::json bindings = nlohmann::json::array();
+ for (const auto& binding : loras->bindings)
+ bindings.push_back({file_key(binding.file->path, files), binding.weight});
+ auto lora_revision = files.dump();
+ ResourceKey identity{source.key, nlohmann::json::array({int(loras->mode), std::move(bindings)}).dump()};
+
+ auto inherit_placement = [&](Value value) {
+ const auto memory = model_from_value(base).model_weights->memory();
+ const bool disk = memory.mode == ModelManager::ResidencyMode::Disk;
+ if (!memory.split && memory.params_backend &&
+ !model_from_value(value).place_params(disk ? "disk" : ggml_backend_name(memory.params_backend), memory.params_backend, disk))
+ throw std::runtime_error("cannot place LoRA model weights");
+ if (model_execution_.eager_load && !model_from_value(value).finish_initialization())
+ throw std::runtime_error("cannot initialize LoRA model weights");
+ return bind_model_variant(base, std::move(value));
+ };
+ const auto last_use = ++resource_epoch;
+ if (auto value = find_resource(identity, source.revision, lora_revision, last_use))
+ return inherit_placement(std::move(*value));
+ auto value = build_model_variant(*this, base, loras);
+ value.model_source = base.model_source;
+ value.loras = std::move(loras);
+ auto dependencies = model_from_value(value).dependencies();
+ return inherit_placement(cache_resource(std::move(identity), source.revision, std::move(lora_revision), std::move(dependencies), std::move(value), last_use));
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/context.h b/src/dgl/context.h
new file mode 100644
index 0000000..30ea1ee
--- /dev/null
+++ b/src/dgl/context.h
@@ -0,0 +1,76 @@
+#ifndef __SD_DGL_CONTEXT_H__
+#define __SD_DGL_CONTEXT_H__
+
+#include <mutex>
+
+#include "core/file_access.h"
+#include "core/ggml_extend_backend.h"
+#include "dgl/models/model_base.h"
+#include "model_manager.h"
+#include "registry.h"
+#include "run_events.h"
+
+namespace sd::dgl {
+ struct Context;
+
+ struct ResourceLoadRequest {
+ Context& context;
+ std::string weights_key;
+ };
+
+ std::string file_key(const std::string& path, nlohmann::json& revision);
+
+ struct Context {
+ using ResourceKey = std::pair<std::string, std::string>; // Component and LoRA configuration.
+
+ struct Resource {
+ std::string revision;
+ std::string lora_revision;
+ Value::Cached value;
+ ModelLoader::Dependencies dependencies;
+ uint64_t last_use = 0;
+ };
+
+ std::mutex mutex;
+ RunEvents* active_run = nullptr;
+ RunCallbacks callbacks;
+ FileAccessPolicy file_access;
+ std::shared_ptr<SDBackendPool> backends = std::make_shared<SDBackendPool>();
+ std::shared_ptr<ModelManager> models = std::make_shared<ModelManager>();
+ std::map<ResourceKey, Resource> resources;
+ uint64_t resource_epoch = 0;
+ const bool keep_resources;
+ bool batched_cfg = true;
+ bool embed_metadata = true;
+
+ explicit Context(bool keep_resources = true);
+ void set(const std::string& key, const std::string& value);
+ Value load(const Registry::Entry& registration, Values& args, bool prepare_only = false);
+ void finish_resource_loading(const Value& value) const;
+ Value derive_model(const Value& base, std::shared_ptr<const models::LoraConfig> loras);
+ bool evict_idle_resource();
+ void prune_expired_resources();
+
+ sd::ParallelExecutor* tensor_executor() const { return model_execution_.tensor_executor.get(); }
+ bool auto_fit() const { return auto_fit_; }
+
+ template <class T>
+ std::shared_ptr<T> make_model(const std::string& weights_key) const {
+ auto model = std::make_shared<T>(models, backends);
+ model->set_shared_weights(weights_key);
+ model->configure_execution(model_execution_);
+ return model;
+ }
+
+ private:
+ std::optional<Value> find_resource(const ResourceKey& identity, const std::string& revision, const std::string& lora_revision, uint64_t last_use);
+ Value cache_resource(ResourceKey identity, std::string revision, std::string lora_revision, ModelLoader::Dependencies dependencies, Value value, uint64_t last_use);
+ bool auto_fit_ = true;
+ models::ModelExecutionOptions model_execution_;
+ size_t conditioning_cache_size_;
+ void set_model_execution_option(const std::string& key, const std::string& value);
+ };
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_CONTEXT_H__
diff --git a/src/dgl/dgl.cpp b/src/dgl/dgl.cpp
new file mode 100644
index 0000000..ee8aef7
--- /dev/null
+++ b/src/dgl/dgl.cpp
@@ -0,0 +1,211 @@
+#include "stable-diffusion.h"
+
+#include <algorithm>
+#include <mutex>
+
+#include "api/api.h"
+#include "context.h"
+#include "diagnostic.h"
+#include "executor.h"
+#include "graph.h"
+#include "result.h"
+
+struct sd_dgl_ctx_t : sd::dgl::Context {
+ using Context::Context;
+};
+
+namespace {
+ using namespace sd::dgl;
+
+ template <class F>
+ sd_status_t api_call(F&& operation, sd_status_t failure = SD_STATUS_INVALID_ARGUMENT) {
+ return sd::api::call([&] {
+ try {
+ operation();
+ sd::api::clear_error();
+ return SD_STATUS_OK;
+ } catch (const Error& error) {
+ return sd::api::error(error.status, nullptr, error.what());
+ }
+ },
+ failure);
+ }
+
+ std::unique_lock<std::mutex> lock_idle(Context* ctx) {
+ if (!ctx)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "context is required");
+ std::unique_lock<std::mutex> lock(ctx->mutex, std::try_to_lock);
+ if (!lock.owns_lock() || ctx->active_run)
+ throw Error(SD_STATUS_BUSY, "DGL context is busy");
+ return lock;
+ }
+
+ class ActiveRun {
+ Context& context_;
+
+ public:
+ RunEvents events;
+
+ explicit ActiveRun(Context& context)
+ : context_(context), events(context.callbacks, false) {
+ context_.active_run = &events;
+ }
+
+ ~ActiveRun() {
+ std::lock_guard<std::mutex> lock(context_.mutex);
+ if (context_.active_run == &events)
+ context_.active_run = nullptr;
+ }
+
+ void complete(std::unique_ptr<sd_dgl_result_t>& result, sd_dgl_result_t** out_result) {
+ std::lock_guard<std::mutex> lock(context_.mutex);
+ events.check_cancelled();
+ *out_result = result.release();
+ context_.active_run = nullptr;
+ }
+ };
+
+ Values bind_inputs(const Graph& graph, const sd_dgl_tensor_input_t* inputs, size_t input_count) {
+ if (!inputs && input_count)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "tensor input array is required");
+ const auto& names = graph.syntax.entry.parameters;
+ Values bound;
+ for (size_t i = 0; i < input_count; ++i) {
+ const auto& input = inputs[i];
+ if (!input.name || !*input.name || !input.tensor)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "each tensor input requires a name and tensor");
+ if (std::find(names.begin(), names.end(), input.name) == names.end())
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "unknown tensor input '" + std::string(input.name) + "'");
+ if (bound.count(input.name))
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "duplicate tensor input '" + std::string(input.name) + "'");
+ auto retained = std::shared_ptr<sd_tensor_t>(sd_tensor_retain(input.tensor), sd_tensor_free);
+ bound.emplace(input.name, Value::object(std::move(retained)));
+ }
+ for (const auto& name : names)
+ if (!bound.count(name))
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "missing tensor input '" + name + "'");
+ return bound;
+ }
+
+ std::unique_ptr<sd_dgl_result_t> execute(Context& ctx, RunEvents& events, const Graph& graph, Values inputs) {
+ sd::FileAccessScope file_access(ctx.file_access);
+ RunEventsScope event_scope(events);
+ RunState state{ctx, events};
+ ctx.models->clear_host_memory_error();
+ try {
+ events.report("executing");
+ return std::make_unique<sd_dgl_result_t>(run(graph, state, std::move(inputs)));
+ } catch (const std::exception& error) {
+ events.check_cancelled();
+ const auto& memory_error = ctx.models->host_memory_error();
+ if (!memory_error.empty())
+ throw Error(SD_STATUS_OUT_OF_MEMORY, std::string(error.what()) + ". " + memory_error);
+ if (dynamic_cast<const Error*>(&error) || dynamic_cast<const std::bad_alloc*>(&error))
+ throw;
+ throw Error(SD_STATUS_GENERATION_FAILED, error.what());
+ }
+ }
+}
+
+const char* sd_dgl_builtin_catalog(void) {
+ const char* result = nullptr;
+ api_call([&] {
+ static const std::string catalog = builtin_catalog();
+ result = catalog.c_str();
+ });
+ return result;
+}
+
+const char* sd_dgl_error(void) {
+ return sd_get_last_error();
+}
+
+sd_dgl_ctx_t* sd_dgl_ctx_create(void) {
+ sd_dgl_ctx_t* result = nullptr;
+ api_call([&] { result = new sd_dgl_ctx_t(); });
+ return result;
+}
+
+sd_dgl_ctx_t* sd_dgl_ctx_create_once(void) {
+ sd_dgl_ctx_t* result = nullptr;
+ api_call([&] { result = new sd_dgl_ctx_t(false); });
+ return result;
+}
+
+void sd_dgl_ctx_free(sd_dgl_ctx_t* ctx) {
+ delete ctx;
+}
+
+bool sd_dgl_ctx_set(sd_dgl_ctx_t* ctx, const char* key, const char* value) {
+ return api_call([&] {
+ if (!key || !value)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "key and value are required");
+ auto lock = lock_idle(ctx);
+ ctx->set(key, value);
+ }) == SD_STATUS_OK;
+}
+
+bool sd_dgl_ctx_set_file_access_callback(sd_dgl_ctx_t* ctx, sd_dgl_file_access_cb_t callback, void* userdata) {
+ return api_call([&] {
+ auto lock = lock_idle(ctx);
+ ctx->file_access = {callback, userdata};
+ }) == SD_STATUS_OK;
+}
+
+sd_status_t sd_dgl_ctx_set_progress_callback(sd_dgl_ctx_t* ctx, sd_generation_progress_cb_t callback, void* userdata) {
+ return api_call([&] {
+ auto lock = lock_idle(ctx);
+ ctx->callbacks.progress = callback;
+ ctx->callbacks.progress_data = userdata;
+ });
+}
+
+sd_status_t sd_dgl_ctx_set_preview_callback(sd_dgl_ctx_t* ctx, sd_dgl_preview_cb_t callback, const char* mode, int interval, bool denoised, bool noisy, void* userdata) {
+ return api_call([&] {
+ auto lock = lock_idle(ctx);
+ const auto preview = callback && mode ? str_to_preview(mode) : PREVIEW_NONE;
+ if (callback && (!mode || preview == PREVIEW_COUNT))
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "preview mode must be none, proj, tae or vae");
+ ctx->callbacks.preview = callback;
+ ctx->callbacks.preview_data = userdata;
+ ctx->callbacks.mode = preview;
+ ctx->callbacks.interval = interval;
+ ctx->callbacks.denoised = denoised;
+ ctx->callbacks.noisy = noisy;
+ });
+}
+
+sd_status_t sd_dgl_ctx_cancel(sd_dgl_ctx_t* ctx) {
+ return api_call([&] {
+ if (!ctx)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "context is required");
+ std::lock_guard<std::mutex> lock(ctx->mutex);
+ if (ctx->active_run)
+ ctx->active_run->cancel();
+ });
+}
+
+sd_status_t sd_dgl_check(const char* source, bool allow_tensor_inputs) {
+ return api_call([&] {
+ if (!source)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "source is required");
+ compile(source, operations(), allow_tensor_inputs);
+ });
+}
+
+sd_status_t sd_dgl_run(sd_dgl_ctx_t* ctx, const char* source, const sd_dgl_tensor_input_t* inputs, size_t input_count, sd_dgl_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ return api_call([&] {
+ if (!source || !result_out)
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "source and result_out are required");
+ auto lock = lock_idle(ctx);
+ ActiveRun active(*ctx);
+ lock.unlock();
+ auto graph = compile(source, operations(), true);
+ active.events.check_cancelled();
+ auto bound = bind_inputs(graph, inputs, input_count);
+ auto result = execute(*ctx, active.events, graph, std::move(bound));
+ active.complete(result, result_out);
+ });
+}
diff --git a/src/dgl/diagnostic.h b/src/dgl/diagnostic.h
new file mode 100644
index 0000000..ffe931d
--- /dev/null
+++ b/src/dgl/diagnostic.h
@@ -0,0 +1,17 @@
+#ifndef __SD_DGL_DIAGNOSTIC_H__
+#define __SD_DGL_DIAGNOSTIC_H__
+
+#include <stdexcept>
+
+#include "stable-diffusion.h"
+
+namespace sd::dgl {
+ struct Error : std::runtime_error {
+ sd_status_t status;
+
+ Error(sd_status_t status, const std::string& message)
+ : std::runtime_error(message), status(status) {}
+ };
+} // namespace sd::dgl
+
+#endif // __SD_DGL_DIAGNOSTIC_H__
diff --git a/src/dgl/executor.cpp b/src/dgl/executor.cpp
new file mode 100644
index 0000000..920f776
--- /dev/null
+++ b/src/dgl/executor.cpp
@@ -0,0 +1,300 @@
+#include "executor.h"
+#include "context.h"
+
+#include <stdexcept>
+
+#include "core/util.h"
+#include "diagnostic.h"
+#include "graph.h"
+#include "indexing.h"
+#include "run_plan.h"
+
+namespace sd::dgl {
+
+ class Executor {
+ struct Frame {
+ Values values;
+ ReferenceCounts remaining;
+ std::vector<const Expr*> call_path;
+
+ bool uses(const std::string& name) const {
+ if (remaining.count(name))
+ return true;
+ auto prefix = name + ".";
+ auto field = remaining.lower_bound(prefix);
+ return field != remaining.end() && field->first.compare(0, prefix.size(), prefix) == 0;
+ }
+
+ void drop_unused_fields(const std::string& name, Value& value) const {
+ if (value.fields.empty() || remaining.count(name))
+ return;
+ for (auto field = value.fields.begin(); field != value.fields.end();) {
+ auto path = name + "." + field->first;
+ if (uses(path)) {
+ drop_unused_fields(path, field->second);
+ ++field;
+ } else {
+ field = value.fields.erase(field);
+ }
+ }
+ }
+
+ Value take(const std::string& reference, const std::string& root) {
+ if (--remaining.at(reference) == 0)
+ remaining.erase(reference);
+ auto& value = values.at(root);
+ if (!uses(root)) {
+ auto result = std::move(value);
+ values.erase(root);
+ return result;
+ }
+ auto result = value;
+ drop_unused_fields(root, value);
+ return result;
+ }
+
+ void discard_captures(const ReferenceCounts& captures) {
+ for (const auto& [reference, count] : captures) {
+ if ((remaining.at(reference) -= count) == 0)
+ remaining.erase(reference);
+ auto root = reference.substr(0, reference.find('.'));
+ auto value = values.find(root);
+ if (value != values.end()) {
+ if (uses(root))
+ drop_unused_fields(root, value->second);
+ else
+ values.erase(value);
+ }
+ }
+ }
+ };
+
+ const Graph& graph_;
+ RunState& state_;
+ size_t visits_ = 0;
+
+ public:
+ Executor(const Graph& graph, RunState& state)
+ : graph_(graph), state_(state) {}
+
+ Value expression(const std::shared_ptr<Expr>& expr, Frame& frame) {
+ state_.events.check_cancelled();
+ if (++visits_ > 100000)
+ throw std::runtime_error("DGL execution exceeded the expression limit");
+ auto source_message = [&](const char* message) {
+ return "line " + std::to_string(expr->line) + ", column " + std::to_string(expr->column) + ": " + message;
+ };
+ auto failure = SD_STATUS_GENERATION_FAILED;
+ try {
+ if (expr->form == Expr::Form::Literal)
+ return Value::literal(expr->literal);
+ if (expr->form == Expr::Form::Block) {
+ const auto& block = graph_.blocks.at(expr.get());
+ Values captures;
+ for (const auto& [reference, count] : block.captures) {
+ auto root = reference.substr(0, reference.find('.'));
+ if (!captures.count(root))
+ captures.emplace(root, frame.values.at(root));
+ }
+ frame.discard_captures(block.captures);
+ auto result = body(expr->body, {std::move(captures), block.references, frame.call_path}, true);
+ if (state_.plan)
+ state_.plan->enter(expr.get(), frame.call_path, true);
+ return result;
+ }
+ if (expr->form == Expr::Form::Conditional) {
+ const auto condition = expression(expr->arguments[0].second, frame);
+ const bool selected = condition.scalar.get<bool>();
+ if (expr->arguments.size() == 2) {
+ if (selected)
+ expression(expr->arguments[1].second, frame);
+ else
+ frame.discard_captures(graph_.blocks.at(expr->arguments[1].second.get()).captures);
+ return Value::void_value();
+ }
+ frame.discard_captures(graph_.blocks.at(expr->arguments[selected ? 2 : 1].second.get()).captures);
+ return expression(expr->arguments[selected ? 1 : 2].second, frame);
+ }
+ if (expr->form == Expr::Form::Reference) {
+ auto dot = expr->name.find('.');
+ auto value = frame.take(expr->name, expr->name.substr(0, dot));
+ while (dot != std::string::npos) {
+ auto next = expr->name.find('.', dot + 1);
+ auto field = expr->name.substr(dot + 1, next == std::string::npos ? next : next - dot - 1);
+ auto found = value.fields.find(field);
+ if (found == value.fields.end())
+ throw Error(SD_STATUS_INVALID_ARGUMENT, "field '" + expr->name + "' is absent from the runtime value");
+ auto child = std::move(found->second);
+ value = std::move(child);
+ dot = next;
+ }
+ return value;
+ }
+ if (expr->form == Expr::Form::List) {
+ std::vector<Value> items;
+ for (const auto& arg : expr->arguments)
+ items.push_back(expression(arg.second, frame));
+ return Value::list(std::move(items));
+ }
+ if (expr->form == Expr::Form::Dict) {
+ Values fields;
+ for (const auto& field : expr->arguments)
+ fields[field.first] = expression(field.second, frame);
+ return Value::dict(std::move(fields));
+ }
+ if (expr->form == Expr::Form::Index) {
+ auto input = expression(expr->arguments[0].second, frame);
+ auto indices = expression(expr->arguments[1].second, frame);
+ auto result = index_value(std::move(input), indices);
+ state_.events.check_cancelled();
+ return result;
+ }
+ const auto& bound = graph_.calls.at(expr.get());
+ if (bound.iteration) {
+ const auto& call = *bound.iteration;
+ auto list = expression(call.items, frame);
+ const auto& parameters = call.function->parameters;
+ const size_t item_index = call.initial ? 1 : 0;
+ const auto& item_param = parameters[item_index];
+ Values locals;
+ for (const auto& arg : call.arguments)
+ if (arg.parameter != item_index)
+ locals.emplace(parameters[arg.parameter], expression(arg.expression, frame));
+ auto call_path = frame.call_path;
+ call_path.push_back(expr.get());
+ if (list.items.size() > 1)
+ LOG_INFO("%s %s: %zu items", expr->name.c_str(), call.process.c_str(), list.items.size());
+ auto run_item = [&](size_t i) {
+ LOG_VERBOSE("%s %s: item %zu/%zu", expr->name.c_str(), call.process.c_str(), i + 1, list.items.size());
+ locals[item_param] = std::move(list.items[i]);
+ return body(call.process, *call.function,
+ i + 1 == list.items.size() ? std::move(locals) : locals, call_path);
+ };
+ if (call.initial) {
+ const auto& accumulator = parameters[0];
+ for (size_t i = 0; i < list.items.size(); ++i)
+ locals[accumulator] = run_item(i);
+ return std::move(locals.at(accumulator));
+ }
+ std::vector<Value> results;
+ results.reserve(list.items.size());
+ for (size_t i = 0; i < list.items.size(); ++i)
+ results.push_back(run_item(i));
+ const auto output = std::find_if(graph_.map_outputs.begin(), graph_.map_outputs.end(), [&](const auto& site) {
+ return site.expression == expr.get() && site.call_path == frame.call_path;
+ });
+ if (output == graph_.map_outputs.end())
+ throw std::runtime_error("missing map output type for this call");
+ return Value::list(std::move(results), output->element);
+ }
+ if (bound.function) {
+ const auto& parameters = bound.function->parameters;
+ Values locals;
+ for (const auto& arg : bound.arguments)
+ locals.emplace(parameters[arg.parameter], expression(arg.expression, frame));
+ auto call_path = frame.call_path;
+ call_path.push_back(expr.get());
+ return body(expr->name, *bound.function, std::move(locals), std::move(call_path));
+ }
+ const auto& operation = bound.operation->second;
+ Values args;
+ auto set_argument = [&](const Argument& argument, Value value) {
+ if (!accepts(argument.type, value.type()))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "invalid type for '" + argument.name + "'");
+ if (argument.validate)
+ argument.validate(value);
+ args.emplace(argument.name, std::move(value));
+ };
+ for (const auto& arg : bound.arguments) {
+ const auto& argument = operation.arguments[arg.parameter];
+ if (argument.callback) {
+ const auto* fn = graph_.callbacks.at(arg.expression.get());
+ auto callback_path = frame.call_path;
+ callback_path.push_back(expr.get());
+ auto callback = std::make_shared<ScriptFunction>([this, fn, name = arg.expression->name, path = std::move(callback_path)](Value input, const Value& captures) {
+ return Executor(graph_, state_).body(name, *fn, {{fn->parameters[0], std::move(input)}, {fn->parameters[1], captures}}, path);
+ });
+ set_argument(argument, Value::object(std::move(callback)));
+ } else {
+ set_argument(argument, expression(arg.expression, frame));
+ }
+ }
+ for (const auto& argument : operation.arguments)
+ if (argument.default_value && !args.count(argument.name))
+ set_argument(argument, *argument.default_value);
+ state_.context.models->clear_host_memory_error();
+ if (operation.resource())
+ failure = SD_STATUS_LOAD_FAILED;
+ state_.events.check_cancelled();
+ Value value = operation.resource() ? (state_.plan ? state_.plan->load_resource(*bound.operation, args) : state_.context.load(*bound.operation, args, true))
+ : operation.execute(args, state_);
+ if (operation.resource()) {
+ if (state_.plan)
+ state_.plan->resource_loaded(expr.get(), value);
+ state_.context.finish_resource_loading(value);
+ }
+ state_.events.check_cancelled();
+ auto output_type = operation.output.same_type_as.empty()
+ ? operation.output.type
+ : args.at(operation.output.same_type_as).type();
+ if (!accepts(output_type, value.type()))
+ throw std::runtime_error("built-in function '" + expr->name + "' returned the wrong type");
+ return value;
+ } catch (const std::bad_alloc&) {
+ throw;
+ } catch (const Error& error) {
+ if (error.status == SD_STATUS_CANCELLED)
+ throw;
+ throw Error(error.status, source_message(error.what()));
+ } catch (const std::exception& error) {
+ throw Error(failure, source_message(error.what()));
+ }
+ }
+
+ Value body(const std::vector<Statement>& statements, Frame frame, bool entry = false) {
+ for (auto value = frame.values.begin(); value != frame.values.end();) {
+ if (frame.uses(value->first)) {
+ frame.drop_unused_fields(value->first, value->second);
+ ++value;
+ } else {
+ value = frame.values.erase(value);
+ }
+ }
+ for (const auto& statement : statements) {
+ if (!statement.value)
+ return Value::void_value();
+ if (entry && state_.plan)
+ state_.plan->enter(statement.value.get(), frame.call_path);
+ auto value = expression(statement.value, frame);
+ if (statement.form == Statement::Form::Return || statement.form == Statement::Form::Result)
+ return value;
+ if (statement.form == Statement::Form::Binding && frame.uses(statement.name)) {
+ frame.drop_unused_fields(statement.name, value);
+ frame.values.emplace(statement.name, std::move(value));
+ }
+ }
+ return Value::void_value();
+ }
+
+ Value body(const std::string& name, const Function& function, Values scope, std::vector<const Expr*> call_path = {}) {
+ return body(function.body, {std::move(scope), graph_.references.at(name), std::move(call_path)}, name == "main");
+ }
+ };
+
+ Value run(const Graph& graph, RunState& state, Values inputs) {
+ state.metadata = state.context.embed_metadata ? graph.source : "";
+ state.context.prune_expired_resources();
+ sd::ParallelScope tensor_scope(state.context.tensor_executor());
+ RunPlan plan(graph, state);
+ state.plan = &plan;
+ try {
+ auto result = Executor(graph, state).body("main", graph.syntax.entry, std::move(inputs));
+ state.plan = nullptr;
+ return result;
+ } catch (...) {
+ state.plan = nullptr;
+ throw;
+ }
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/executor.h b/src/dgl/executor.h
new file mode 100644
index 0000000..1922090
--- /dev/null
+++ b/src/dgl/executor.h
@@ -0,0 +1,23 @@
+#ifndef __SD_DGL_EXECUTOR_H__
+#define __SD_DGL_EXECUTOR_H__
+
+#include "registry.h"
+
+namespace sd::dgl {
+
+ struct Graph;
+ struct Context;
+ class RunEvents;
+ class RunPlan;
+ struct RunState {
+ Context& context;
+ RunEvents& events;
+ RunPlan* plan = nullptr;
+ std::string metadata;
+ };
+
+ Value run(const Graph& graph, RunState& state, Values inputs);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_EXECUTOR_H__
diff --git a/src/dgl/graph.cpp b/src/dgl/graph.cpp
new file mode 100644
index 0000000..b607163
--- /dev/null
+++ b/src/dgl/graph.cpp
@@ -0,0 +1,736 @@
+#include "graph.h"
+#include "indexing.h"
+
+#include <algorithm>
+#include <set>
+#include <stdexcept>
+
+#include "diagnostic.h"
+
+namespace sd::dgl {
+
+ static bool is_literal_argument(const Expr& expr) {
+ if (expr.form == Expr::Form::Literal)
+ return true;
+ if (expr.form != Expr::Form::List && expr.form != Expr::Form::Dict)
+ return false;
+ return std::all_of(expr.arguments.begin(), expr.arguments.end(), [](const auto& entry) { return is_literal_argument(*entry.second); });
+ }
+
+ std::vector<BoundArgument> bind_arguments(const Expr& expr, const std::vector<std::string>& names, bool all_required) {
+ std::vector<BoundArgument> args;
+ args.reserve(expr.arguments.size());
+ std::vector<bool> bound(names.size());
+ size_t position = 0;
+ for (const auto& [name, expression] : expr.arguments) {
+ size_t parameter;
+ if (name.empty()) {
+ if (position >= names.size())
+ throw std::runtime_error("too many arguments for '" + expr.name + "'");
+ parameter = position++;
+ } else {
+ const auto found = std::find(names.begin(), names.end(), name);
+ if (found == names.end())
+ throw std::runtime_error("unknown argument '" + name + "' for '" + expr.name + "'");
+ parameter = static_cast<size_t>(found - names.begin());
+ }
+ if (bound[parameter])
+ throw std::runtime_error("duplicate argument '" + names[parameter] + "'");
+ bound[parameter] = true;
+ args.push_back({parameter, expression});
+ }
+ if (all_required) {
+ const auto missing = std::find(bound.begin(), bound.end(), false);
+ if (missing != bound.end())
+ throw std::runtime_error("missing function argument '" + names[missing - bound.begin()] + "'");
+ }
+ return args;
+ }
+
+ std::vector<std::string> argument_names(const Operation& operation) {
+ std::vector<std::string> names;
+ for (const auto& arg : operation.arguments)
+ names.push_back(arg.name);
+ return names;
+ }
+
+ IterationCall bind_iteration(const Expr& expr, const Syntax& syntax) {
+ bool fold = expr.name == "fold";
+ size_t process_index = fold ? 2 : 1;
+ if (expr.arguments.size() <= process_index)
+ throw std::runtime_error(fold ? "fold requires items, initial and process" : "map requires items and process");
+ auto named = [&](size_t i, const char* name) { return expr.arguments[i].first.empty() || expr.arguments[i].first == name; };
+ if (!named(0, "items") || (fold && !named(1, "initial")) || !named(process_index, "process"))
+ throw std::runtime_error(fold ? "fold starts with items, initial and process" : "map starts with items and process");
+ const auto& process = expr.arguments[process_index].second;
+ auto found = syntax.functions.find(process->name);
+ if (process->form != Expr::Form::Reference || found == syntax.functions.end())
+ throw std::runtime_error(expr.name + " process must name a function");
+ const auto& function = found->second;
+ if (function.parameters.size() < (fold ? 2 : 1))
+ throw std::runtime_error(fold ? "fold process requires accumulator and item parameters" : "map process requires an item parameter");
+ Expr call = expr;
+ call.arguments.erase(call.arguments.begin() + process_index);
+ call.arguments[0].first.clear();
+ if (fold) {
+ call.arguments[1].first.clear();
+ std::swap(call.arguments[0], call.arguments[1]);
+ }
+ auto bound = bind_arguments(call, function.parameters, true);
+ return {&function, process->name, expr.arguments[0].second, fold ? expr.arguments[1].second : nullptr, std::move(bound)};
+ }
+
+ struct Checked {
+ Type type;
+ std::shared_ptr<Expr> literal{};
+ std::map<std::string, Checked> fields{};
+ std::optional<size_t> tensor_rank;
+ Kind tensor_scalar = Kind::Any;
+ std::optional<size_t> length;
+ std::vector<Checked> items;
+ bool complete_fields = false;
+ };
+
+ using Types = std::map<std::string, Checked>;
+
+ static Checked checked_output(const OutputSignature& signature) {
+ Checked result{signature.type};
+ for (const auto& [name, field] : signature.fields)
+ result.fields.emplace(name, checked_output(field));
+ return result;
+ }
+
+ static bool same_shape(const Checked& a, const Checked& b) {
+ if (!(a.type == b.type) || a.fields.size() != b.fields.size())
+ return false;
+ for (const auto& field : a.fields) {
+ auto other = b.fields.find(field.first);
+ if (other == b.fields.end() || !same_shape(field.second, other->second))
+ return false;
+ }
+ return true;
+ }
+
+ static Checked shape_only(const Checked& value) {
+ Checked result{value.type};
+ result.tensor_rank = value.tensor_rank;
+ result.tensor_scalar = value.tensor_scalar;
+ for (const auto& field : value.fields)
+ result.fields.emplace(field.first, shape_only(field.second));
+ return result;
+ }
+
+ static void clear_tensor_facts(Checked& value) {
+ value.tensor_rank.reset();
+ value.tensor_scalar = Kind::Any;
+ for (auto& field : value.fields)
+ clear_tensor_facts(field.second);
+ }
+
+ static void keep_common_fields(Checked& value, const Checked& other) {
+ if (value.tensor_rank != other.tensor_rank)
+ value.tensor_rank.reset();
+ if (value.tensor_scalar != other.tensor_scalar)
+ value.tensor_scalar = Kind::Any;
+ for (auto field = value.fields.begin(); field != value.fields.end();) {
+ const auto match = other.fields.find(field->first);
+ if (match == other.fields.end() || !(field->second.type == match->second.type))
+ field = value.fields.erase(field);
+ else {
+ keep_common_fields(field->second, match->second);
+ ++field;
+ }
+ }
+ }
+
+ static void summarize_list(Checked& list) {
+ if (list.items.empty())
+ return;
+ auto item = shape_only(list.items.front());
+ for (size_t i = 1; i < list.items.size(); ++i)
+ keep_common_fields(item, list.items[i]);
+ list.fields = std::move(item.fields);
+ list.tensor_rank = item.tensor_rank;
+ list.tensor_scalar = item.tensor_scalar;
+ }
+
+ static void infer_tensor_result(const std::string& name, const Types& args, Checked& result) {
+ const auto find = [&](const char* key) -> const Checked* {
+ const auto found = args.find(key);
+ return found == args.end() ? nullptr : &found->second;
+ };
+ const auto input = find("input");
+ const auto dtype = find("dtype");
+ const auto scalar = [&] {
+ if (!dtype)
+ return Kind::Number;
+ if (!dtype->literal)
+ return Kind::Any;
+ const auto& text = dtype->literal->literal;
+ return text == "u8" || text == "i32" || text == "i64" ? Kind::Integer : Kind::Number;
+ };
+ if (name == "tensor::shape" || name == "latent_shape" || name == "audio_latent_shape" || name == "tensor::arange") {
+ result.tensor_rank = 1;
+ result.tensor_scalar = Kind::Integer;
+ if (name == "tensor::shape" && input)
+ result.length = input->tensor_rank;
+ } else if (name == "tensor::vector") {
+ result.tensor_rank = 1;
+ result.tensor_scalar = scalar();
+ if (const auto values = find("values"))
+ result.length = values->length;
+ } else if (name == "tensor::zeros" || name == "tensor::full" || name == "tensor::literal" ||
+ name == "tensor::reshape" || name == "tensor::broadcast_to") {
+ if (const auto shape = find("shape"))
+ result.tensor_rank = shape->length;
+ result.tensor_scalar = input ? input->tensor_scalar : scalar();
+ } else if (name == "tensor::unsqueeze") {
+ if (input && input->tensor_rank)
+ result.tensor_rank = *input->tensor_rank + 1;
+ if (input)
+ result.tensor_scalar = input->tensor_scalar;
+ } else if (name == "tensor::pad" || name == "tensor::crop" || name == "tensor::permute" ||
+ name == "tensor::interpolate" || name == "tensor::translate" || name == "tensor::index_select" ||
+ name == "tensor::index_fill" || name == "tensor::cast") {
+ if (input) {
+ result.tensor_rank = input->tensor_rank;
+ result.tensor_scalar = name == "tensor::cast" ? scalar() : input->tensor_scalar;
+ }
+ } else if (name == "tensor::zeros_like" || name == "tensor::randn_like") {
+ if (const auto like = find("like")) {
+ result.tensor_rank = like->tensor_rank;
+ result.tensor_scalar = like->tensor_scalar;
+ }
+ } else if (name == "tensor::concat") {
+ if (const auto items = find("items")) {
+ result.tensor_rank = items->tensor_rank;
+ result.tensor_scalar = items->tensor_scalar;
+ }
+ } else if (name == "make_sigmas") {
+ result.tensor_rank = 1;
+ result.tensor_scalar = Kind::Number;
+ } else if (name == "sefi::make_sigmas") {
+ for (const char* field : {"sigmas", "semantic_sigmas"}) {
+ result.fields.at(field).tensor_rank = 1;
+ result.fields.at(field).tensor_scalar = Kind::Number;
+ }
+ } else if (name == "image::load") {
+ result.tensor_rank = 3;
+ result.tensor_scalar = Kind::Integer;
+ } else if (name == "audio::load" || name == "audio_vae_decode") {
+ result.fields.at("samples").tensor_rank = 2;
+ result.fields.at("samples").tensor_scalar = Kind::Number;
+ } else if (name == "video::load") {
+ auto& frames = result.fields.at("video").fields.at("frames");
+ frames.tensor_rank = 4;
+ frames.tensor_scalar = Kind::Integer;
+ result.fields.at("audio").fields.at("samples").tensor_rank = 2;
+ result.fields.at("audio").fields.at("samples").tensor_scalar = Kind::Number;
+ }
+ }
+
+ class Checker {
+ Graph& graph_;
+ const Registry& registry_;
+ std::set<std::string> active_;
+ size_t visits_ = 0;
+ unsigned callback_depth_ = 0;
+ std::vector<const Expr*> call_path_;
+
+ const BoundCall& resolve(const Expr& expr) {
+ const auto existing = graph_.calls.find(&expr);
+ if (existing != graph_.calls.end())
+ return existing->second;
+ BoundCall call;
+ if (expr.name == "map" || expr.name == "fold") {
+ call.iteration = bind_iteration(expr, graph_.syntax);
+ } else if (const auto user = graph_.syntax.functions.find(expr.name); user != graph_.syntax.functions.end()) {
+ call.function = &user->second;
+ call.arguments = bind_arguments(expr, call.function->parameters, true);
+ } else {
+ const auto found = registry_.find(expr.name);
+ if (found == registry_.end())
+ throw std::runtime_error("unknown function '" + expr.name + "'");
+ call.operation = &*graph_.operations.try_emplace(found->first, found->second).first;
+ call.arguments = bind_arguments(expr, argument_names(call.operation->second));
+ }
+ return graph_.calls.emplace(&expr, std::move(call)).first->second;
+ }
+
+ void constrain(const Checked& value, const Argument& argument) {
+ if (!accepts(argument.type, value.type))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "argument '" + argument.name + "' expects " + type_name(argument.type) + ", got " + type_name(value.type));
+ if (value.literal && argument.validate)
+ argument.validate(Value::literal(value.literal->literal));
+ }
+
+ Checked check_tensor_to_list(const Types& args) {
+ const auto& input = args.at("input");
+ if (input.tensor_rank && *input.tensor_rank != 1)
+ throw std::runtime_error("tensor::to_list requires a rank-one tensor");
+ const auto option = args.find("element");
+ std::string element = "auto";
+ if (option != args.end()) {
+ if (!option->second.literal)
+ throw std::runtime_error("tensor::to_list element must be known at compile time");
+ element = option->second.literal->literal.get<std::string>();
+ }
+ const auto kind = element == "auto" ? input.tensor_scalar : element == "int" ? Kind::Integer
+ : Kind::Number;
+ if (kind == Kind::Any)
+ throw std::runtime_error("tensor::to_list cannot infer dtype; specify element = \"int\" or \"number\"");
+ if (input.tensor_scalar != Kind::Any && input.tensor_scalar != kind)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "tensor::to_list element does not match input dtype; use tensor::cast to convert it");
+ Checked result{{Kind::List, kind}};
+ result.length = input.length;
+ return result;
+ }
+
+ Checked check_tensor_index(const Types& args) {
+ Checked result{{Kind::Any}};
+ const auto input = args.find("input"), indices = args.find("indices");
+ if (input == args.end() || indices == args.end() || !indices->second.length ||
+ indices->second.items.size() != *indices->second.length)
+ return result;
+ const auto& items = indices->second.items;
+ if (items.empty())
+ throw std::runtime_error("tensor indexing requires at least one axis");
+ if (input->second.tensor_rank && items.size() > *input->second.tensor_rank)
+ throw std::runtime_error("tensor indices exceed the known tensor rank");
+ size_t integers = 0;
+ bool slice = false, known = true;
+ for (const auto& axis : items) {
+ for (const auto& [name, field] : axis.fields) {
+ if (name != "index" && name != "start" && name != "end")
+ throw std::runtime_error("unknown tensor index field '" + name + "'");
+ if (field.type.kind != Kind::Integer && field.type.kind != Kind::Any)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "tensor index field '" + name + "' must be an integer");
+ }
+ if (axis.fields.count("index")) {
+ if (axis.fields.size() != 1)
+ throw std::runtime_error("a tensor index cannot also have slice endpoints");
+ ++integers;
+ } else if (axis.complete_fields) {
+ slice = true;
+ } else {
+ known = false;
+ }
+ }
+ if (slice)
+ result.type = {Kind::Tensor};
+ if (known && input->second.tensor_rank) {
+ const auto rank = *input->second.tensor_rank - integers;
+ if (rank) {
+ result.type = {Kind::Tensor};
+ result.tensor_rank = rank;
+ result.tensor_scalar = input->second.tensor_scalar;
+ } else {
+ result.type = {input->second.tensor_scalar == Kind::Any ? Kind::Number : input->second.tensor_scalar};
+ }
+ }
+ return result;
+ }
+
+ Checked check_list_index(const Checked& input, const Checked& indices) {
+ if (indices.items.size() != 1)
+ throw std::runtime_error("list indexing requires exactly one integer index or slice");
+ const auto& axis = indices.items.front();
+ const bool slice = !axis.fields.count("index");
+ if (!slice && axis.fields.size() != 1)
+ throw std::runtime_error("a list index cannot also have slice endpoints");
+ Values constant;
+ bool known = axis.complete_fields;
+ for (const auto& [name, field] : axis.fields) {
+ if (name != "index" && name != "start" && name != "end")
+ throw std::runtime_error("unknown list index field '" + name + "'");
+ if (field.type.kind != Kind::Integer && field.type.kind != Kind::Any)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "list index field '" + name + "' must be an integer");
+ if (field.literal)
+ constant.emplace(name, Value::literal(field.literal->literal));
+ else
+ known = false;
+ }
+ if (known && input.length) {
+ const auto selected = select_list(*input.length, constant);
+ if (input.items.size() == *input.length) {
+ if (!slice)
+ return input.items[selected.begin];
+ auto result = input;
+ result.items.assign(input.items.begin() + selected.begin, input.items.begin() + selected.end);
+ result.length = result.items.size();
+ summarize_list(result);
+ return result;
+ }
+ }
+ auto result = shape_only(input);
+ result.type = slice ? input.type : Type{input.type.element};
+ return result;
+ }
+
+ Checked function_at(const Expr& site, const std::string& name, const Types& params) {
+ call_path_.push_back(&site);
+ auto result = function(name, params);
+ call_path_.pop_back();
+ return result;
+ }
+
+ Checked function(const std::string& name, const Types& params) {
+ for (const auto& [parameter, value] : params)
+ if (value.type.kind == Kind::Void)
+ throw std::runtime_error("argument '" + parameter + "' has no value");
+ if (!active_.insert(name).second || active_.size() > 32)
+ throw std::runtime_error("recursive functions are not supported");
+ auto result = body(graph_.syntax.functions.at(name).body, params);
+ active_.erase(name);
+ return result;
+ }
+
+ public:
+ Checker(Graph& graph, const Registry& registry)
+ : graph_(graph), registry_(registry) {}
+
+ Checked expression(const std::shared_ptr<Expr>& expr, const Types& scope, Kind list_element = Kind::Any, bool discard_result = false) {
+ if (++visits_ > 100000)
+ throw std::runtime_error("expanded graph limit exceeded");
+ try {
+ if (expr->form == Expr::Form::Literal)
+ return {Value::literal(expr->literal).type(), expr};
+ if (expr->form == Expr::Form::Block)
+ return body(expr->body, scope, list_element);
+ if (expr->form == Expr::Form::Conditional) {
+ const auto condition = expression(expr->arguments[0].second, scope);
+ constrain(condition, required("if condition", Kind::Boolean));
+ if (!condition.literal)
+ throw std::runtime_error("if condition must be a boolean known before resource planning");
+ if (expr->arguments.size() == 2) {
+ if (!discard_result)
+ throw std::runtime_error("if used as a value requires else");
+ expression(expr->arguments[1].second, scope);
+ return {{Kind::Void}};
+ }
+ auto selected = expression(expr->arguments[1].second, scope, list_element);
+ auto alternative = expression(expr->arguments[2].second, scope, list_element);
+ if (!same_shape(selected, alternative))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "if branches must return the same type and dictionary fields, got " +
+ type_name(selected.type) + " and " + type_name(alternative.type));
+ return condition.literal->literal.get<bool>() ? std::move(selected) : std::move(alternative);
+ }
+ if (expr->form == Expr::Form::Reference) {
+ size_t dot = expr->name.find('.');
+ auto found = scope.find(expr->name.substr(0, dot));
+ if (found == scope.end())
+ throw std::runtime_error("unknown binding '" + expr->name + "'");
+ Checked result = found->second;
+ while (dot != std::string::npos) {
+ auto next = expr->name.find('.', dot + 1);
+ auto field = expr->name.substr(dot + 1, next == std::string::npos ? next : next - dot - 1);
+ if (result.type.kind != Kind::List && result.fields.count(field)) {
+ auto child = result.fields.at(field);
+ result = std::move(child);
+ } else
+ throw std::runtime_error("unknown field '" + expr->name + "'");
+ dot = next;
+ }
+ return result;
+ }
+ if (expr->form == Expr::Form::List) {
+ Checked result{{Kind::List}};
+ result.length = expr->arguments.size();
+ auto& element = result.type.element;
+ for (const auto& item : expr->arguments) {
+ auto checked = expression(item.second, scope);
+ if (list_element != Kind::Any) {
+ constrain(checked, required("list item", list_element));
+ if (checked.type.kind == Kind::Any)
+ checked.type.kind = list_element;
+ }
+ if (checked.type.kind == Kind::Any || checked.type.kind == Kind::List || checked.type.kind == Kind::Void)
+ throw std::runtime_error("list items require a known non-list type");
+ if (element == Kind::Any) {
+ element = checked.type.kind;
+ result.fields = shape_only(checked).fields;
+ result.tensor_rank = checked.tensor_rank;
+ result.tensor_scalar = checked.tensor_scalar;
+ }
+ if ((element == Kind::Integer && checked.type.kind == Kind::Number) || (element == Kind::Number && checked.type.kind == Kind::Integer))
+ element = Kind::Number;
+ else if (!accepts_list_element(element, checked.type.kind))
+ throw std::runtime_error("list items have different types");
+ keep_common_fields(result, checked);
+ result.items.push_back(std::move(checked));
+ }
+ return result;
+ }
+ if (expr->form == Expr::Form::Dict) {
+ Checked result{{Kind::Dict}};
+ result.complete_fields = true;
+ for (const auto& field : expr->arguments) {
+ auto value = expression(field.second, scope);
+ if (value.type.kind == Kind::Void)
+ throw std::runtime_error("dict field '" + field.first + "' has no value");
+ result.fields.emplace(field.first, std::move(value));
+ }
+ return result;
+ }
+ if (expr->form == Expr::Form::Index) {
+ auto input = expression(expr->arguments[0].second, scope);
+ auto indices = expression(expr->arguments[1].second, scope);
+ if (input.type.kind == Kind::List)
+ return check_list_index(input, indices);
+ if (input.type.kind != Kind::Tensor && input.type.kind != Kind::Any)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "indexing requires a list or tensor");
+ return check_tensor_index({{"input", input}, {"indices", indices}});
+ }
+ const auto& bound = resolve(*expr);
+ if (bound.iteration) {
+ const auto& call = *bound.iteration;
+ auto list = expression(call.items, scope);
+ if (list.type.kind != Kind::List || list.type.element == Kind::Any)
+ throw std::runtime_error(expr->name + " requires a typed list");
+ const auto& parameters = call.function->parameters;
+ size_t item_index = call.initial ? 1 : 0;
+ auto item = shape_only(list);
+ item.type = {list.type.element};
+ Types locals{{parameters[item_index], std::move(item)}};
+ for (const auto& arg : call.arguments)
+ if (arg.parameter != item_index)
+ locals.emplace(parameters[arg.parameter], expression(arg.expression, scope));
+ if (call.initial) {
+ auto shape = shape_only(locals.at(parameters[0]));
+ // Iterations may change tensor rank or dtype while preserving the language type.
+ clear_tensor_facts(shape);
+ if (shape.type.kind == Kind::Any)
+ throw std::runtime_error("fold requires a typed accumulator");
+ locals[parameters[0]] = shape;
+ if (!same_shape(shape, function_at(*expr, call.process, locals)))
+ throw std::runtime_error("fold process must preserve the accumulator type and fields");
+ return shape;
+ }
+ auto result = function_at(*expr, call.process, locals);
+ if (result.type.kind == Kind::List || result.type.kind == Kind::Any || result.type.kind == Kind::Void)
+ throw std::runtime_error("map process must return one typed value");
+ const auto output = std::find_if(graph_.map_outputs.begin(), graph_.map_outputs.end(), [&](const auto& site) {
+ return site.expression == expr.get() && site.call_path == call_path_;
+ });
+ if (output == graph_.map_outputs.end())
+ graph_.map_outputs.push_back({expr.get(), call_path_, result.type.kind});
+ else
+ output->element = result.type.kind;
+ result = shape_only(result);
+ result.type = {Kind::List, result.type.kind};
+ result.length = list.length;
+ return result;
+ }
+ if (bound.function) {
+ const auto& parameters = bound.function->parameters;
+ Types locals;
+ for (const auto& arg : bound.arguments)
+ locals.emplace(parameters[arg.parameter], expression(arg.expression, scope));
+ return function_at(*expr, expr->name, locals);
+ }
+ const auto& [operation_name, operation] = *bound.operation;
+ const auto& args = bound.arguments;
+ if (callback_depth_ && (operation.resource() || operation.sampling))
+ throw std::runtime_error("prediction transforms cannot load models or start sampling");
+ Types checked_args;
+ for (const auto& arg : args) {
+ const auto& argument = operation.arguments[arg.parameter];
+ if (argument.callback)
+ continue;
+ const auto element = argument.type.kind == Kind::IntVector ? Kind::Integer
+ : argument.type.kind == Kind::List ? argument.type.element
+ : Kind::Any;
+ auto checked = expression(arg.expression, scope, element);
+ if (operation.resource() && !is_literal_argument(*arg.expression) &&
+ !(active_.empty() && checked.literal))
+ throw std::runtime_error("resource arguments must be literals or literal bindings in main");
+ constrain(checked, argument);
+ checked_args.emplace(argument.name, std::move(checked));
+ }
+ for (const auto& arg : args) {
+ const auto& argument = operation.arguments[arg.parameter];
+ if (!argument.callback)
+ continue;
+ const auto& expression = arg.expression;
+ const auto fn = graph_.syntax.functions.find(expression->name);
+ if ((expression->form != Expr::Form::Reference && expression->form != Expr::Form::FunctionReference) ||
+ fn == graph_.syntax.functions.end() || fn->second.parameters.size() != 2)
+ throw std::runtime_error(argument.name + " must name a function with two parameters");
+ expression->form = Expr::Form::FunctionReference;
+ graph_.callbacks.insert_or_assign(expression.get(), &fn->second);
+ const auto& signature = *argument.callback;
+ Checked step{{Kind::Dict}};
+ for (const auto& [name, type] : signature.input_fields)
+ step.fields.emplace(name, Checked{type});
+ const auto callback_args = checked_args.find(signature.args_parameter);
+ auto captures = callback_args == checked_args.end() ? Checked{{Kind::Dict}} : callback_args->second;
+ ++callback_depth_;
+ auto result = function_at(*expr, fn->first, {{fn->second.parameters[0], std::move(step)}, {fn->second.parameters[1], std::move(captures)}});
+ --callback_depth_;
+ constrain(result, required(argument.name + " return value", Kind::Dict));
+ for (const auto& [name, field] : result.fields) {
+ const auto expected = signature.output_fields.find(name);
+ if (expected == signature.output_fields.end())
+ throw std::runtime_error("unknown " + argument.name + " result field '" + name + "'");
+ if (!accepts(expected->second, field.type))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "invalid type for " + argument.name + " result field '" + name + "'");
+ }
+ checked_args.emplace(argument.name, Checked{{Kind::Function}});
+ }
+ Checked result{operation.output.type};
+ if (operation_name == "math::add" || operation_name == "math::sub" || operation_name == "math::mul") {
+ const auto a = checked_args.find("a"), b = checked_args.find("b");
+ if (a != checked_args.end() && b != checked_args.end()) {
+ const auto left = a->second.type.kind, right = b->second.type.kind;
+ result.type = {left == Kind::Integer && right == Kind::Integer ? Kind::Integer
+ : left == Kind::Number || right == Kind::Number ? Kind::Number
+ : Kind::Any};
+ }
+ }
+ for (const auto& argument : operation.arguments) {
+ const auto found = checked_args.find(argument.name);
+ if (found == checked_args.end()) {
+ if (argument.required)
+ throw std::runtime_error("missing argument '" + argument.name + "'");
+ continue;
+ }
+ const auto& checked = found->second;
+ if (operation.output.same_type_as == argument.name) {
+ result.type = checked.type;
+ result.fields = shape_only(checked).fields;
+ }
+ }
+ if (operation_name == "zero_out") {
+ const auto& condition = checked_args.at("condition");
+ result = shape_only(condition);
+ if (const auto selected = checked_args.find("fields"); selected != checked_args.end())
+ for (const auto& item : selected->second.items) {
+ if (!item.literal)
+ continue;
+ const auto name = item.literal->literal.get<std::string>();
+ const auto field = condition.fields.find(name);
+ if (field == condition.fields.end()) {
+ if (condition.complete_fields)
+ throw std::runtime_error("zero_out: condition has no field '" + name + "'");
+ } else if (field->second.type.kind != Kind::Tensor && !(field->second.type == Type{Kind::List, Kind::Tensor})) {
+ throw Error(SD_STATUS_TYPE_MISMATCH, "zero_out: field '" + name + "' must be a tensor or tensor list");
+ }
+ }
+ }
+ if (operation_name == "list::replace") {
+ result = checked_args.at("items");
+ const auto& value = checked_args.at("value");
+ if (result.type.element == Kind::Any || !accepts_list_element(result.type.element, value.type.kind))
+ throw Error(SD_STATUS_TYPE_MISMATCH, "replacement must match the list element type: expected " +
+ type_name({result.type.element}) + ", got " + type_name(value.type));
+ const auto& index = checked_args.at("index");
+ if (index.literal && result.length && result.items.size() == *result.length) {
+ const auto selected = select_list(*result.length, {{"index", Value::literal(index.literal->literal)}});
+ result.items[selected.begin] = value;
+ summarize_list(result);
+ } else {
+ result.items.clear();
+ keep_common_fields(result, value);
+ }
+ }
+ for (const auto& field : operation.output.fields)
+ result.fields[field.first] = checked_output(field.second);
+ infer_tensor_result(operation_name, checked_args, result);
+ if (operation_name == "tensor::index")
+ result = check_tensor_index(checked_args);
+ else if (operation_name == "tensor::to_list")
+ result = check_tensor_to_list(checked_args);
+ return result;
+ } catch (const std::bad_alloc&) {
+ throw;
+ } catch (const std::exception& error) {
+ const auto* diagnostic = dynamic_cast<const Error*>(&error);
+ throw Error(diagnostic ? diagnostic->status : SD_STATUS_INVALID_ARGUMENT,
+ "line " + std::to_string(expr->line) + ", column " + std::to_string(expr->column) + ": " + error.what());
+ }
+ }
+
+ Checked body(const std::vector<Statement>& statements, Types scope, Kind list_element = Kind::Any) {
+ for (const auto& statement : statements) {
+ if (!statement.value)
+ return {{Kind::Void}};
+ auto checked = expression(statement.value, scope,
+ statement.form == Statement::Form::Result ? list_element : Kind::Any,
+ statement.form == Statement::Form::Call);
+ if (statement.form == Statement::Form::Call)
+ continue;
+ if (statement.form == Statement::Form::Result)
+ return checked;
+ if (checked.type.kind == Kind::Void)
+ throw std::runtime_error("line " + std::to_string(statement.value->line) + ", column " + std::to_string(statement.value->column) + ": function has no return value; call it as a standalone statement");
+ if (statement.form == Statement::Form::Return)
+ return checked;
+ if (!scope.emplace(statement.name, checked).second)
+ throw std::runtime_error("binding shadows existing name '" + statement.name + "'");
+ }
+ return {{Kind::Void}};
+ }
+ };
+
+ static void count_references(const Expr& expr, ReferenceCounts& counts, Graph& graph) {
+ if (expr.form == Expr::Form::Block) {
+ auto [found, inserted] = graph.blocks.try_emplace(&expr);
+ auto& block = found->second;
+ if (inserted) {
+ std::set<std::string> locals;
+ for (const auto& statement : expr.body) {
+ ReferenceCounts references;
+ if (statement.value)
+ count_references(*statement.value, references, graph);
+ for (const auto& [name, count] : references) {
+ block.references[name] += count;
+ if (!locals.count(name.substr(0, name.find('.'))))
+ block.captures[name] += count;
+ }
+ if (statement.form == Statement::Form::Binding)
+ locals.insert(statement.name);
+ }
+ }
+ for (const auto& [name, count] : block.captures)
+ counts[name] += count;
+ return;
+ }
+ if (expr.form == Expr::Form::Reference)
+ ++counts[expr.name];
+ const bool iteration = expr.form == Expr::Form::Call && (expr.name == "map" || expr.name == "fold");
+ const size_t process = expr.name == "fold" ? 2 : 1;
+ for (size_t i = 0; i < expr.arguments.size(); ++i) {
+ if (!iteration || i != process)
+ count_references(*expr.arguments[i].second, counts, graph);
+ }
+ }
+
+ static ReferenceCounts reference_counts(const Function& function, Graph& graph) {
+ ReferenceCounts counts;
+ for (const auto& statement : function.body)
+ if (statement.value)
+ count_references(*statement.value, counts, graph);
+ return counts;
+ }
+
+ Graph compile(const std::string& source, const Registry& registry, bool host_inputs) {
+ Graph graph;
+ graph.source = source;
+ graph.syntax = parse_syntax(source, host_inputs);
+ for (const auto& function : graph.syntax.functions) {
+ if (function.first == "map" || function.first == "fold" || registry.count(function.first))
+ throw std::runtime_error("function name '" + function.first + "' is reserved for a built-in function");
+ }
+ Checker checker(graph, registry);
+ Types parameters;
+ for (const auto& name : graph.syntax.entry.parameters)
+ parameters.emplace(name, Checked{{Kind::Tensor}});
+ checker.body(graph.syntax.entry.body, std::move(parameters));
+ graph.references.emplace("main", reference_counts(graph.syntax.entry, graph));
+ for (const auto& [name, function] : graph.syntax.functions)
+ graph.references.emplace(name, reference_counts(function, graph));
+ analyze_resources(graph);
+ return graph;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/graph.h b/src/dgl/graph.h
new file mode 100644
index 0000000..4121b79
--- /dev/null
+++ b/src/dgl/graph.h
@@ -0,0 +1,84 @@
+#ifndef __SD_DGL_GRAPH_H__
+#define __SD_DGL_GRAPH_H__
+
+#include "registry.h"
+#include "syntax/syntax.h"
+
+namespace sd::dgl {
+
+ using ReferenceCounts = std::map<std::string, size_t>;
+
+ struct BlockReferences {
+ ReferenceCounts references;
+ ReferenceCounts captures;
+ };
+
+ struct BoundArgument {
+ size_t parameter;
+ std::shared_ptr<Expr> expression;
+ };
+
+ struct IterationCall {
+ const Function* function;
+ std::string process;
+ std::shared_ptr<Expr> items, initial;
+ std::vector<BoundArgument> arguments;
+ };
+
+ struct BoundCall {
+ const Registry::Entry* operation = nullptr;
+ const Function* function = nullptr;
+ std::vector<BoundArgument> arguments;
+ std::optional<IterationCall> iteration;
+ };
+
+ struct Graph {
+ struct ResourceCall {
+ const Expr* expression;
+ const Registry::Entry* operation;
+ Values arguments;
+ bool deferred = false;
+ };
+ struct ResourceUse {
+ const Expr* resource;
+ bool preview = false;
+ };
+ struct Phase {
+ const Expr* statement;
+ std::string name;
+ std::vector<ResourceUse> resources;
+ std::vector<const Expr*> call_path;
+ bool after = false;
+ };
+ struct MapOutput {
+ const Expr* expression;
+ std::vector<const Expr*> call_path;
+ Kind element;
+ };
+ std::vector<MapOutput> map_outputs;
+ std::string source;
+ Syntax syntax;
+ Registry::Operations operations;
+ std::map<const Expr*, BoundCall> calls;
+ std::map<const Expr*, const Function*> callbacks;
+ std::map<std::string, ReferenceCounts> references;
+ std::map<const Expr*, BlockReferences> blocks;
+ std::vector<ResourceCall> resources;
+ std::vector<Phase> phases;
+
+ Graph() = default;
+ Graph(Graph&&) = default;
+ Graph& operator=(Graph&&) = default;
+ Graph(const Graph&) = delete;
+ Graph& operator=(const Graph&) = delete;
+ };
+
+ std::vector<BoundArgument> bind_arguments(const Expr& expr, const std::vector<std::string>& names, bool all_required = false);
+ std::vector<std::string> argument_names(const Operation& operation);
+ IterationCall bind_iteration(const Expr& expr, const Syntax& syntax);
+ Graph compile(const std::string& source, const Registry& registry, bool host_inputs = false);
+ void analyze_resources(Graph& graph);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_GRAPH_H__
diff --git a/src/dgl/graph_resources.cpp b/src/dgl/graph_resources.cpp
new file mode 100644
index 0000000..7d47c4c
--- /dev/null
+++ b/src/dgl/graph_resources.cpp
@@ -0,0 +1,302 @@
+#include "graph.h"
+#include "indexing.h"
+
+#include <set>
+
+namespace sd::dgl {
+ struct ResourceValue {
+ std::optional<Value> literal;
+ std::map<const Expr*, bool> origins;
+ std::map<std::string, ResourceValue> fields;
+ std::vector<ResourceValue> items;
+ bool known_items = false;
+ };
+
+ static void merge_origins(std::map<const Expr*, bool>& into, const ResourceValue& value, bool preview = false) {
+ for (const auto& [origin, conditional] : value.origins) {
+ const bool condition = preview || conditional;
+ auto inserted = into.emplace(origin, condition);
+ if (!condition)
+ inserted.first->second = false;
+ }
+ for (const auto& [_, field] : value.fields)
+ merge_origins(into, field, preview);
+ for (const auto& item : value.items)
+ merge_origins(into, item, preview);
+ }
+
+ static bool captures_model(Kind kind) {
+ return is_model_kind(kind) || kind == Kind::Model || kind == Kind::Lora || kind == Kind::Predictor ||
+ kind == Kind::SamplingObserver || kind == Kind::ControlBindings || kind == Kind::IPAdapterBindings || kind == Kind::PuLIDBindings;
+ }
+
+ static bool merge_resources(ResourceValue& into, const ResourceValue& from) {
+ bool changed = false;
+ if (into.literal) {
+ into.literal.reset();
+ changed = true;
+ }
+ for (const auto& [origin, preview] : from.origins) {
+ auto inserted = into.origins.emplace(origin, preview);
+ if (inserted.second)
+ changed = true;
+ else if (inserted.first->second && !preview) {
+ inserted.first->second = false;
+ changed = true;
+ }
+ }
+ for (const auto& [name, field] : from.fields) {
+ auto inserted = into.fields.emplace(name, field);
+ changed = inserted.second || changed;
+ if (!inserted.second)
+ changed = merge_resources(inserted.first->second, field) || changed;
+ }
+ if (into.known_items && (!from.known_items || into.items.size() != from.items.size())) {
+ into.known_items = false;
+ changed = true;
+ }
+ for (size_t i = 0; i < from.items.size(); ++i) {
+ if (i == into.items.size()) {
+ into.items.push_back(from.items[i]);
+ changed = true;
+ } else
+ changed = merge_resources(into.items[i], from.items[i]) || changed;
+ }
+ return changed;
+ }
+
+ class ResourceAnalysis {
+ Graph& graph_;
+ std::set<const Expr*> discovered_;
+ std::map<const Expr*, bool> used_;
+ size_t iteration_depth_ = 0;
+ std::vector<const Expr*> call_path_;
+ Graph::Phase pending_{};
+
+ void flush_phase() {
+ if (used_.empty())
+ return;
+ for (const auto& resource : graph_.resources)
+ if (auto use = used_.find(resource.expression); use != used_.end())
+ pending_.resources.push_back({use->first, use->second});
+ graph_.phases.push_back(pending_);
+ pending_.resources.clear();
+ used_.clear();
+ }
+
+ void boundary(const Expr* expression, bool after = false) {
+ if (iteration_depth_)
+ return;
+ flush_phase();
+ pending_ = {expression, expression->name.empty() ? "if" : expression->name, {}, call_path_, after};
+ }
+
+ using Scope = std::map<std::string, ResourceValue>;
+
+ ResourceValue body(const std::vector<Statement>& statements, Scope scope, bool phases = false) {
+ for (const auto& statement : statements) {
+ if (!statement.value)
+ return {};
+ if (phases)
+ boundary(statement.value.get());
+ auto value = expression(statement.value, scope);
+ if (statement.form == Statement::Form::Return || statement.form == Statement::Form::Result)
+ return value;
+ if (statement.form == Statement::Form::Binding)
+ scope[statement.name] = std::move(value);
+ }
+ return {};
+ }
+
+ ResourceValue expression(const std::shared_ptr<Expr>& expression, const Scope& scope) {
+ const auto& expr = *expression;
+ if (expr.form == Expr::Form::Literal)
+ return {Value::literal(expr.literal)};
+ if (expr.form == Expr::Form::Block) {
+ auto result = body(expr.body, scope, true);
+ boundary(&expr, true);
+ return result;
+ }
+ if (expr.form == Expr::Form::Conditional) {
+ const auto condition = this->expression(expr.arguments[0].second, scope);
+ if (!condition.literal || condition.literal->type().kind != Kind::Boolean)
+ throw std::runtime_error("if condition must be a boolean known before resource planning");
+ const bool selected = condition.literal->scalar.get<bool>();
+ if (expr.arguments.size() == 2) {
+ if (selected)
+ this->expression(expr.arguments[1].second, scope);
+ return {};
+ }
+ return this->expression(expr.arguments[selected ? 1 : 2].second, scope);
+ }
+ if (expr.form == Expr::Form::FunctionReference)
+ return {};
+ if (expr.form == Expr::Form::Reference) {
+ auto dot = expr.name.find('.');
+ const auto found = scope.find(expr.name.substr(0, dot));
+ if (found == scope.end())
+ return {};
+ auto value = found->second;
+ while (dot != std::string::npos) {
+ auto next = expr.name.find('.', dot + 1);
+ const auto field = value.fields.find(expr.name.substr(dot + 1, next == std::string::npos ? next : next - dot - 1));
+ if (field == value.fields.end()) {
+ ResourceValue aggregate;
+ merge_origins(aggregate.origins, value);
+ return aggregate;
+ }
+ auto child = field->second;
+ value = std::move(child);
+ dot = next;
+ }
+ return value;
+ }
+ if (expr.form == Expr::Form::List || expr.form == Expr::Form::Dict) {
+ ResourceValue result;
+ result.known_items = expr.form == Expr::Form::List;
+ bool constant = true;
+ Values fields;
+ std::vector<Value> items;
+ for (const auto& [name, item] : expr.arguments) {
+ auto value = this->expression(item, scope);
+ constant = constant && bool(value.literal);
+ if (expr.form == Expr::Form::List) {
+ if (value.literal)
+ items.push_back(*value.literal);
+ result.items.push_back(std::move(value));
+ } else {
+ if (value.literal)
+ fields.emplace(name, *value.literal);
+ result.fields.emplace(name, std::move(value));
+ }
+ }
+ if (constant)
+ result.literal = expr.form == Expr::Form::List ? Value::list(std::move(items)) : Value::dict(std::move(fields));
+ return result;
+ }
+ if (expr.form == Expr::Form::Index) {
+ auto source = this->expression(expr.arguments[0].second, scope);
+ const auto indices = this->expression(expr.arguments[1].second, scope);
+ if (source.known_items && indices.literal && indices.literal->items.size() == 1) {
+ const auto selected = select_list(source.items.size(), indices.literal->items[0].fields);
+ if (!selected.slice)
+ return source.items[selected.begin];
+ if (source.literal)
+ source.literal = index_value(*source.literal, *indices.literal);
+ source.items = std::vector<ResourceValue>(source.items.begin() + selected.begin, source.items.begin() + selected.end);
+ return source;
+ }
+ ResourceValue result;
+ for (const auto& item : source.items)
+ merge_resources(result, item);
+ merge_origins(result.origins, source);
+ return result;
+ }
+ const auto& call = graph_.calls.at(&expr);
+ if (call.iteration) {
+ const auto& iteration = *call.iteration;
+ auto collection = this->expression(iteration.items, scope);
+ Scope locals;
+ const size_t item_parameter = iteration.initial ? 1 : 0;
+ for (const auto& arg : iteration.arguments)
+ if (arg.parameter != item_parameter)
+ locals.emplace(iteration.function->parameters[arg.parameter], this->expression(arg.expression, scope));
+ if (collection.literal && collection.items.empty())
+ return iteration.initial ? locals.at(iteration.function->parameters[0]) : ResourceValue{};
+ ResourceValue item;
+ for (const auto& candidate : collection.items) {
+ merge_origins(item.origins, candidate);
+ for (const auto& [field, child] : candidate.fields)
+ merge_origins(item.fields[field].origins, child);
+ }
+ merge_origins(item.origins, collection);
+ locals[iteration.function->parameters[item_parameter]] = std::move(item);
+ ++iteration_depth_;
+ if (iteration.initial) {
+ auto accumulator = locals.at(iteration.function->parameters[0]);
+ for (;;) {
+ locals[iteration.function->parameters[0]] = accumulator;
+ auto output = body(iteration.function->body, locals);
+ if (!merge_resources(accumulator, output))
+ break;
+ }
+ --iteration_depth_;
+ return accumulator;
+ }
+ auto output = body(iteration.function->body, std::move(locals));
+ --iteration_depth_;
+ ResourceValue result;
+ result.items.push_back(std::move(output));
+ return result;
+ }
+ if (call.function) {
+ Scope locals;
+ for (const auto& arg : call.arguments)
+ locals.emplace(call.function->parameters[arg.parameter], this->expression(arg.expression, scope));
+ call_path_.push_back(&expr);
+ auto result = body(call.function->body, std::move(locals));
+ call_path_.pop_back();
+ return result;
+ }
+ const auto& [operation_name, operation] = *call.operation;
+ Scope args;
+ for (const auto& arg : call.arguments)
+ if (!operation.arguments[arg.parameter].callback)
+ args.emplace(operation.arguments[arg.parameter].name, this->expression(arg.expression, scope));
+ for (const auto& argument : operation.arguments)
+ if (argument.default_value && !args.count(argument.name))
+ args[argument.name].literal = *argument.default_value;
+ if (operation.resource()) {
+ if (discovered_.insert(&expr).second) {
+ Values arguments;
+ for (const auto& [name, value] : args) {
+ if (!value.literal)
+ throw std::runtime_error("cannot resolve resource argument '" + name + "'");
+ arguments.emplace(name, *value.literal);
+ }
+ graph_.resources.push_back({&expr, call.operation, std::move(arguments), iteration_depth_ != 0});
+ }
+ ResourceValue value;
+ value.origins.emplace(&expr, false);
+ return value;
+ }
+ if (operation_name == "list::replace") {
+ auto result = args.at("items");
+ const auto& value = args.at("value");
+ const auto& index = args.at("index");
+ result.literal.reset();
+ if (result.known_items && index.literal) {
+ const auto selected = select_list(result.items.size(), {{"index", *index.literal}});
+ result.items[selected.begin] = value;
+ } else if (result.known_items) {
+ for (auto& item : result.items)
+ merge_resources(item, value);
+ } else
+ merge_resources(result, value);
+ return result;
+ }
+ ResourceValue inputs;
+ for (const auto& [_, value] : args)
+ merge_origins(inputs.origins, value, operation.model_use == Operation::ModelUse::Preview);
+ if (operation.model_use == Operation::ModelUse::Capture || operation.model_use == Operation::ModelUse::Preview || captures_model(operation.output.type.kind))
+ return inputs;
+ if (operation.model_use != Operation::ModelUse::Metadata)
+ merge_origins(used_, inputs);
+ if (!operation.output.same_type_as.empty())
+ return args.at(operation.output.same_type_as);
+ return {};
+ }
+
+ public:
+ explicit ResourceAnalysis(Graph& graph)
+ : graph_(graph) {}
+ void run() {
+ body(graph_.syntax.entry.body, {}, true);
+ flush_phase();
+ }
+ };
+
+ void analyze_resources(Graph& graph) {
+ ResourceAnalysis(graph).run();
+ }
+}
diff --git a/src/dgl/indexing.cpp b/src/dgl/indexing.cpp
new file mode 100644
index 0000000..6697a65
--- /dev/null
+++ b/src/dgl/indexing.cpp
@@ -0,0 +1,86 @@
+#include "indexing.h"
+
+#include <algorithm>
+#include <type_traits>
+
+#include "dgl/ops/tensor.h"
+
+namespace sd::dgl {
+ ListSelection select_list(size_t count, const Values& axis) {
+ for (const auto& [name, value] : axis) {
+ if (name != "index" && name != "start" && name != "end")
+ throw std::runtime_error("unknown list index field '" + name + "'");
+ if (value.type().kind != Kind::Integer)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "list index field '" + name + "' must be an integer");
+ }
+ const auto length = int64_t(count);
+ if (axis.count("index")) {
+ if (axis.size() != 1)
+ throw std::runtime_error("a list index cannot also have slice endpoints");
+ auto index = get_int(axis, "index");
+ if (index < 0)
+ index += length;
+ if (index < 0 || index >= length)
+ throw std::runtime_error("list index out of range");
+ return {size_t(index), size_t(index + 1), false};
+ }
+ auto endpoint = [&](const char* name, int64_t fallback) {
+ const auto found = axis.find(name);
+ if (found == axis.end())
+ return fallback;
+ auto value = found->second.scalar.get<int64_t>();
+ if (value < 0)
+ value += length;
+ return std::clamp(value, int64_t(0), length);
+ };
+ const auto begin = endpoint("start", 0);
+ return {size_t(begin), size_t(std::max(begin, endpoint("end", length))), true};
+ }
+
+ Value index_value(Value input, const Value& descriptors) {
+ if (input.type().kind == Kind::List) {
+ if (descriptors.type().kind != Kind::List || descriptors.items.size() != 1 || descriptors.items[0].type().kind != Kind::Dict)
+ throw std::runtime_error("list indexing requires exactly one integer index or slice");
+ const auto selected = select_list(input.items.size(), descriptors.items[0].fields);
+ if (!selected.slice)
+ return std::move(input.items[selected.begin]);
+ std::vector<Value> items;
+ for (size_t i = selected.begin; i < selected.end; ++i)
+ items.push_back(std::move(input.items[i]));
+ return Value::list(std::move(items), input.type().element);
+ }
+ if (input.type().kind != Kind::Tensor)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "indexing requires a list or tensor");
+ std::vector<ops::TensorIndex> indices;
+ for (const auto& axis : descriptors.items) {
+ if (axis.type().kind != Kind::Dict)
+ throw std::runtime_error("tensor indices must be dicts with index or start/end integers");
+ for (const auto& [name, value] : axis.fields) {
+ if (name != "index" && name != "start" && name != "end")
+ throw std::runtime_error("unknown tensor index field '" + name + "'");
+ if (value.type().kind != Kind::Integer)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "tensor index field '" + name + "' must be an integer");
+ }
+ if (axis.fields.count("index")) {
+ if (axis.fields.size() != 1)
+ throw std::runtime_error("a tensor index cannot also have slice endpoints");
+ indices.emplace_back(get_int(axis.fields, "index"));
+ } else {
+ ops::TensorSlice slice;
+ if (axis.fields.count("start"))
+ slice.start = get_int(axis.fields, "start");
+ if (axis.fields.count("end"))
+ slice.end = get_int(axis.fields, "end");
+ indices.emplace_back(slice);
+ }
+ }
+ auto result = ops::index_tensor(input.as<sd_tensor_t>(), indices);
+ return std::visit([](auto value) -> Value {
+ if constexpr (std::is_same_v<decltype(value), std::shared_ptr<sd_tensor_t>>)
+ return Value::object(std::move(value));
+ else
+ return Value::literal(value);
+ },
+ std::move(result));
+ }
+}
diff --git a/src/dgl/indexing.h b/src/dgl/indexing.h
new file mode 100644
index 0000000..316500b
--- /dev/null
+++ b/src/dgl/indexing.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_INDEXING_H__
+#define __SD_DGL_INDEXING_H__
+
+#include "value.h"
+
+namespace sd::dgl {
+ struct ListSelection {
+ size_t begin, end;
+ bool slice;
+ };
+
+ ListSelection select_list(size_t count, const Values& axis);
+ Value index_value(Value input, const Value& indices);
+}
+
+#endif // __SD_DGL_INDEXING_H__
diff --git a/src/dgl/model_value.cpp b/src/dgl/model_value.cpp
new file mode 100644
index 0000000..a57cd07
--- /dev/null
+++ b/src/dgl/model_value.cpp
@@ -0,0 +1,96 @@
+#include "dgl/model_value.h"
+
+#include <type_traits>
+
+#include "context.h"
+#include "dgl/models/audio_encoder.h"
+#include "dgl/models/audio_vae.h"
+#include "dgl/models/control.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/photomaker.h"
+#include "dgl/models/pulid.h"
+#include "dgl/models/text_encoder.h"
+#include "dgl/models/vae.h"
+#include "dgl/models/vision_encoder.h"
+#include "models/animatediff.h"
+#include "models/detail_detector.h"
+#include "models/diffusion_model.h"
+#include "models/latent_upscaler.h"
+#include "models/upscaler.h"
+
+namespace sd::dgl {
+ template <class F>
+ static decltype(auto) visit_model(const Value& value, F&& visit) {
+ switch (value.type().kind) {
+ case Kind::Denoiser:
+ return visit(value.as<models::DiffusionModel>()->dit);
+ case Kind::TextEncoder:
+ return visit(value.as<models::TextEncoder>());
+ case Kind::VAE:
+ return visit(value.as<models::VAE>());
+ case Kind::ControlNet:
+ return visit(value.as<models::ControlNet>());
+ case Kind::VisionEncoder:
+ return visit(value.as<models::VisionEncoder>());
+ case Kind::AudioEncoder:
+ return visit(value.as<models::AudioEncoder>());
+ case Kind::AudioVAE:
+ return visit(value.as<models::AudioVAE>());
+ case Kind::IPAdapter:
+ return visit(value.as<models::IPAdapter>());
+ case Kind::PhotoMaker:
+ return visit(value.as<models::PhotoMaker>());
+ case Kind::PuLID:
+ return visit(value.as<models::PuLID>());
+ case Kind::AnimateDiff:
+ return visit(value.as<models::AnimateDiff>());
+ case Kind::LatentUpscaler:
+ return visit(value.as<models::LatentUpscaler>());
+ case Kind::Upscaler:
+ return visit(value.as<models::ImageUpscaler>());
+ case Kind::Detector:
+ return visit(value.as<models::DetailDetector>());
+ default:
+ throw Error(SD_STATUS_TYPE_MISMATCH, "expected a model, got " + type_name(value.type()));
+ }
+ }
+
+ models::Model& model_from_value(const Value& value) {
+ return visit_model(value, [](const auto& model) -> models::Model& { return *model; });
+ }
+
+ void set_conditioning_cache_size(const Value& value, size_t capacity) {
+ if (value.type().kind == Kind::TextEncoder)
+ value.as<models::TextEncoder>()->set_conditioning_cache_size(capacity);
+ }
+
+ Value build_model_variant(Context& context, const Value& base, const std::shared_ptr<const models::LoraConfig>& loras) {
+ return visit_model(base, [&](const auto& model) {
+ using T = typename std::decay_t<decltype(model)>::element_type;
+ auto variant = context.make_model<T>(base.model_source->key);
+ if (!variant->init_variant(*model, loras, context.keep_resources))
+ throw std::runtime_error("cannot build LoRA variant of " + type_name(base.type()));
+ if constexpr (std::is_same_v<T, models::DiT>)
+ return Value::object(std::make_shared<models::DiffusionModel>(models::DiffusionModel{std::move(variant)}));
+ else
+ return Value::object(std::move(variant));
+ });
+ }
+
+ Value with_animatediff(const Value& base, const std::shared_ptr<models::AnimateDiff>& motion) {
+ const auto base_model = base.as<models::DiffusionModel>();
+ motion->validate_target(*base_model->dit);
+ auto model = std::make_shared<models::DiffusionModel>(models::DiffusionModel{
+ base_model->dit, motion, base_model->source ? base_model->source : base_model});
+ auto value = Value::object(std::move(model));
+ value.model_source = base.model_source;
+ value.loras = base.loras;
+ return value;
+ }
+
+ Value bind_model_variant(const Value& base, Value variant) {
+ const auto* base_model = base.get_if<models::DiffusionModel>();
+ return base_model && base_model->motion ? with_animatediff(variant, base_model->motion) : std::move(variant);
+ }
+}
diff --git a/src/dgl/model_value.h b/src/dgl/model_value.h
new file mode 100644
index 0000000..fe0d848
--- /dev/null
+++ b/src/dgl/model_value.h
@@ -0,0 +1,16 @@
+#ifndef __SD_DGL_MODEL_VALUE_H__
+#define __SD_DGL_MODEL_VALUE_H__
+
+#include "value.h"
+
+namespace sd::dgl {
+ struct Context;
+
+ models::Model& model_from_value(const Value& value);
+ void set_conditioning_cache_size(const Value& value, size_t capacity);
+ Value build_model_variant(Context& context, const Value& base, const std::shared_ptr<const models::LoraConfig>& loras);
+ Value bind_model_variant(const Value& base, Value variant);
+ Value with_animatediff(const Value& base, const std::shared_ptr<models::AnimateDiff>& motion);
+}
+
+#endif // __SD_DGL_MODEL_VALUE_H__
diff --git a/src/dgl/models/animatediff.cpp b/src/dgl/models/animatediff.cpp
new file mode 100644
index 0000000..1dffb79
--- /dev/null
+++ b/src/dgl/models/animatediff.cpp
@@ -0,0 +1,77 @@
+#include "dgl/models/animatediff.h"
+
+#include <climits>
+
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/dit.h"
+#include "model/diffusion/unet.hpp"
+
+namespace sd::dgl::models {
+ using MotionRunner = ::AnimateDiff::MotionRunner;
+
+ AnimateDiff::AnimateDiff(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::AnimateDiff, SDBackendModule::DIFFUSION, [this] { return build_runners(); }) {}
+
+ bool AnimateDiff::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::AnimateDiff>);
+ }
+
+ bool AnimateDiff::build_variant(const Model& source, ModelLoader loader) {
+ return runtime_.build_variant(static_cast<const AnimateDiff&>(source).runtime_, std::move(loader));
+ }
+
+ bool AnimateDiff::build_runners() {
+ if (runtime_.version != VERSION_SD1)
+ throw std::runtime_error("AnimateDiff requires family = \"sd1\"");
+
+ const auto& weights = model_weights->loader().get_tensor_storage_map();
+ const std::string prefix = std::string(MotionRunner::prefix) + ".";
+ const auto first_pe = weights.find(prefix + "down_blocks.0.motion_modules.0.temporal_transformer.transformer_blocks.0.attention_blocks.0.pos_encoder.pe");
+ if (first_pe == weights.end() || first_pe->second.ne[1] <= 0 || first_pe->second.ne[1] > INT_MAX)
+ throw std::runtime_error("AnimateDiff checkpoint has no valid positional encoding");
+
+ ::AnimateDiff::MotionModuleConfig config;
+ config.max_frames = int(first_pe->second.ne[1]);
+ config.enable_mid_block = weights.count(prefix + "mid_block.motion_modules.0.temporal_transformer.proj_in.weight") != 0;
+ for (const auto& [name, tensor] : weights) {
+ if (starts_with(name, prefix) && ends_with(name, ".pos_encoder.pe") &&
+ (tensor.ne[1] != config.max_frames || tensor.ne[2] != 1))
+ throw std::runtime_error("inconsistent AnimateDiff positional encoding at '" + name + "'");
+ }
+
+ runner_ = std::make_shared<MotionRunner>(runtime_.backend_for(), weights, config, model_weights);
+ runner_->set_scale_overrides(runtime_.params.linear_scale,
+ runtime_.params.attn_scale);
+ runner_->set_sage_attention_enabled(runtime_.params.sage_attn);
+ return runtime_.register_runner_params(runtime_.component, runner_);
+ }
+
+ void AnimateDiff::validate_target(const DiT& target) const {
+ const auto* unet = dynamic_cast<const UNetModelRunner*>(target.diffusion_model.get());
+ if (!unet || !sd_version_is_sd1(target.version()) || unet->config.tiny_unet ||
+ unet->config.model_channels != 320 || unet->config.num_res_blocks != 2 ||
+ unet->config.channel_mult != std::vector<int>{1, 2, 4, 4})
+ throw std::runtime_error("AnimateDiff requires a compatible SD 1.5 UNet");
+ if (runner_->compute_backend() != unet->compute_backend())
+ throw std::runtime_error("AnimateDiff and the UNet require the same compute backend");
+ }
+
+ void AnimateDiff::validate_latent(const Tensor<float>& latent) const {
+ if (latent.empty() || latent.dim() != 4)
+ throw std::runtime_error("AnimateDiff expects a latent with shape [width, height, channels, frames]");
+ if (latent.shape()[3] > runner_->max_frames())
+ throw std::runtime_error("AnimateDiff checkpoint supports up to " + std::to_string(runner_->max_frames()) + " frames");
+ }
+
+ ggml_tensor* AnimateDiff::apply(GGMLRunnerContext& ctx, UNetBlockStage stage, int level, int block, ggml_tensor* activation, int frames) const {
+ const auto block_name = stage == UNetBlockStage::Middle ? std::string("mid_block")
+ : (stage == UNetBlockStage::Down ? "down_blocks." : "up_blocks.") + std::to_string(level);
+ return runner_->forward(ctx, block_name + ".motion_modules." + std::to_string(block), activation, frames);
+ }
+
+ size_t AnimateDiff::extra_graph_size() const {
+ return runner_->extra_graph_size();
+ }
+}
diff --git a/src/dgl/models/animatediff.h b/src/dgl/models/animatediff.h
new file mode 100644
index 0000000..5bef60b
--- /dev/null
+++ b/src/dgl/models/animatediff.h
@@ -0,0 +1,33 @@
+#ifndef __SD_DGL_MODELS_ANIMATEDIFF_H__
+#define __SD_DGL_MODELS_ANIMATEDIFF_H__
+
+#include "dgl/models/model_runtime.h"
+#include "model/diffusion/model.hpp"
+
+namespace AnimateDiff {
+ class MotionRunner;
+}
+
+namespace sd::dgl::models {
+ class DiT;
+
+ class AnimateDiff final : public Model, public UNetBlockProcessor {
+ ModelRuntime runtime_;
+ std::shared_ptr<::AnimateDiff::MotionRunner> runner_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override;
+ bool build_runners();
+
+ public:
+ AnimateDiff(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(ModelLoadParams params);
+ void validate_target(const DiT& target) const;
+ void validate_latent(const Tensor<float>& latent) const;
+ ggml_tensor* apply(GGMLRunnerContext& ctx, UNetBlockStage stage, int level, int block, ggml_tensor* activation, int frames) const override;
+ size_t extra_graph_size() const override;
+ };
+}
+
+#endif // __SD_DGL_MODELS_ANIMATEDIFF_H__
diff --git a/src/dgl/models/audio_encoder.cpp b/src/dgl/models/audio_encoder.cpp
new file mode 100644
index 0000000..796ad95
--- /dev/null
+++ b/src/dgl/models/audio_encoder.cpp
@@ -0,0 +1,46 @@
+#include "dgl/models/audio_encoder.h"
+
+#include <algorithm>
+
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+#include "model/audio/wav2vec2.hpp"
+#include "runtime/audio_processing.h"
+
+namespace sd::dgl::models {
+ AudioEncoder::AudioEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::AudioEncoder, SDBackendModule::AUDIO_ENCODER, [this] { return build_runners(); }) {}
+
+ bool AudioEncoder::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::AudioEncoder>);
+ }
+
+ bool AudioEncoder::build_runners() {
+ audio_encoder = std::make_shared<Wav2Vec2::Wav2Vec2ModelRunner>(runtime_.backend_for(), model_weights->loader().get_tensor_storage_map(), "wav2vec2.", model_weights);
+ runtime_.configure_runner(*audio_encoder);
+ return runtime_.register_runner_params(runtime_.component, audio_encoder);
+ }
+
+ sd::Tensor<float> AudioEncoder::get_audio_embedding(const sd::Audio& audio) {
+ if (audio_encoder == nullptr) {
+ LOG_ERROR("audio encoder model is not loaded");
+ return {};
+ }
+ if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) {
+ LOG_ERROR("invalid driving audio");
+ return {};
+ }
+ auto mono = sd::audio::downmix_to_mono(audio.data, audio.sample_count, audio.channels);
+ if (mono.empty()) {
+ LOG_ERROR("audio mono downmix failed");
+ return {};
+ }
+ mono = sd::audio::resample_audio(mono.data(), mono.size(), audio.sample_rate, 16000);
+ if (mono.empty()) {
+ LOG_ERROR("audio resample to 16 kHz failed");
+ return {};
+ }
+ return audio_encoder->compute(n_threads, mono);
+ }
+}
diff --git a/src/dgl/models/audio_encoder.h b/src/dgl/models/audio_encoder.h
new file mode 100644
index 0000000..beb5575
--- /dev/null
+++ b/src/dgl/models/audio_encoder.h
@@ -0,0 +1,30 @@
+#ifndef __SD_DGL_MODELS_AUDIO_ENCODER_H__
+#define __SD_DGL_MODELS_AUDIO_ENCODER_H__
+
+#include "core/media.h"
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+namespace sd::dgl::models {
+ class AudioEncoder final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ return runtime_.build_variant(static_cast<const AudioEncoder&>(source).runtime_, std::move(loader));
+ }
+
+ std::shared_ptr<Wav2Vec2::Wav2Vec2ModelRunner> audio_encoder;
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ bool init(ModelLoadParams params);
+ AudioEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ Tensor<float> get_audio_embedding(const sd::Audio& audio);
+ };
+}
+
+#endif // __SD_DGL_MODELS_AUDIO_ENCODER_H__
diff --git a/src/dgl/models/audio_vae.cpp b/src/dgl/models/audio_vae.cpp
new file mode 100644
index 0000000..c6dda8e
--- /dev/null
+++ b/src/dgl/models/audio_vae.cpp
@@ -0,0 +1,43 @@
+#include "dgl/models/audio_vae.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "dgl/models/component_builders.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/vae.h"
+#include "dgl/run_events.h"
+#include "model/vae/audio_geometry.hpp"
+#include "model/vae/audio_vae.hpp"
+
+namespace sd::dgl::models {
+ AudioVAE::AudioVAE(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::AudioVAE, SDBackendModule::VAE, [this] { return build_runners(); }) {}
+
+ bool AudioVAE::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::AudioVAE>);
+ }
+
+ bool AudioVAE::build_runners() {
+ audio_vae_model = component_builders::build_audio_vae_runner(runtime_.model_build_context());
+ if (!audio_vae_model) {
+ return false;
+ }
+ runtime_.configure_runner(*audio_vae_model, true);
+ return runtime_.register_runner_params(runtime_.component, audio_vae_model);
+ }
+
+ std::vector<int64_t> AudioVAE::latent_shape(int frames, int fps) const {
+ return sd::vae::audio_latent_shape(version(), frames, fps);
+ }
+
+ sd::Tensor<float> AudioVAE::decode_ltx_audio_latent(const sd::Tensor<float>& audio_latent) {
+ if (audio_vae_model == nullptr || audio_latent.empty()) {
+ return {};
+ }
+ report_run_progress("audio_decode");
+ auto waveform = audio_vae_model->decode(n_threads, audio_latent);
+ return waveform;
+ }
+}
diff --git a/src/dgl/models/audio_vae.h b/src/dgl/models/audio_vae.h
new file mode 100644
index 0000000..4b90cd5
--- /dev/null
+++ b/src/dgl/models/audio_vae.h
@@ -0,0 +1,31 @@
+#ifndef __SD_DGL_MODELS_AUDIO_VAE_H__
+#define __SD_DGL_MODELS_AUDIO_VAE_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+namespace sd::dgl::models {
+ class AudioVAE final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ return runtime_.build_variant(static_cast<const AudioVAE&>(source).runtime_, std::move(loader));
+ }
+
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ bool init(ModelLoadParams params);
+ std::shared_ptr<AudioVAERunner> audio_vae_model;
+
+ AudioVAE(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ std::vector<int64_t> latent_shape(int frames, int fps) const;
+ Tensor<float> decode_ltx_audio_latent(const Tensor<float>& latent);
+ };
+}
+
+#endif // __SD_DGL_MODELS_AUDIO_VAE_H__
diff --git a/src/dgl/models/circular_axes.h b/src/dgl/models/circular_axes.h
new file mode 100644
index 0000000..614a52a
--- /dev/null
+++ b/src/dgl/models/circular_axes.h
@@ -0,0 +1,30 @@
+#ifndef __SD_DGL_MODELS_CIRCULAR_AXES_H__
+#define __SD_DGL_MODELS_CIRCULAR_AXES_H__
+
+#include <functional>
+
+namespace sd::dgl::models {
+ class CircularAxesScope {
+ std::function<void()> restore_;
+
+ public:
+ template <class T>
+ CircularAxesScope(T& model, bool x, bool y) {
+ const bool old_x = model.circular_x;
+ const bool old_y = model.circular_y;
+ if (old_x != x || old_y != y) {
+ restore_ = [&model, old_x, old_y] { model.apply_circular_axes(old_x, old_y); };
+ model.apply_circular_axes(x, y);
+ }
+ }
+
+ ~CircularAxesScope() {
+ if (restore_)
+ restore_();
+ }
+ CircularAxesScope(const CircularAxesScope&) = delete;
+ CircularAxesScope& operator=(const CircularAxesScope&) = delete;
+ };
+}
+
+#endif // __SD_DGL_MODELS_CIRCULAR_AXES_H__
diff --git a/src/pipeline/model_builders.cpp b/src/dgl/models/component_builders.cpp
similarity index 50%
rename from src/pipeline/model_builders.cpp
rename to src/dgl/models/component_builders.cpp
index 389b199..b13e2da 100644
--- a/src/pipeline/model_builders.cpp
+++ b/src/dgl/models/component_builders.cpp
@@ -1,5 +1,6 @@
-#include "model_builders.h"
-#include "core/option_types.h"
+#include "dgl/models/component_builders.h"
+
+#include "model/vae/geometry.hpp"
#include <cmath>
#include <cstring>
@@ -8,9 +9,9 @@
#include "conditioning/conditioner.hpp"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
-#include "extensions/generation_extension.h"
-#include "model/adapter/ip_adapter.hpp"
-#include "model/audio/wav2vec2.hpp"
+#include "dgl/models/control.h"
+#include "dgl/models/relay_vsr.h"
+#include "dgl/models/vae.h"
#include "model/diffusion/control.hpp"
#include "model/diffusion/model.hpp"
#include "model/vae/auto_encoder_kl.hpp"
@@ -20,50 +21,20 @@
#include "model/vae/mage_vae.hpp"
#include "model/vae/minimax_h3_audio_vae.hpp"
#include "model/vae/minimax_h3_vae.hpp"
+#include "model/vae/seedvr2_vae.hpp"
#include "model/vae/tae.hpp"
#include "model/vae/vae.hpp"
#include "model/vae/wan_vae.hpp"
#include "model_definition.h"
-namespace sd::model_builders {
-
- bool ensure_backend_pair(SDBackendManager& backends, SDBackendModule module) {
- if (backends.runtime_backend(module) == nullptr) {
- LOG_ERROR("failed to initialize %s backend", sd_backend_module_name(module));
- return false;
- }
- if (backends.params_backend(module) == nullptr) {
- LOG_ERROR("failed to initialize %s params backend", sd_backend_module_name(module));
- return false;
- }
- return true;
- }
+namespace sd::dgl::models::component_builders {
- static SDVersion sd_vae_format_to_version(enum sd_vae_format_t format, SDVersion fallback) {
- switch (format) {
- case SD_VAE_FORMAT_FLUX:
- return VERSION_FLUX;
- case SD_VAE_FORMAT_SD3:
- return VERSION_SD3;
- case SD_VAE_FORMAT_FLUX2:
- return VERSION_FLUX2;
- case SD_VAE_FORMAT_WAN:
- return VERSION_WAN2;
- case SD_VAE_FORMAT_AUTO:
- default:
- return fallback;
- }
- }
-
- bool build_core_runners(const Context& ctx, CoreRunners& runners) try {
- const auto* sd_ctx_params = &ctx.params;
- const auto& tensor_storage_map = ctx.tensor_storage_map;
- const auto version = ctx.version;
- const auto& weight_manager = ctx.weight_manager;
+ bool build_family_runners(const Context& ctx, CoreRunners& runners, TokenizerConfig tokenizers) try {
+ const auto* sd_ctx_params = &ctx.params;
+ const auto version = ctx.version;
CoreRunners result;
- TokenizerConfig tokenizers(sd_ctx_params->tokenizer);
- if (!ensure_backend_pair(ctx.backends, SDBackendModule::TE) ||
- !ensure_backend_pair(ctx.backends, SDBackendModule::DIFFUSION)) {
+ if ((ctx.build_conditioner && !sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::TE)) ||
+ (ctx.build_diffusion && !sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::DIFFUSION))) {
return false;
}
@@ -73,21 +44,6 @@ namespace sd::model_builders {
return false;
}
- if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0 && result.clip_vision == nullptr) {
- if (!ensure_backend_pair(ctx.backends, SDBackendModule::CLIP_VISION)) {
- return false;
- }
- result.clip_vision = std::make_shared<FrozenCLIPVisionEmbedder>(ctx.backends.runtime_backend(SDBackendModule::CLIP_VISION),
- tensor_storage_map,
- weight_manager);
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) {
- result.ip_adapter = std::make_shared<IPAdapter::IPAdapterRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "ip_adapter",
- weight_manager);
- }
if (result.conditioner) {
result.conditioner->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
}
@@ -97,12 +53,6 @@ namespace sd::model_builders {
if (result.high_noise_diffusion) {
result.high_noise_diffusion->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
}
- if (result.clip_vision) {
- result.clip_vision->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
- }
- if (result.ip_adapter) {
- result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
- }
tokenizers.validate_usage();
runners = std::move(result);
return true;
@@ -111,35 +61,42 @@ namespace sd::model_builders {
return false;
}
- bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) try {
+ std::shared_ptr<AudioVAERunner> build_audio_vae_runner(const Context& ctx) {
+ if (!sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::VAE)) {
+ return nullptr;
+ }
+ const auto backend = ctx.backends.runtime_backend(SDBackendModule::VAE);
+ std::shared_ptr<AudioVAERunner> audio;
+ if (sd_version_is_minimax_h3(ctx.version)) {
+ audio = std::make_shared<MiniMaxH3::AudioVAERunner>(backend, ctx.tensor_storage_map, "", ctx.weight_manager);
+ } else {
+ audio = std::make_shared<LTXV::LTXAudioVAERunner>(backend, ctx.tensor_storage_map, "", ctx.weight_manager);
+ }
+ audio->set_scale_overrides(ctx.params.linear_scale, ctx.params.attn_scale);
+ return audio;
+ }
+
+ std::shared_ptr<::VAE> build_tae_runner(const Context& ctx, bool decode_only) {
+ const auto backend = ctx.backends.runtime_backend(SDBackendModule::VAE);
+ std::shared_ptr<::VAE> model;
+ if (find_model_traits(ctx.version)->video_tae) {
+ model = std::make_shared<TinyVideoAutoEncoder>(backend, ctx.tensor_storage_map, "decoder", decode_only,
+ ctx.version, ctx.weight_manager);
+ } else {
+ model = std::make_shared<TinyImageAutoEncoder>(backend, ctx.tensor_storage_map, "decoder.layers", decode_only,
+ ctx.version, ctx.weight_manager);
+ }
+ model->set_scale_overrides(ctx.params.linear_scale, ctx.params.attn_scale);
+ return model;
+ }
+
+ std::shared_ptr<::VAE> build_vae_runner(const Context& ctx, const ModelLoader& loader) try {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- VAERunners result;
- if (!ensure_backend_pair(ctx.backends, SDBackendModule::VAE)) {
- return false;
- }
-
- auto create_tae = [&](bool decode_only) -> std::shared_ptr<VAE> {
- if (find_model_traits(version)->video_tae) {
- return std::make_shared<TinyVideoAutoEncoder>(ctx.backends.runtime_backend(SDBackendModule::VAE),
- tensor_storage_map,
- "decoder",
- decode_only,
- version,
- weight_manager);
-
- } else {
- auto model = std::make_shared<TinyImageAutoEncoder>(ctx.backends.runtime_backend(SDBackendModule::VAE),
- tensor_storage_map,
- "decoder.layers",
- decode_only,
- version,
- weight_manager);
- return model;
- }
- };
+ if (!sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::VAE))
+ return {};
sd_vae_format_t vae_format = sd_ctx_params->vae_format;
if (vae_format < SD_VAE_FORMAT_AUTO || vae_format >= SD_VAE_FORMAT_COUNT) {
@@ -148,17 +105,24 @@ namespace sd::model_builders {
}
SDVersion vae_version = version;
if (find_model_traits(version)->vae_format_override && vae_format != SD_VAE_FORMAT_AUTO) {
- vae_version = sd_vae_format_to_version(vae_format, vae_version);
+ vae_version = sd::vae::version_for_format(vae_format, vae_version);
}
- auto create_vae = [&]() -> std::shared_ptr<VAE> {
- if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::LTXAV) {
+ auto create_vae = [&]() -> std::shared_ptr<::VAE> {
+ if (vae_version == VERSION_RELAY_VSR) {
+ return build_relay_vsr_vae(ctx, loader);
+ } else if (vae_version == VERSION_SEEDVR2) {
+ const char* prefix = tensor_storage_map.count("first_stage_model.encoder.conv_out.bias") ? "first_stage_model" : "vae";
+ return std::make_shared<SeedVR2::SeedVR2VAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
+ tensor_storage_map, weight_manager, prefix);
+ } else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::LTXAV) {
return std::make_shared<LTXVideoVAE>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
false,
version,
- weight_manager);
+ weight_manager,
+ false);
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::MINIMAX_H3) {
if (version == VERSION_MINIMAX_H3_IMAGE) {
auto proj = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight");
@@ -208,93 +172,45 @@ namespace sd::model_builders {
weight_manager);
const auto* definition = find_model_definition(version);
if (definition && definition->configure_vae) {
+ sd::model_builders::VAEOptions options;
definition->configure_vae(ctx, options, *model);
}
return model;
}
};
+ std::shared_ptr<::VAE> model;
if (find_model_traits(version)->fake_vae) {
LOG_INFO("using FakeVAE");
- result.vae = std::make_shared<FakeVAE>(version,
- ctx.backends.runtime_backend(SDBackendModule::VAE),
- weight_manager);
- } else if (options.use_tae && !options.tae_preview_only) {
- LOG_INFO("using TAE for encoding / decoding");
- result.vae = create_tae(false);
+ model = std::make_shared<FakeVAE>(version, ctx.backends.runtime_backend(SDBackendModule::VAE), weight_manager);
} else {
LOG_INFO("using VAE for encoding / decoding");
- result.vae = create_vae();
- if (options.use_tae && options.tae_preview_only) {
- LOG_INFO("using TAE for preview");
- result.preview = create_tae(true);
- }
- }
-
- if (options.use_audio_vae) {
- if (find_model_traits(version)->vae_kind == ModelVAEKind::MINIMAX_H3) {
- result.audio = std::make_shared<MiniMaxH3::AudioVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
- tensor_storage_map,
- "",
- weight_manager);
- } else {
- result.audio = std::make_shared<LTXV::LTXAudioVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
- tensor_storage_map,
- "",
- weight_manager);
- }
+ model = create_vae();
}
-
- if (result.vae) {
- result.vae->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
- }
- if (result.preview) {
- result.preview->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
- }
- if (result.audio) {
- result.audio->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
- }
- runners = std::move(result);
- return true;
+ if (model)
+ model->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
+ return model;
} catch (const std::exception& error) {
- LOG_ERROR("failed to build VAE runners: %s", error.what());
- return false;
+ LOG_ERROR("failed to build VAE runner: %s", error.what());
+ return {};
}
- bool build_control_net_runner(const Context& ctx, std::shared_ptr<ControlNet>& runner) {
+ bool build_control_net_runner(const Context& ctx, std::shared_ptr<::ControlNet>& runner) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- if (sd_version_is_z_image(version)) {
- return true;
- }
- if (!ensure_backend_pair(ctx.backends, SDBackendModule::CONTROL_NET)) {
+ if (!sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::CONTROL_NET)) {
return false;
}
- auto control_net = std::make_shared<ControlNet>(ctx.backends.runtime_backend(SDBackendModule::CONTROL_NET),
- tensor_storage_map,
- version,
- "",
- weight_manager);
+ auto control_net = std::make_shared<::ControlNet>(ctx.backends.runtime_backend(SDBackendModule::CONTROL_NET),
+ tensor_storage_map,
+ version,
+ "",
+ weight_manager);
control_net->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
runner = std::move(control_net);
return true;
}
- bool build_extension_runners(const GenerationExtensionInitContext& ctx,
- std::vector<std::shared_ptr<GenerationExtension>>& extensions) {
- std::vector<std::shared_ptr<GenerationExtension>> result;
- for (auto extension : {create_photomaker_extension(), create_pulid_extension()}) {
- if (!extension->init(ctx)) {
- return false;
- }
- if (extension->is_enabled()) {
- result.push_back(std::move(extension));
- }
- }
- extensions = std::move(result);
- return true;
- }
-
-} // namespace sd::model_builders
+} // namespace sd::dgl::models::component_builders
diff --git a/src/dgl/models/component_builders.h b/src/dgl/models/component_builders.h
new file mode 100644
index 0000000..f676bf8
--- /dev/null
+++ b/src/dgl/models/component_builders.h
@@ -0,0 +1,21 @@
+#ifndef __SD_DGL_MODELS_COMPONENT_BUILDERS_H__
+#define __SD_DGL_MODELS_COMPONENT_BUILDERS_H__
+
+#include "model_build_context.h"
+#include "tokenizers/tokenizer_config.h"
+
+class ModelLoader;
+
+namespace sd::dgl::models::component_builders {
+ using sd::model_builders::Context;
+ using sd::model_builders::CoreRunners;
+
+ bool build_family_runners(const Context& ctx, CoreRunners& runners, TokenizerConfig tokenizers = {});
+ std::shared_ptr<::VAE> build_vae_runner(const Context& ctx, const ModelLoader& loader);
+ std::shared_ptr<::VAE> build_tae_runner(const Context& ctx, bool decode_only);
+ std::shared_ptr<AudioVAERunner> build_audio_vae_runner(const Context& ctx);
+ bool build_control_net_runner(const Context& ctx, std::shared_ptr<::ControlNet>& runner);
+
+} // namespace sd::dgl::models::component_builders
+
+#endif // __SD_DGL_MODELS_COMPONENT_BUILDERS_H__
diff --git a/src/dgl/models/component_sources.cpp b/src/dgl/models/component_sources.cpp
new file mode 100644
index 0000000..bcc3466
--- /dev/null
+++ b/src/dgl/models/component_sources.cpp
@@ -0,0 +1,241 @@
+#include "dgl/models/component_sources.h"
+
+#include <algorithm>
+
+#include "core/file_access.h"
+#include "core/util.h"
+#include "dgl/models/audio_encoder.h"
+#include "dgl/models/audio_vae.h"
+#include "dgl/models/control.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/photomaker.h"
+#include "dgl/models/pulid.h"
+#include "dgl/models/vae.h"
+#include "model_loader.h"
+
+namespace sd::dgl::models {
+
+ SDVersion resolve_family(SDVersion family, const std::string& family_from, ModelLoader::Dependencies* dependencies, ModelLoader* metadata) {
+ if (family_from.empty())
+ return family;
+ if (family != VERSION_COUNT)
+ throw std::runtime_error("family_from requires family = \"auto\"");
+
+ ModelLoader probe;
+ ModelLoader::FileId file;
+ if (!probe.add_file(family_from, "", &file))
+ throw std::runtime_error("cannot read model family from '" + family_from + "'");
+ family = probe.get_sd_version();
+ if (family == VERSION_COUNT) {
+ probe.set_file_prefix(file, "model.diffusion_model.");
+ family = probe.get_sd_version();
+ }
+ if (family == VERSION_COUNT)
+ throw std::runtime_error("cannot identify model family from '" + family_from + "'");
+ if (dependencies)
+ *dependencies = probe.dependencies();
+ if (metadata)
+ *metadata = std::move(probe);
+ return family;
+ }
+
+ static std::string family_probe_key(const std::string& path) {
+ require_file_access(path, SD_DGL_FILE_READ_MODEL);
+ std::error_code error;
+ auto canonical = std::filesystem::weakly_canonical(std::filesystem::u8path(path), error);
+ return error ? path : canonical.u8string();
+ }
+
+ const FamilyProbeCache::Entry* FamilyProbeCache::find(const std::string& path) {
+ auto entry = entries_.find(path);
+ if (entry == entries_.end())
+ return nullptr;
+ bool changed = false;
+ if (ModelLoader::dependencies_changed(entry->second.dependencies, changed) && !changed)
+ return &entry->second;
+ entries_.erase(entry);
+ return nullptr;
+ }
+
+ void FamilyProbeCache::store(const std::string& path, Entry entry) {
+ bool changed = false;
+ if (!ModelLoader::dependencies_changed(entry.dependencies, changed) || changed)
+ throw std::runtime_error("model sources changed while detecting family from '" + path + "'");
+ const auto family = entry.family;
+ entries_.insert_or_assign(path, std::move(entry));
+ LOG_INFO("Detected model family: %s (%s)", sd_version_name(family), std::filesystem::u8path(path).filename().u8string().c_str());
+ }
+
+ SDVersion FamilyProbeCache::resolve(SDVersion family, const std::string& family_from, ModelLoader::Dependencies& dependencies) {
+ dependencies.clear();
+ if (family_from.empty())
+ return family;
+ if (family != VERSION_COUNT)
+ throw std::runtime_error("family_from requires family = \"auto\"");
+ const auto path = family_probe_key(family_from);
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (const auto* entry = find(path)) {
+ dependencies = entry->dependencies;
+ return entry->family;
+ }
+ family = resolve_family(family, path, &dependencies);
+ store(path, {family, dependencies});
+ return family;
+ }
+
+ void FamilyProbeCache::remember(const std::string& source, SDVersion family, const ModelLoader& loader) {
+ if (family == VERSION_COUNT)
+ return;
+ const auto path = family_probe_key(source);
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (const auto* entry = find(path); entry && entry->family == family)
+ return;
+ store(path, {family, loader.dependencies()});
+ }
+
+ bool init_component_loader(ModelLoader& loader, const ModelLoadParams& params, ModelComponent component, SDVersion& family, const WeightSources& sources, FamilyProbeCache* family_probes) {
+ const std::string path = params.path;
+ ModelLoader::FileId file;
+ if (!loader.add_file(path, "", &file)) {
+ return false;
+ }
+ const auto& weights = loader.get_tensor_storage_map();
+ auto has_prefix = [&](const char* prefix) {
+ return std::any_of(weights.begin(), weights.end(), [prefix](const auto& entry) {
+ return starts_with(entry.first, prefix);
+ });
+ };
+ const auto detected = loader.get_sd_version();
+ if (family == VERSION_COUNT)
+ family = detected;
+ std::string prefix;
+ switch (component) {
+ case ModelComponent::Diffusion:
+ if (!has_prefix("model.diffusion_model.") &&
+ !has_prefix("diffusion_model.") && !has_prefix("transformer.") && !has_prefix("unet.")) {
+ prefix = "model.diffusion_model.";
+ }
+ break;
+ case ModelComponent::VAE:
+ if (detected == VERSION_COUNT && !has_prefix("first_stage_model.") && !has_prefix("vae.")) {
+ prefix = "vae.";
+ }
+ if (family == VERSION_COUNT && detected == VERSION_COUNT) {
+ for (const auto* key : {"decoder.conv1.weight", "first_stage_model.decoder.conv1.weight", "vae.decoder.conv1.weight"}) {
+ auto conv = weights.find(key);
+ if (conv == weights.end() || conv->second.n_dims < 4) {
+ continue;
+ }
+ const auto channels = conv->second.ne[conv->second.n_dims - 2];
+ if (channels == 16 || channels == 48) {
+ family = channels == 48 ? VERSION_WAN2_2_TI2V : VERSION_WAN2;
+ break;
+ }
+ }
+ }
+ break;
+ case ModelComponent::Conditioner:
+ if (detected == VERSION_COUNT && !has_prefix("cond_stage_model.") && !has_prefix("conditioner.embedders.") &&
+ !has_prefix("text_encoders.") && !has_prefix("te.") && !has_prefix("clip_l.")) {
+ if (has_prefix("encoder.block.") || has_prefix("enc.blk.") || has_prefix("encoder.blk.")) {
+ prefix = "text_encoders.t5xxl.transformer.";
+ if (family == VERSION_COUNT && detected == VERSION_COUNT) {
+ family = VERSION_WAN2;
+ }
+ } else if (has_prefix("text_model.")) {
+ prefix = "clip_l.";
+ } else {
+ prefix = "text_encoders.llm.";
+ }
+ }
+ break;
+ case ModelComponent::ControlNet:
+ if (family == VERSION_COUNT && weights.count("control_all_x_embedder.2-1.weight"))
+ family = VERSION_Z_IMAGE;
+ break;
+ case ModelComponent::AudioVAE:
+ case ModelComponent::IPAdapter:
+ break;
+ case ModelComponent::LatentUpsampler: {
+ const auto looks_like_ltx = [&](const std::string& p) {
+ return weights.count(p + "initial_conv.weight") && weights.count(p + "initial_norm.weight");
+ };
+ const auto looks_like_h3 = [&](const std::string& p) {
+ return weights.count(p + "conv_in.weight") && weights.count(p + "embed.2.weight");
+ };
+ const bool ltx_prefixed = looks_like_ltx("ltx_latent_upsampler.");
+ const bool h3_prefixed = looks_like_h3("h3_latent_upscaler.");
+ const bool ltx = ltx_prefixed || looks_like_ltx("");
+ const bool h3 = h3_prefixed || looks_like_h3("");
+ if (ltx == h3)
+ throw std::runtime_error("cannot identify the latent upscaler checkpoint as LTX or MiniMax H3");
+ const auto detected_family = ltx ? VERSION_LTXAV : VERSION_MINIMAX_H3;
+ if (family != VERSION_COUNT && family != detected_family)
+ throw std::runtime_error("latent upscaler weights do not match the requested family");
+ family = detected_family;
+ if (!ltx_prefixed && !h3_prefixed)
+ prefix = ltx ? "ltx_latent_upsampler." : "h3_latent_upscaler.";
+ break;
+ }
+ case ModelComponent::PuLID:
+ if (!has_prefix("model.diffusion_model."))
+ prefix = "model.diffusion_model.";
+ if (family == VERSION_COUNT)
+ family = VERSION_FLUX;
+ break;
+ case ModelComponent::AnimateDiff:
+ if (has_prefix("motion_module."))
+ prefix = "model.diffusion_model.";
+ else if (!has_prefix("model.diffusion_model.motion_module."))
+ prefix = "model.diffusion_model.motion_module.";
+ if (family == VERSION_COUNT)
+ family = VERSION_SD1;
+ break;
+ case ModelComponent::PhotoMaker:
+ if (!has_prefix("pmid.")) {
+ prefix = "pmid.";
+ }
+ break;
+ case ModelComponent::CLIPVision:
+ if (!has_prefix("clip_vision.")) {
+ prefix = "clip_vision.";
+ }
+ if (family == VERSION_COUNT) {
+ family = VERSION_WAN2;
+ }
+ break;
+ case ModelComponent::AudioEncoder:
+ if (!has_prefix("wav2vec2.")) {
+ prefix = "wav2vec2.";
+ }
+ if (family == VERSION_COUNT) {
+ family = VERSION_WAN2_2_S2V;
+ }
+ break;
+ default:
+ return false;
+ }
+ if (!loader.set_file_prefix(file, prefix)) {
+ return false;
+ }
+ if (family_probes && params.family == VERSION_COUNT) {
+ const auto source_family = detected == VERSION_COUNT ? loader.get_sd_version() : detected;
+ family_probes->remember(path, source_family, loader);
+ }
+ for (const auto& [source, source_prefix] : sources) {
+ if (!source.empty() && !loader.init_from_file(source, source_prefix))
+ return false;
+ }
+ loader.convert_tensors_name(family);
+ if (family == VERSION_COUNT) {
+ family = loader.get_sd_version();
+ }
+ if (family == VERSION_COUNT) {
+ LOG_ERROR("cannot identify component family for '%s'; specify family", path.c_str());
+ return false;
+ }
+ loader.set_wtype_override(sd_type_to_ggml_type(params.wtype), params.tensor_type_rules.c_str());
+ return true;
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/component_sources.h b/src/dgl/models/component_sources.h
new file mode 100644
index 0000000..ba7518b
--- /dev/null
+++ b/src/dgl/models/component_sources.h
@@ -0,0 +1,46 @@
+#ifndef __SD_DGL_MODELS_COMPONENT_SOURCES_H__
+#define __SD_DGL_MODELS_COMPONENT_SOURCES_H__
+
+#include <mutex>
+#include <utility>
+#include <vector>
+
+#include "dgl/models/model_load_params.h"
+#include "model.h"
+#include "model_component.h"
+#include "model_loader.h"
+#include "stable-diffusion.h"
+
+class ModelLoader;
+
+namespace sd::dgl::models {
+ SDVersion resolve_family(SDVersion family, const std::string& family_from, ModelLoader::Dependencies* dependencies = nullptr, ModelLoader* metadata = nullptr);
+
+ class FamilyProbeCache {
+ struct Entry {
+ SDVersion family;
+ ModelLoader::Dependencies dependencies;
+ };
+
+ std::mutex mutex_;
+ std::map<std::string, Entry> entries_;
+
+ const Entry* find(const std::string& path);
+ void store(const std::string& path, Entry entry);
+
+ public:
+ SDVersion resolve(SDVersion family, const std::string& family_from, ModelLoader::Dependencies& dependencies);
+ void remember(const std::string& path, SDVersion family, const ModelLoader& loader);
+ };
+
+ using WeightSources = std::vector<std::pair<std::string, std::string>>;
+
+ bool init_component_loader(ModelLoader& loader, const ModelLoadParams& params, ModelComponent component, SDVersion& family, const WeightSources& sources = {}, FamilyProbeCache* family_probes = nullptr);
+
+ template <ModelComponent Component>
+ bool load_component_catalog(ModelLoader& loader, const ModelLoadParams& params, SDVersion& family, FamilyProbeCache& family_probes) {
+ return init_component_loader(loader, params, Component, family, {}, &family_probes);
+ }
+}
+
+#endif // __SD_DGL_MODELS_COMPONENT_SOURCES_H__
diff --git a/src/dgl/models/control.cpp b/src/dgl/models/control.cpp
new file mode 100644
index 0000000..701d9bd
--- /dev/null
+++ b/src/dgl/models/control.cpp
@@ -0,0 +1,108 @@
+#include "dgl/models/control.h"
+
+#include "conditioning/conditioner.hpp"
+#include "dgl/models/component_builders.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/dit.h"
+#include "model/diffusion/control.hpp"
+#include "model/diffusion/z_image.hpp"
+
+namespace sd::dgl::models {
+ ControlNet::ControlNet(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::ControlNet, SDBackendModule::CONTROL_NET, [this] { return build_runners(); }) {}
+
+ bool ControlNet::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::ControlNet>);
+ }
+
+ bool ControlNet::build_variant(const Model& source, ModelLoader loader) {
+ const auto& base = static_cast<const ControlNet&>(source);
+ return runtime_.build_variant(base.runtime_, std::move(loader));
+ }
+
+ bool ControlNet::build_runners() {
+ const auto& weights = model_weights->loader().get_tensor_storage_map();
+ if (sd_version_is_z_image(version())) {
+ z_image_control_ = std::make_shared<ZImage::ControlNetRunner>(runtime_.backend_for(), weights, model_weights);
+ z_image_control_->set_scale_overrides(runtime_.params.linear_scale,
+ runtime_.params.attn_scale);
+ runtime_.configure_runner(*z_image_control_);
+ return runtime_.register_runner_params(runtime_.component, z_image_control_);
+ }
+ if (!sd_version_is_unet(version()) || ZImage::ControlNet::has_weights(weights))
+ throw std::runtime_error("ControlNet weights do not match the requested family");
+ if (!component_builders::build_control_net_runner(runtime_.model_build_context(), control_net) || !control_net) {
+ return false;
+ }
+ runtime_.configure_runner(*control_net, true);
+ return runtime_.register_runner_params(runtime_.component, control_net);
+ }
+
+ void ControlNet::validate_hint(const Tensor<float>& hint, sd_control_type_t type) const {
+ if (type == SD_CONTROL_TYPE_COUNT)
+ throw std::runtime_error("invalid ControlNet type");
+ const auto channels = z_image_control_ ? ZImage::CONTROL_IN_CHANNELS : 3;
+ if (hint.empty() || hint.dim() != 4 || hint.shape()[2] != channels)
+ throw std::runtime_error(z_image_control_ ? "Z-Image ControlNet expects a 33-channel latent"
+ : "ControlNet expects an RGB float tensor in [width, height, channels, batch] order");
+ if (z_image_control_) {
+ if (type != SD_CONTROL_TYPE_NONE)
+ throw std::runtime_error("Z-Image ControlNet uses the type encoded in its hint");
+ } else if (!control_net->control_index(type)) {
+ throw std::runtime_error("invalid type for this ControlNet");
+ }
+ }
+
+ void ControlNet::validate_target(const DiT& target) const {
+ if (target.version() != version())
+ throw std::runtime_error("ControlNet family differs from the diffusion model");
+ if (z_image_control_) {
+ const auto* model = dynamic_cast<const ZImage::ZImageRunner*>(target.diffusion_model.get());
+ if (!model || !z_image_control_->model.compatible(model->config))
+ throw std::runtime_error("Z-Image ControlNet architecture differs from the diffusion model");
+ if (z_image_control_->compute_backend() != model->compute_backend())
+ throw std::runtime_error("Z-Image ControlNet and DiT require the same compute backend");
+ }
+ }
+
+ void ControlNet::apply_circular_axes(bool x, bool y) {
+ circular_x = x;
+ circular_y = y;
+ if (x || y)
+ LOG_DEBUG("Using circular padding for convolutions (x=%s, y=%s)", x ? "true" : "false", y ? "true" : "false");
+ if (control_net)
+ control_net->set_circular_axes(x, y);
+ }
+
+ std::vector<Tensor<float>> ControlNet::compute_sample_controls(const ControlBinding& binding,
+ const Tensor<float>& input,
+ const Tensor<float>& timesteps,
+ const SDCondition& condition) {
+ const auto& shape = binding.hint->shape();
+ if (input.dim() != 4 || shape[0] != input.shape()[0] * 8 || shape[1] != input.shape()[1] * 8 ||
+ (shape[3] != 1 && shape[3] != input.shape()[3]))
+ throw std::runtime_error("ControlNet hint size or batch does not match the diffusion latent");
+ const auto cache_key = "controlnet.guided_hint." + std::to_string(reinterpret_cast<uintptr_t>(binding.hint.get())) +
+ "." + std::to_string(circular_x) + "." + std::to_string(circular_y);
+ auto control_result = control_net->compute(n_threads,
+ input,
+ *binding.hint,
+ timesteps,
+ condition.c_crossattn,
+ condition.c_vector,
+ cache_key,
+ binding.type);
+ if (!control_result || control_result->empty())
+ throw std::runtime_error("ControlNet did not return residuals");
+ return std::move(*control_result);
+ }
+
+ ZImageControlInput ControlNet::z_image_input(const ControlBinding& binding, const Tensor<float>& input) const {
+ const auto& shape = binding.hint->shape();
+ if (input.dim() != 4 || shape[0] != input.shape()[0] || shape[1] != input.shape()[1] || shape[3] != input.shape()[3])
+ throw std::runtime_error("Z-Image ControlNet hint size or batch does not match the diffusion latent");
+ return {z_image_control_.get(), binding.hint.get(), binding.strength};
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/control.h b/src/dgl/models/control.h
new file mode 100644
index 0000000..e6d1a9e
--- /dev/null
+++ b/src/dgl/models/control.h
@@ -0,0 +1,62 @@
+#ifndef __SD_DGL_MODELS_CONTROL_H__
+#define __SD_DGL_MODELS_CONTROL_H__
+
+#include "core/option_types.h"
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+struct SDCondition;
+struct ZImageControlInput;
+namespace ZImage {
+ struct ControlNetRunner;
+}
+
+namespace sd::dgl::models {
+ class ControlNet;
+
+ struct ControlBinding {
+ std::shared_ptr<ControlNet> model;
+ std::shared_ptr<const Tensor<float>> hint;
+ sd_control_type_t type;
+ float strength;
+ float start_sigma;
+ float end_sigma;
+ };
+
+ struct ControlBindings {
+ std::vector<ControlBinding> entries;
+ };
+
+ struct UNetControls {
+ std::vector<Tensor<float>> residuals;
+ float strength = 1.f;
+ };
+
+ class ControlNet final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+
+ std::shared_ptr<::ControlNet> control_net;
+ std::shared_ptr<ZImage::ControlNetRunner> z_image_control_;
+
+ bool build_variant(const Model& base, ModelLoader loader) override;
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ bool circular_x = false;
+ bool circular_y = false;
+ ControlNet(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(ModelLoadParams params);
+ void validate_hint(const Tensor<float>& hint, sd_control_type_t type) const;
+ void validate_target(const DiT& target) const;
+ void apply_circular_axes(bool x, bool y);
+ std::vector<Tensor<float>> compute_sample_controls(const ControlBinding& binding, const Tensor<float>& input, const Tensor<float>& timesteps, const SDCondition& condition);
+ ZImageControlInput z_image_input(const ControlBinding& binding, const Tensor<float>& input) const;
+ };
+}
+
+#endif
diff --git a/src/dgl/models/denoising_parameterization.cpp b/src/dgl/models/denoising_parameterization.cpp
new file mode 100644
index 0000000..803ef4a
--- /dev/null
+++ b/src/dgl/models/denoising_parameterization.cpp
@@ -0,0 +1,216 @@
+#include "dgl/models/denoising_parameterization.h"
+
+#include <cmath>
+#include <stdexcept>
+
+#include "core/util.h"
+#include "model/diffusion/minimax_h3.hpp"
+#include "model_definition.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl::models {
+ static void calculate_alphas_cumprod(float* alphas_cumprod,
+ float linear_start = 0.00085f,
+ float linear_end = 0.0120f,
+ int timesteps = TIMESTEPS) {
+ float ls_sqrt = sqrtf(linear_start);
+ float le_sqrt = sqrtf(linear_end);
+ float amount = le_sqrt - ls_sqrt;
+ float product = 1.0f;
+ for (int i = 0; i < timesteps; i++) {
+ float beta = ls_sqrt + amount * ((float)i / (timesteps - 1));
+ product *= 1.0f - powf(beta, 2.0f);
+ alphas_cumprod[i] = product;
+ }
+ }
+
+ static void calculate_alphas_cumprod_linear_beta(float* alphas_cumprod,
+ float beta_start,
+ float beta_end,
+ int timesteps = TIMESTEPS) {
+ float product = 1.0f;
+ for (int i = 0; i < timesteps; i++) {
+ float beta = beta_start + (beta_end - beta_start) * ((float)i / (timesteps - 1));
+ product *= 1.0f - beta;
+ alphas_cumprod[i] = product;
+ }
+ }
+
+ DenoisingParameterization::DenoisingParameterization(SDVersion version, prediction_t prediction, const String2TensorStorage& tensors, const std::vector<float>& file_alphas)
+ : version_(version), prediction_(prediction) {
+ const auto* definition = find_model_definition(version);
+ if (!definition)
+ throw std::runtime_error("unknown diffusion model family");
+ const auto& policy = definition->sampling;
+ default_flow_shift_ = policy.flow_shift_from_weights ? policy.flow_shift_from_weights(tensors) : policy.default_flow_shift;
+ const auto set_denoiser = [&](auto instance) {
+ using T = typename decltype(instance)::element_type;
+ denoiser_ = std::move(instance);
+ copy_ = [](const Denoiser& source) -> std::shared_ptr<Denoiser> {
+ return std::make_shared<T>(static_cast<const T&>(source));
+ };
+ };
+ switch (prediction_) {
+ case EPS_PRED:
+ LOG_INFO("running in eps-prediction mode");
+ set_denoiser(std::make_shared<CompVisDenoiser>());
+ break;
+ case V_PRED:
+ LOG_INFO("running in v-prediction mode");
+ set_denoiser(std::make_shared<CompVisVDenoiser>());
+ break;
+ case EDM_V_PRED:
+ LOG_INFO("running in v-prediction EDM mode");
+ set_denoiser(std::make_shared<EDMVDenoiser>());
+ break;
+ case FLOW_PRED: {
+ LOG_INFO("running in FLOW mode");
+ set_denoiser(std::make_shared<DiscreteFlowDenoiser>());
+ break;
+ }
+ case FLUX_FLOW_PRED: {
+ LOG_INFO("running in Flux FLOW mode");
+ set_denoiser(std::make_shared<FluxFlowDenoiser>());
+ break;
+ }
+ case FLUX3_FLOW_PRED: {
+ LOG_INFO("running in Flux.3 FLOW mode");
+ set_denoiser(std::make_shared<Flux3FlowDenoiser>());
+ break;
+ }
+ case H3_AV_FLOW_PRED: {
+ if (version_ == VERSION_MINIMAX_H3_IMAGE) {
+ LOG_INFO("running in MiniMax H3 Image FLOW mode");
+ set_denoiser(std::make_shared<MiniMaxH3ImageDenoiser>(MiniMaxH3::pdd_sigmas));
+ break;
+ }
+ LOG_INFO("running in MiniMax H3 AV FLOW mode");
+ set_denoiser(std::make_shared<DiscreteFlowDenoiser>());
+ break;
+ }
+ case MING_IMAGE_FLOW_PRED: {
+ LOG_INFO("running in Ming-Image FLOW mode");
+ set_denoiser(std::make_shared<MingImageFlowDenoiser>());
+ break;
+ }
+ case SEFI_FLOW_PRED: {
+ LOG_INFO("running in SeFi-Image dual-time FLOW mode");
+ set_denoiser(std::make_shared<SefiFlowDenoiser>());
+ break;
+ }
+ case MINIT2I_FLOW_PRED: {
+ LOG_INFO("running in MiniT2I FLOW mode");
+ set_denoiser(std::make_shared<MiniT2IFlowDenoiser>());
+ break;
+ }
+ case SENSENOVA_U1_FLOW_PRED: {
+ LOG_INFO("running in SenseNova U1.5 FLOW mode");
+ set_denoiser(std::make_shared<SenseNovaU1FlowDenoiser>(default_flow_shift_));
+ break;
+ }
+ default: {
+ LOG_ERROR("Unknown predition type %i", prediction_);
+ throw std::runtime_error("unknown prediction type");
+ }
+ }
+
+ if (auto* flow = dynamic_cast<DiscreteFlowDenoiser*>(denoiser_.get()))
+ flow->set_shift(default_flow_shift_);
+
+ auto comp_vis_denoiser = std::dynamic_pointer_cast<CompVisDenoiser>(denoiser_);
+ if (!comp_vis_denoiser) {
+ return;
+ }
+ std::vector<float> alphas_cumprod(TIMESTEPS);
+ if (file_alphas.size() == TIMESTEPS) {
+ alphas_cumprod = file_alphas;
+ } else if (sd_version_is_pixart(version_)) {
+ // PixArt checkpoints train with a linear beta schedule (0.0001 -> 0.02)
+ // instead of the scaled_linear schedule used by SD1.x/SDXL.
+ calculate_alphas_cumprod_linear_beta(alphas_cumprod.data(), 0.0001f, 0.02f);
+ } else {
+ calculate_alphas_cumprod(alphas_cumprod.data());
+ }
+ for (int i = 0; i < TIMESTEPS; i++) {
+ comp_vis_denoiser->sigmas[i] = std::sqrt((1 - alphas_cumprod[i]) / alphas_cumprod[i]);
+ comp_vis_denoiser->log_sigmas[i] = std::log(comp_vis_denoiser->sigmas[i]);
+ }
+ }
+
+ std::shared_ptr<Denoiser> DenoisingParameterization::schedule(float flow_shift, int sequence_length) const {
+ auto schedule = copy_(*denoiser_);
+ if (auto* flow = dynamic_cast<DiscreteFlowDenoiser*>(schedule.get())) {
+ flow->set_shift(flow_shift == 0.f ? default_flow_shift_ : flow_shift);
+ }
+ if (auto* ming = dynamic_cast<MingImageFlowDenoiser*>(schedule.get()); ming && sequence_length > 0) {
+ ming->set_sequence_length(sequence_length);
+ }
+ return schedule;
+ }
+
+ float DenoisingParameterization::schedule_sigma(double t, float flow_shift, int sequence_length) const {
+ auto schedule = this->schedule(flow_shift, sequence_length);
+ if (dynamic_cast<MiniT2IFlowDenoiser*>(schedule.get()) || dynamic_cast<SenseNovaU1FlowDenoiser*>(schedule.get()))
+ return schedule->t_to_sigma(static_cast<float>(1.0 - t));
+ if (dynamic_cast<DiscreteFlowDenoiser*>(schedule.get()) && !dynamic_cast<MiniMaxH3ImageDenoiser*>(schedule.get()))
+ return t == 0.0 ? 0.f : schedule->t_to_sigma(static_cast<float>(t * TIMESTEPS - 1.0));
+ return schedule->t_to_sigma(static_cast<float>(t * (TIMESTEPS - 1)));
+ }
+
+ float DenoisingParameterization::timestep(float sigma) const {
+ float t = denoiser_->sigma_to_t(sigma);
+ if (sd_version_is_anima(version_)) {
+ return t / static_cast<float>(TIMESTEPS);
+ }
+ if (sd_version_is_boogu_image(version_)) {
+ return t / static_cast<float>(TIMESTEPS);
+ }
+ if (version_ == VERSION_HIDREAM_O1) {
+ return 1.0f - t / static_cast<float>(TIMESTEPS);
+ }
+ if (sd_version_is_z_image(version_) || sd_version_is_z_image_l2p(version_) || sd_version_is_ideogram4(version_) || version_ == VERSION_MING_IMAGE) {
+ return 1000.f - t;
+ }
+ return t;
+ }
+
+ bool DenoisingParameterization::is_flow() const {
+ return dynamic_cast<DiscreteFlowDenoiser*>(denoiser_.get()) != nullptr;
+ }
+
+ bool DenoisingParameterization::compatible_noise(const DenoisingParameterization& other) const {
+ for (const auto special : {MINIT2I_FLOW_PRED, SENSENOVA_U1_FLOW_PRED, SEFI_FLOW_PRED}) {
+ if (prediction_ == special || other.prediction_ == special)
+ return prediction_ == other.prediction_;
+ }
+ return is_flow() == other.is_flow();
+ }
+
+ bool DenoisingParameterization::compatible_forward(const DenoisingParameterization& other) const {
+ return version_ == other.version_ && prediction_ == other.prediction_;
+ }
+
+ bool DenoisingParameterization::compatible_scalings(const DenoisingParameterization& other) const {
+ const auto* left = dynamic_cast<const CompVisDenoiser*>(denoiser_.get());
+ const auto* right = dynamic_cast<const CompVisDenoiser*>(other.denoiser_.get());
+ if (left || right)
+ return left && right && left->sigma_data == right->sigma_data &&
+ bool(dynamic_cast<const CompVisVDenoiser*>(left)) == bool(dynamic_cast<const CompVisVDenoiser*>(right));
+ if (is_flow() && other.is_flow())
+ return bool(dynamic_cast<const SenseNovaU1FlowDenoiser*>(denoiser_.get())) ==
+ bool(dynamic_cast<const SenseNovaU1FlowDenoiser*>(other.denoiser_.get()));
+ return prediction_ == other.prediction_;
+ }
+
+ std::vector<float> DenoisingParameterization::scalings(float sigma) const {
+ return denoiser_->get_scalings(sigma);
+ }
+
+ Tensor<float> DenoisingParameterization::add_noise(float sigma, const Tensor<float>& noise, const Tensor<float>& base) const {
+ return denoiser_->noise_scaling(sigma, noise, base);
+ }
+
+ Tensor<float> DenoisingParameterization::finish(float sigma, const Tensor<float>& latent) const {
+ return denoiser_->inverse_noise_scaling(sigma, latent);
+ }
+}
diff --git a/src/dgl/models/denoising_parameterization.h b/src/dgl/models/denoising_parameterization.h
new file mode 100644
index 0000000..b661492
--- /dev/null
+++ b/src/dgl/models/denoising_parameterization.h
@@ -0,0 +1,38 @@
+#ifndef __SD_DGL_MODELS_DENOISING_PARAMETERIZATION_H__
+#define __SD_DGL_MODELS_DENOISING_PARAMETERIZATION_H__
+
+#include <memory>
+#include <vector>
+
+#include "core/option_types.h"
+#include "core/tensor.hpp"
+#include "model.h"
+
+struct Denoiser;
+
+namespace sd::dgl::models {
+ class DenoisingParameterization {
+ SDVersion version_;
+ prediction_t prediction_;
+ float default_flow_shift_;
+ std::shared_ptr<Denoiser> denoiser_;
+ std::shared_ptr<Denoiser> (*copy_)(const Denoiser&) = nullptr;
+
+ public:
+ DenoisingParameterization(SDVersion version, prediction_t prediction, const String2TensorStorage& tensors, const std::vector<float>& alphas_cumprod);
+ prediction_t prediction() const { return prediction_; }
+ Denoiser* native() const { return denoiser_.get(); }
+ bool is_flow() const;
+ bool compatible_noise(const DenoisingParameterization& other) const;
+ bool compatible_forward(const DenoisingParameterization& other) const;
+ bool compatible_scalings(const DenoisingParameterization& other) const;
+ float timestep(float sigma) const;
+ std::vector<float> scalings(float sigma) const;
+ std::shared_ptr<Denoiser> schedule(float flow_shift, int sequence_length = 0) const;
+ float schedule_sigma(double t, float flow_shift, int sequence_length) const;
+ Tensor<float> add_noise(float sigma, const Tensor<float>& noise, const Tensor<float>& base) const;
+ Tensor<float> finish(float sigma, const Tensor<float>& latent) const;
+ };
+}
+
+#endif // __SD_DGL_MODELS_DENOISING_PARAMETERIZATION_H__
diff --git a/src/dgl/models/detail_detector.cpp b/src/dgl/models/detail_detector.cpp
new file mode 100644
index 0000000..3968f97
--- /dev/null
+++ b/src/dgl/models/detail_detector.cpp
@@ -0,0 +1,378 @@
+#include "dgl/models/detail_detector.h"
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <utility>
+
+#include "core/image_geometry.h"
+#include "core/util.h"
+#include "json.hpp"
+#include "model/detector/yolov8.h"
+#include "model_loader.h"
+#include "model_manager.h"
+
+namespace sd::dgl::models {
+
+ struct LetterboxInput {
+ sd::Tensor<float> tensor;
+ float scale = 1.f;
+ int pad_x = 0;
+ int pad_y = 0;
+ };
+
+ static std::string trim_copy(const std::string& value) {
+ size_t first = value.find_first_not_of(" \t\r\n");
+ if (first == std::string::npos) {
+ return "";
+ }
+ size_t last = value.find_last_not_of(" \t\r\n");
+ return value.substr(first, last - first + 1);
+ }
+
+ static bool parse_int(const std::string& value, int* result) {
+ try {
+ size_t used = 0;
+ int parsed = std::stoi(trim_copy(value), &used);
+ if (used != trim_copy(value).size()) {
+ return false;
+ }
+ *result = parsed;
+ return true;
+ } catch (...) {
+ return false;
+ }
+ }
+
+ static float image_channel(const sd::Image& image, int x, int y, int channel) {
+ if (image.data == nullptr || image.width == 0 || image.height == 0 || image.channel == 0) {
+ return 0.f;
+ }
+ x = std::clamp(x, 0, static_cast<int>(image.width) - 1);
+ y = std::clamp(y, 0, static_cast<int>(image.height) - 1);
+ int source_channel = image.channel == 1 ? 0 : std::min(channel, static_cast<int>(image.channel) - 1);
+ size_t index = (static_cast<size_t>(y) * image.width + x) * image.channel + source_channel;
+ return image.data[index] / 255.f;
+ }
+
+ static float bilinear_channel(const sd::Image& image, float x, float y, int channel) {
+ int x0 = static_cast<int>(std::floor(x));
+ int y0 = static_cast<int>(std::floor(y));
+ int x1 = x0 + 1;
+ int y1 = y0 + 1;
+ float wx = x - x0;
+ float wy = y - y0;
+ float a = image_channel(image, x0, y0, channel) * (1.f - wx) + image_channel(image, x1, y0, channel) * wx;
+ float b = image_channel(image, x0, y1, channel) * (1.f - wx) + image_channel(image, x1, y1, channel) * wx;
+ return a * (1.f - wy) + b * wy;
+ }
+
+ static LetterboxInput make_letterbox_input(sd::Image image, int input_size) {
+ LetterboxInput result;
+ result.tensor = sd::full<float>({input_size, input_size, 3, 1}, 114.f / 255.f);
+ result.scale = std::min(static_cast<float>(input_size) / image.width,
+ static_cast<float>(input_size) / image.height);
+ int resized_width = std::max(1, static_cast<int>(std::round(image.width * result.scale)));
+ int resized_height = std::max(1, static_cast<int>(std::round(image.height * result.scale)));
+ result.pad_x = (input_size - resized_width) / 2;
+ result.pad_y = (input_size - resized_height) / 2;
+
+ for (int y = 0; y < resized_height; ++y) {
+ float source_y = (y + 0.5f) / result.scale - 0.5f;
+ for (int x = 0; x < resized_width; ++x) {
+ float source_x = (x + 0.5f) / result.scale - 0.5f;
+ for (int c = 0; c < 3; ++c) {
+ result.tensor.index(x + result.pad_x, y + result.pad_y, c, 0) = bilinear_channel(image, source_x, source_y, c);
+ }
+ }
+ }
+ return result;
+ }
+
+ static float logistic(float x) {
+ if (x >= 0.f) {
+ float z = std::exp(-x);
+ return 1.f / (1.f + z);
+ }
+ float z = std::exp(x);
+ return z / (1.f + z);
+ }
+
+ static float dfl_expectation(const float* values, int64_t anchor_count, int channel_offset, int reg_max, int64_t anchor) {
+ float maximum = -std::numeric_limits<float>::infinity();
+ for (int i = 0; i < reg_max; ++i) {
+ maximum = std::max(maximum, values[anchor + anchor_count * (channel_offset + i)]);
+ }
+ float denominator = 0.f;
+ float numerator = 0.f;
+ for (int i = 0; i < reg_max; ++i) {
+ float probability = std::exp(values[anchor + anchor_count * (channel_offset + i)] - maximum);
+ denominator += probability;
+ numerator += probability * i;
+ }
+ return denominator > 0.f ? numerator / denominator : 0.f;
+ }
+
+ static float box_iou(const DetailDetection& a, const DetailDetection& b) {
+ float x1 = std::max(a.x1, b.x1);
+ float y1 = std::max(a.y1, b.y1);
+ float x2 = std::min(a.x2, b.x2);
+ float y2 = std::min(a.y2, b.y2);
+ float intersection = std::max(0.f, x2 - x1) * std::max(0.f, y2 - y1);
+ float area_a = std::max(0.f, a.x2 - a.x1) * std::max(0.f, a.y2 - a.y1);
+ float area_b = std::max(0.f, b.x2 - b.x1) * std::max(0.f, b.y2 - b.y1);
+ float union_area = area_a + area_b - intersection;
+ return union_area > 0.f ? intersection / union_area : 0.f;
+ }
+
+ static std::vector<DetailDetection> decode_detections(const sd::Tensor<float>& raw,
+ const YOLOv8Config& config,
+ const LetterboxInput& letterbox,
+ sd::Image image,
+ const DetectionOptions& params) {
+ std::vector<DetailDetection> candidates;
+ if (raw.empty() || raw.dim() < 2) {
+ return candidates;
+ }
+ int64_t anchor_count = raw.shape()[0];
+ int64_t channels = raw.shape()[1];
+ if (channels != config.reg_max * 4 + config.num_classes) {
+ LOG_ERROR("unexpected YOLOv8 output channels: %lld", static_cast<long long>(channels));
+ return candidates;
+ }
+
+ const std::array<int, 3> strides = {8, 16, 32};
+ std::array<int64_t, 4> offsets = {0, 0, 0, 0};
+ for (int i = 0; i < 3; ++i) {
+ int grid = params.input_size / strides[i];
+ offsets[i + 1] = offsets[i] + static_cast<int64_t>(grid) * grid;
+ }
+ if (offsets[3] != anchor_count) {
+ LOG_ERROR("unexpected YOLOv8 anchor count: %lld (expected %lld)",
+ static_cast<long long>(anchor_count),
+ static_cast<long long>(offsets[3]));
+ return candidates;
+ }
+
+ const float* values = raw.data();
+ for (int scale_index = 0; scale_index < 3; ++scale_index) {
+ int stride = strides[scale_index];
+ int grid = params.input_size / stride;
+ for (int64_t anchor = offsets[scale_index]; anchor < offsets[scale_index + 1]; ++anchor) {
+ int64_t local = anchor - offsets[scale_index];
+ int grid_x = static_cast<int>(local % grid);
+ int grid_y = static_cast<int>(local / grid);
+
+ float confidence = 0.f;
+ int class_id = 0;
+ for (int c = 0; c < config.num_classes; ++c) {
+ float score = logistic(values[anchor + anchor_count * (config.reg_max * 4 + c)]);
+ if (score > confidence) {
+ confidence = score;
+ class_id = c;
+ }
+ }
+ if (confidence < params.confidence) {
+ continue;
+ }
+
+ float left = dfl_expectation(values, anchor_count, 0 * config.reg_max, config.reg_max, anchor);
+ float top = dfl_expectation(values, anchor_count, 1 * config.reg_max, config.reg_max, anchor);
+ float right = dfl_expectation(values, anchor_count, 2 * config.reg_max, config.reg_max, anchor);
+ float bottom = dfl_expectation(values, anchor_count, 3 * config.reg_max, config.reg_max, anchor);
+ float center_x = (grid_x + 0.5f) * stride;
+ float center_y = (grid_y + 0.5f) * stride;
+
+ DetailDetection detection;
+ detection.x1 = (center_x - left * stride - letterbox.pad_x) / letterbox.scale;
+ detection.y1 = (center_y - top * stride - letterbox.pad_y) / letterbox.scale;
+ detection.x2 = (center_x + right * stride - letterbox.pad_x) / letterbox.scale;
+ detection.y2 = (center_y + bottom * stride - letterbox.pad_y) / letterbox.scale;
+ detection.x1 = std::clamp(detection.x1, 0.f, static_cast<float>(image.width));
+ detection.y1 = std::clamp(detection.y1, 0.f, static_cast<float>(image.height));
+ detection.x2 = std::clamp(detection.x2, 0.f, static_cast<float>(image.width));
+ detection.y2 = std::clamp(detection.y2, 0.f, static_cast<float>(image.height));
+ detection.confidence = confidence;
+ detection.class_id = class_id;
+ if (detection.x2 > detection.x1 && detection.y2 > detection.y1) {
+ candidates.push_back(detection);
+ }
+ }
+ }
+
+ std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) {
+ return a.confidence > b.confidence;
+ });
+ std::vector<DetailDetection> selected;
+ for (const auto& candidate : candidates) {
+ bool suppressed = false;
+ for (const auto& kept : selected) {
+ if (candidate.class_id == kept.class_id && box_iou(candidate, kept) > params.nms_threshold) {
+ suppressed = true;
+ break;
+ }
+ }
+ if (!suppressed) {
+ selected.push_back(candidate);
+ if (static_cast<int>(selected.size()) >= params.max_detections) {
+ break;
+ }
+ }
+ }
+ return selected;
+ }
+
+ static std::vector<std::string> parse_class_names(const std::string& value, int class_count) {
+ std::vector<std::string> names(static_cast<size_t>(class_count));
+ try {
+ nlohmann::json parsed = nlohmann::json::parse(value);
+ if (parsed.is_array()) {
+ for (size_t i = 0; i < names.size() && i < parsed.size(); ++i) {
+ if (parsed[i].is_string()) {
+ names[i] = parsed[i].get<std::string>();
+ }
+ }
+ } else if (parsed.is_object()) {
+ for (const auto& item : parsed.items()) {
+ int class_id = 0;
+ if (parse_int(item.key(), &class_id) && class_id >= 0 && class_id < class_count &&
+ item.value().is_string()) {
+ names[static_cast<size_t>(class_id)] = item.value().get<std::string>();
+ }
+ }
+ }
+ } catch (const std::exception&) {
+ }
+ return names;
+ }
+
+ DetailDetector::DetailDetector(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)), backend_manager(std::move(backends)) {
+ params_placement_ = [this](const std::string& name) {
+ std::string error;
+ if (!backend_manager.init(params_.backend.c_str(), name.c_str(), nullptr, &error))
+ throw std::runtime_error(error);
+ params_.params_backend = name;
+ return true;
+ };
+ }
+
+ DetailDetector::~DetailDetector() {
+ if (detector) {
+ detector->runner_end();
+ set_weight_adapter(nullptr);
+ }
+ release_weights();
+ }
+
+ bool DetailDetector::init(DetailDetectorParams params) {
+ params_ = std::move(params);
+ ModelLoader loader;
+ if (!loader.init_from_file(params_.path)) {
+ LOG_ERROR("failed to load ADetailer detector: '%s'", params_.path.c_str());
+ return false;
+ }
+ return build(std::move(loader));
+ }
+
+ bool DetailDetector::build_variant(const Model& source, ModelLoader loader) {
+ params_ = static_cast<const DetailDetector&>(source).params_;
+ return build(std::move(loader));
+ }
+
+ bool DetailDetector::build(ModelLoader loader) {
+ std::string error;
+ if (!backend_manager.init(params_.backend.c_str(), params_.params_backend.c_str(), nullptr, &error)) {
+ LOG_ERROR("ADetailer backend config failed: %s", error.c_str());
+ return false;
+ }
+ ggml_backend_t backend = backend_manager.runtime_backend(SDBackendModule::DETECTOR);
+ ggml_backend_t params_backend = backend_manager.params_backend(SDBackendModule::DETECTOR);
+ if (backend == nullptr || params_backend == nullptr) {
+ LOG_ERROR("failed to initialize detector backend");
+ return false;
+ }
+
+ if (!model_weights->set_loader(std::move(loader)))
+ return false;
+ const auto& catalog = model_weights->loader();
+ detector = std::make_shared<YOLOv8Runner>(backend, catalog.get_tensor_storage_map(), model_weights);
+ if (!detector || !detector->model || !detector->config.valid) {
+ LOG_ERROR("unsupported YOLOv8 detection weights: '%s'", params_.path.c_str());
+ return false;
+ }
+
+ class_names.assign(static_cast<size_t>(detector->config.num_classes), "");
+ auto names_item = catalog.get_metadata().find("yolov8.names");
+ if (names_item != catalog.get_metadata().end()) {
+ class_names = parse_class_names(names_item->second, detector->config.num_classes);
+ }
+
+ std::map<std::string, ggml_tensor*> tensors;
+ detector->get_param_tensors(tensors);
+ if (!model_weights->register_param_tensors(ModelComponent::Detector,
+ std::move(tensors),
+ backend_manager.params_backend_is_disk(SDBackendModule::DETECTOR)
+ ? ModelManager::ResidencyMode::Disk
+ : ModelManager::ResidencyMode::ParamBackend,
+ backend,
+ params_backend) ||
+ !model_weights->validate_registered_tensors()) {
+ LOG_ERROR("failed to register YOLOv8 detector tensors");
+ return false;
+ }
+ return load_weights_eagerly();
+ }
+
+ bool DetailDetector::attach_loras() {
+ return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::DETECTOR,
+ n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+ }
+
+ void DetailDetector::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
+ detector->set_weight_adapter(adapter);
+ }
+
+ std::vector<DetailDetection> DetailDetector::predict(sd::Image image,
+ const DetectionOptions& options) {
+ auto params = options;
+ if (params.input_size <= 0 || params.input_size > 4096)
+ throw std::runtime_error("input_size must be between 1 and 4096");
+ params.input_size = int(sd::align_image_dimension(params.input_size, 32));
+ if (params.input_size != options.input_size)
+ LOG_VERBOSE("detector: aligning input_size from %d to %d", options.input_size, params.input_size);
+ Model::ExecutionScope scope(*this);
+ scope.require_ready();
+ LetterboxInput input = make_letterbox_input(image, params.input_size);
+ int64_t start = ggml_time_ms();
+ sd::Tensor<float> raw = detector->compute(n_threads, input.tensor);
+ detector->runner_end();
+ if (raw.empty()) {
+ LOG_ERROR("YOLOv8 detector inference failed");
+ return {};
+ }
+ auto detections = decode_detections(raw, detector->config, input, image, params);
+ LOG_INFO("ADetailer detected %zu object(s), taking %.2fs",
+ detections.size(),
+ (ggml_time_ms() - start) / 1000.f);
+ for (size_t i = 0; i < detections.size(); ++i) {
+ const auto& detection = detections[i];
+ std::string object = "class_" + std::to_string(detection.class_id);
+ if (detection.class_id >= 0 && static_cast<size_t>(detection.class_id) < class_names.size() &&
+ !class_names[static_cast<size_t>(detection.class_id)].empty()) {
+ object = class_names[static_cast<size_t>(detection.class_id)];
+ }
+ LOG_INFO("ADetailer detection %zu: object=%s, class_id=%d, confidence=%.3f, bbox=[x1=%.1f, y1=%.1f, x2=%.1f, y2=%.1f]",
+ i + 1,
+ object.c_str(),
+ detection.class_id,
+ detection.confidence,
+ detection.x1,
+ detection.y1,
+ detection.x2,
+ detection.y2);
+ }
+ return detections;
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/detail_detector.h b/src/dgl/models/detail_detector.h
new file mode 100644
index 0000000..5aa573d
--- /dev/null
+++ b/src/dgl/models/detail_detector.h
@@ -0,0 +1,61 @@
+#ifndef __SD_DGL_MODELS_DETAIL_DETECTOR_H__
+#define __SD_DGL_MODELS_DETAIL_DETECTOR_H__
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "config.h"
+#include "core/ggml_extend_backend.h"
+#include "dgl/models/model_base.h"
+#include "stable-diffusion.h"
+
+struct YOLOv8Runner;
+
+namespace sd::dgl::models {
+
+ struct DetailDetection {
+ float x1 = 0.f;
+ float y1 = 0.f;
+ float x2 = 0.f;
+ float y2 = 0.f;
+ float confidence = 0.f;
+ int class_id = 0;
+ };
+
+ struct DetectionOptions {
+ int input_size = 640;
+ float confidence = 0.3f;
+ float nms_threshold = 0.45f;
+ int max_detections = 100;
+ };
+
+ struct DetailDetectorParams {
+ std::string path;
+ std::string backend;
+ std::string params_backend;
+ };
+
+ struct DetailDetector final : public Model {
+ SDBackendManager backend_manager;
+ std::shared_ptr<YOLOv8Runner> detector;
+ std::vector<std::string> class_names;
+
+ DetailDetector(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ ~DetailDetector() override;
+
+ bool init(DetailDetectorParams params);
+ std::vector<DetailDetection> predict(sd::Image image,
+ const DetectionOptions& params);
+
+ private:
+ DetailDetectorParams params_;
+ bool build(ModelLoader loader);
+ bool build_variant(const Model& base, ModelLoader loader) override;
+ bool attach_loras() override;
+ void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override;
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_DETAIL_DETECTOR_H__
diff --git a/src/dgl/models/diffusion_model.h b/src/dgl/models/diffusion_model.h
new file mode 100644
index 0000000..92dc776
--- /dev/null
+++ b/src/dgl/models/diffusion_model.h
@@ -0,0 +1,18 @@
+#ifndef __SD_DGL_MODELS_DIFFUSION_MODEL_H__
+#define __SD_DGL_MODELS_DIFFUSION_MODEL_H__
+
+#include <memory>
+
+namespace sd::dgl::models {
+ class AnimateDiff;
+ class DiT;
+
+ struct DiffusionModel {
+ std::shared_ptr<DiT> dit;
+ std::shared_ptr<AnimateDiff> motion;
+ // Keep the original resource reachable through the context's weak cache.
+ std::shared_ptr<const DiffusionModel> source;
+ };
+}
+
+#endif // __SD_DGL_MODELS_DIFFUSION_MODEL_H__
diff --git a/src/dgl/models/diffusion_prediction.h b/src/dgl/models/diffusion_prediction.h
new file mode 100644
index 0000000..4a7cb78
--- /dev/null
+++ b/src/dgl/models/diffusion_prediction.h
@@ -0,0 +1,75 @@
+#ifndef __SD_DGL_MODELS_DIFFUSION_PREDICTION_H__
+#define __SD_DGL_MODELS_DIFFUSION_PREDICTION_H__
+
+#include <functional>
+#include <limits>
+
+#include "core/tensor.hpp"
+#include "model/common/rope_circular.hpp"
+
+struct AttentionProcessor;
+struct FluxImageBlockProcessor;
+struct UNetBlockProcessor;
+struct SDCondition;
+
+namespace sd_sample {
+ struct SampleStepCacheDispatcher;
+}
+
+namespace sd::dgl::models {
+
+ struct ControlBinding;
+ struct UNetControls;
+
+ struct DiffusionPrediction {
+ sd::Tensor<float> output;
+ float skip_scale = 0.f;
+ float output_scale = 0.f;
+ };
+
+ struct PredictionParameters {
+ float sigma, timestep, input_scale, skip_scale, output_scale;
+ };
+
+ struct DiffusionPredictionOptions {
+ sd::Tensor<float> vace_context, audio_embedding, video_positions;
+ std::vector<int> skip_layers;
+ float guidance = 3.5f;
+ float vace_strength = 1.f;
+ float fps = 24.f;
+ float video_sigma_shift = 12.f;
+ float audio_sigma_shift = 3.f;
+ int audio_length = 0;
+ Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED;
+ bool force_ref_timestep_zero = false;
+ std::function<PredictionParameters(PredictionParameters)> transform;
+ };
+
+ struct DiffusionPredictionStep {
+ const sd::Tensor<float>& base;
+ const sd::Tensor<float>& mask;
+ int index = 1;
+ size_t schedule_size = 0;
+ // Native caches use the signed solver marker to identify intermediate evaluations.
+ int solver_index = 1;
+ bool completes_step = true;
+ float target_sigma = std::numeric_limits<float>::quiet_NaN();
+ float lookahead_sigma = std::numeric_limits<float>::quiet_NaN();
+ float semantic_sigma = std::numeric_limits<float>::quiet_NaN();
+ bool mask_input = false;
+ sd_sample::SampleStepCacheDispatcher* cache = nullptr;
+ std::vector<const ControlBinding*> controls;
+ std::function<UNetControls(const Tensor<float>&, const Tensor<float>&, const SDCondition&)> compute_controls;
+ const AttentionProcessor* attn2_processor = nullptr;
+ const FluxImageBlockProcessor* flux_image_processor = nullptr;
+ const UNetBlockProcessor* unet_block_processor = nullptr;
+
+ void blend_mask(Tensor<float>& latent) const {
+ if (!mask.empty())
+ latent = latent * mask + base * (1.f - mask);
+ }
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_DIFFUSION_PREDICTION_H__
diff --git a/src/dgl/models/dit.cpp b/src/dgl/models/dit.cpp
new file mode 100644
index 0000000..9999f46
--- /dev/null
+++ b/src/dgl/models/dit.cpp
@@ -0,0 +1,419 @@
+#include "dgl/models/dit.h"
+
+#include <utility>
+
+#include "conditioning/conditioner.hpp"
+#include "core/ggml_tensor_utils.h"
+#include "core/util.h"
+#include "dgl/models/component_builders.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/control.h"
+#include "dgl/models/denoising_parameterization.h"
+#include "dgl/models/diffusion_prediction.h"
+#include "model/diffusion/animatediff.hpp"
+#include "model/diffusion/minimax_h3.hpp"
+#include "model/diffusion/z_image.hpp"
+#include "model_definition.h"
+#include "runtime/denoiser.hpp"
+#include "runtime/guidance.h"
+#include "runtime/sample-cache.h"
+
+namespace sd::dgl::models {
+
+ DiT::DiT(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::Diffusion, SDBackendModule::DIFFUSION, [this] { return build_runners(); }) {}
+
+ int DiT::get_latent_channel() const {
+ return find_model_definition(version())->geometry.latent_channels;
+ }
+
+ bool DiT::init(DiTLoadParams params) {
+ return runtime_.init(std::move(params.model),
+ [](ModelLoader& loader, const ModelLoadParams& model, SDVersion& family, FamilyProbeCache& family_probes) {
+ return init_component_loader(loader, model, ModelComponent::Diffusion, family, {}, &family_probes);
+ });
+ }
+
+ bool DiT::build_variant(const Model& source, ModelLoader loader) {
+ const auto& base = static_cast<const DiT&>(source);
+ default_prediction_ = base.default_prediction_;
+ file_alphas_cumprod = base.file_alphas_cumprod;
+ reuse_file_alphas_ = true;
+ return runtime_.build_variant(base.runtime_, std::move(loader));
+ }
+
+ bool DiT::build_runners() {
+ parameterizations_.clear();
+ if (!std::exchange(reuse_file_alphas_, false))
+ load_alphas_cumprod();
+ auto context = runtime_.model_build_context();
+ context.build_conditioner = false;
+ context.build_diffusion = true;
+ context.build_auxiliary = false;
+ sd::model_builders::CoreRunners runners;
+ if (!component_builders::build_family_runners(context, runners) || !runners.diffusion) {
+ return false;
+ }
+ diffusion_model = std::move(runners.diffusion);
+ high_noise_diffusion_model = std::move(runners.high_noise_diffusion);
+ runtime_.configure_runner(*diffusion_model, version() != VERSION_SEEDVR2);
+ if (high_noise_diffusion_model) {
+ runtime_.configure_runner(*high_noise_diffusion_model, true);
+ }
+ if (runtime_.params.sage_attn && !set_sage_attention_enabled(true)) {
+ return false;
+ }
+ return runtime_.register_runner_params(runtime_.component, diffusion_model) &&
+ runtime_.register_runner_params(ModelComponent::HighNoiseDiffusion, high_noise_diffusion_model);
+ }
+
+ void DiT::apply_circular_axes(bool x, bool y) {
+ circular_x = x;
+ circular_y = y;
+ if (x || y)
+ LOG_DEBUG("Using circular padding for convolutions (x=%s, y=%s)", x ? "true" : "false", y ? "true" : "false");
+ diffusion_model->set_circular_axes(x, y);
+ if (high_noise_diffusion_model) {
+ high_noise_diffusion_model->set_circular_axes(x, y);
+ }
+ }
+
+ static sd::Tensor<float> batch_two_condition_tensors(const sd::Tensor<float>& a, const sd::Tensor<float>& b) {
+ if (a.empty() || b.empty() || a.dim() != b.dim()) {
+ return {};
+ }
+ if (a.dim() == 1) {
+ if (a.shape() != b.shape()) {
+ return {};
+ }
+ auto batched = sd::ops::concat(a, b, 0);
+ batched.reshape_({a.shape()[0], 2});
+ return batched;
+ }
+ const int64_t batch_dim = a.dim() - 1;
+ for (int64_t d = 0; d < batch_dim; d++) {
+ if (a.shape()[d] != b.shape()[d]) {
+ return {};
+ }
+ }
+ return sd::ops::concat(a, b, static_cast<size_t>(batch_dim));
+ }
+
+ void DiT::load_alphas_cumprod() {
+ file_alphas_cumprod.clear();
+
+ std::vector<float> loaded_alphas;
+ if (!model_weights->load_float_tensor("alphas_cumprod", loaded_alphas)) {
+ return;
+ }
+ if (loaded_alphas.size() != TIMESTEPS) {
+ LOG_WARN("ignore alphas_cumprod from model file: expected %d values, got %zu",
+ TIMESTEPS,
+ loaded_alphas.size());
+ return;
+ }
+ for (float alpha : loaded_alphas) {
+ if (!std::isfinite(alpha) || alpha <= 0.0f || alpha > 1.0f) {
+ LOG_WARN("ignore invalid alphas_cumprod from model file");
+ return;
+ }
+ }
+
+ file_alphas_cumprod = std::move(loaded_alphas);
+ LOG_VERBOSE("loaded alphas_cumprod from model file");
+ }
+
+ bool DiT::set_sage_attention_enabled(bool enabled) {
+ if (!diffusion_model) {
+ return false;
+ }
+ if (enabled) {
+#ifndef SD_USE_UPSTREAM_GGML
+ auto* ctx = ggml_init({4 * ggml_tensor_overhead(), nullptr, true});
+ if (ctx == nullptr) {
+ return false;
+ }
+ auto* q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 128, 128, 1, 1);
+ auto* k = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 128, 128, 1, 1);
+ auto* v = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 128, 128, 1, 1);
+ GGMLRunnerContext probe_ctx;
+ probe_ctx.ggml_ctx = ctx;
+ auto* op = ggml_ext_sage_attn(&probe_ctx, q, k, v, 1.f / sqrtf(128.f), GGML_SAGE_ATTN_AUTO);
+ bool supported = true;
+ for (auto backend : runtime_.backend_manager.runtime_backends(SDBackendModule::DIFFUSION)) {
+ if (!ggml_backend_supports_op(backend, op)) {
+ LOG_ERROR("SageAttention is unavailable on %s; it requires patched GGML, CUDA Toolkit 12.0 or newer, and SM80 or newer kernels",
+ ggml_backend_name(backend));
+ supported = false;
+ }
+ }
+ ggml_free(ctx);
+ if (!supported) {
+ return false;
+ }
+#else
+ LOG_ERROR("SageAttention requires -DSD_USE_UPSTREAM_GGML=OFF and a CUDA backend");
+ return false;
+#endif
+ }
+ diffusion_model->set_sage_attention_enabled(enabled);
+ if (high_noise_diffusion_model) {
+ high_noise_diffusion_model->set_sage_attention_enabled(enabled);
+ }
+ if (enabled) {
+ LOG_INFO("Using SageAttention in the diffusion model; CUDA selects the supported kernel, unsupported layers use flash/default attention");
+ }
+ return true;
+ }
+
+ std::shared_ptr<const DenoisingParameterization> DiT::make_parameterization(prediction_t prediction) {
+ std::lock_guard<std::recursive_mutex> lock(prediction_mutex_);
+ if (prediction == PREDICTION_COUNT) {
+ if (default_prediction_ == PREDICTION_COUNT) {
+ const auto& policy = find_model_definition(version())->sampling;
+ auto resolved = policy.default_prediction;
+ if (policy.probe_v_prediction) {
+ Model::ExecutionScope scope(*this);
+ scope.require_ready();
+ resolved = is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version())) ? V_PRED : EPS_PRED;
+ } else if (policy.prediction_from_weights) {
+ resolved = policy.prediction_from_weights(model_weights->loader().get_tensor_storage_map());
+ }
+ default_prediction_ = resolved;
+ }
+ prediction = default_prediction_;
+ }
+ auto found = parameterizations_.find(prediction);
+ if (found == parameterizations_.end()) {
+ auto value = std::make_shared<const DenoisingParameterization>(version(), prediction, model_weights->loader().get_tensor_storage_map(), file_alphas_cumprod);
+ found = parameterizations_.emplace(prediction, std::move(value)).first;
+ }
+ return found->second;
+ }
+
+ bool DiT::is_using_v_parameterization_for_sd2(bool is_inpaint) {
+ struct RunnerEndOnExit {
+ GGMLRunner* runner = nullptr;
+ ~RunnerEndOnExit() {
+ if (runner != nullptr) {
+ runner->runner_end();
+ }
+ }
+ };
+ RunnerEndOnExit diffusion_runner_end{diffusion_model.get()};
+
+ sd::Tensor<float> x_t = sd::full<float>({8, 8, 4, 1}, 0.5f);
+ sd::Tensor<float> c = sd::full<float>({1024, 2, 1, 1}, 0.5f);
+ sd::Tensor<float> steps = sd::full<float>({1}, 999.0f);
+ sd::Tensor<float> concat;
+ if (is_inpaint) {
+ concat = sd::zeros<float>({8, 8, 5, 1});
+ }
+
+ int64_t t0 = ggml_time_ms();
+ sd::Tensor<float> out;
+ DiffusionParams diffusion_params;
+ diffusion_params.x = &x_t;
+ diffusion_params.timesteps = &steps;
+ diffusion_params.context = &c;
+ diffusion_params.extra = UNetDiffusionExtra{};
+ if (!concat.empty()) {
+ diffusion_params.c_concat = &concat;
+ }
+ auto out_opt = diffusion_model->compute(n_threads, diffusion_params);
+ GGML_ASSERT(!out_opt.empty());
+ out = std::move(out_opt);
+
+ double result = static_cast<double>((out - x_t).mean());
+ int64_t t1 = ggml_time_ms();
+ LOG_VERBOSE("check is_using_v_parameterization_for_sd2, taking %.2fs", (t1 - t0) * 1.0f / 1000);
+ return result < -1;
+ }
+
+ std::vector<float> DiT::process_timesteps(const std::vector<float>& timesteps,
+ const sd::Tensor<float>& init_latent,
+ const sd::Tensor<float>& denoise_mask) {
+ if (diffusion_model->get_desc() == "Wan2.2-TI2V-5B") {
+ int64_t frame_count = init_latent.shape()[2];
+ auto new_timesteps = std::vector<float>(static_cast<size_t>(frame_count), timesteps[0]);
+
+ if (!denoise_mask.empty() && denoise_mask.dim() >= 4 && denoise_mask.shape()[2] == frame_count) {
+ for (int64_t frame = 0; frame < frame_count; ++frame) {
+ float value = denoise_mask.dim() == 5 ? denoise_mask.index(0, 0, frame, 0, 0) : denoise_mask.index(0, 0, frame, 0);
+ if (value == 0.f) {
+ new_timesteps[static_cast<size_t>(frame)] = 0.f;
+ }
+ }
+ }
+ return new_timesteps;
+ }
+ if (diffusion_model->get_desc() == "Wan2.2-S2V-14B") {
+ int64_t frame_count = init_latent.shape()[2];
+ return std::vector<float>(static_cast<size_t>(frame_count), timesteps[0]);
+ } else {
+ return timesteps;
+ }
+ }
+
+ std::vector<float> DiT::process_ltxav_video_timesteps(const std::vector<float>& timesteps,
+ const sd::Tensor<float>& init_latent,
+ const sd::Tensor<float>& denoise_mask) {
+ if (timesteps.empty() || denoise_mask.empty() || init_latent.dim() < 4 || denoise_mask.dim() < 4) {
+ return timesteps;
+ }
+
+ int64_t width = init_latent.shape()[0];
+ int64_t height = init_latent.shape()[1];
+ int64_t frames = init_latent.shape()[2];
+ if (denoise_mask.shape()[0] != width ||
+ denoise_mask.shape()[1] != height ||
+ denoise_mask.shape()[2] != frames ||
+ denoise_mask.shape()[3] < 1) {
+ LOG_WARN("unexpected LTXAV denoise mask shape for timestep processing");
+ return timesteps;
+ }
+
+ std::vector<float> video_timesteps(static_cast<size_t>(width * height * frames));
+ size_t idx = 0;
+ for (int64_t t = 0; t < frames; ++t) {
+ for (int64_t h = 0; h < height; ++h) {
+ for (int64_t w = 0; w < width; ++w) {
+ float mask = denoise_mask.dim() == 5 ? denoise_mask.index(w, h, t, 0, 0)
+ : denoise_mask.index(w, h, t, 0);
+ video_timesteps[idx++] = mask * timesteps[0];
+ }
+ }
+ }
+ return video_timesteps;
+ }
+
+ int DiT::get_diffusion_model_down_factor() {
+ return find_model_definition(version())->geometry.diffusion_down_factor;
+ }
+
+ DiffusionPrediction DiT::predict_diffusion(const sd::Tensor<float>& x,
+ const DenoisingParameterization& parameterization,
+ float sigma,
+ const SDCondition& condition,
+ const std::vector<sd::Tensor<float>>& references,
+ const DiffusionPredictionOptions& options,
+ const DiffusionPredictionStep& step) {
+ const auto& shape = x.shape();
+ const auto channel_axis = shape.size() == 5 ? 3 : 2;
+ const auto down_factor = get_diffusion_model_down_factor();
+ const bool packed = sd_version_is_ltxav(version()) || sd_version_is_minimax_h3(version());
+ const bool unet_batch = shape.size() == 4 && sd_version_is_unet(version());
+ if ((shape.size() != 4 && shape.size() != 5) ||
+ (packed ? shape[channel_axis] < get_latent_channel() : shape[channel_axis] != get_latent_channel()) ||
+ (!unet_batch && shape.back() != 1) || shape[0] % down_factor || shape[1] % down_factor) {
+ throw std::runtime_error("latent shape does not match the diffusion model");
+ }
+ if (!condition.c_concat.empty()) {
+ auto concat_shape = condition.c_concat.shape();
+ if (concat_shape.size() == shape.size()) {
+ concat_shape[channel_axis] = shape[channel_axis];
+ }
+ if (concat_shape != shape) {
+ throw std::runtime_error("concat conditioning must match the latent outside its channel axis");
+ }
+ }
+
+ auto scaling = parameterization.scalings(sigma);
+ const bool has_semantic_time = std::isfinite(step.semantic_sigma);
+ if (parameterization.prediction() == SEFI_FLOW_PRED && !has_semantic_time)
+ throw std::runtime_error("SeFi predictions require both noise times; use sefi::sample");
+ auto base_times = has_semantic_time ? std::vector<float>{sigma * TIMESTEPS} : std::vector<float>{parameterization.timestep(sigma)};
+ if (options.transform) {
+ const auto transformed = options.transform({sigma, base_times.front(), scaling[2], scaling[0], scaling[1]});
+ base_times.front() = transformed.timestep;
+ scaling[2] = transformed.input_scale;
+ scaling[0] = transformed.skip_scale;
+ scaling[1] = transformed.output_scale;
+ }
+ const bool separate_audio_time = sd_version_is_ltxav(version()) && !step.mask.empty();
+ auto times = has_semantic_time
+ ? std::vector<float>{step.semantic_sigma * TIMESTEPS, base_times.front()}
+ : separate_audio_time ? process_ltxav_video_timesteps(base_times, step.base, step.mask)
+ : process_timesteps(base_times, step.base, step.mask);
+ auto input = x * scaling[2];
+ if (step.mask_input)
+ step.blend_mask(input);
+ sd::Tensor<float> cached;
+ if (step.cache && step.cache->before_condition(&condition, input, &cached)) {
+ return {std::move(cached), scaling[0], scaling[1]};
+ }
+ auto timesteps = sd::Tensor<float>::from_vector(std::move(times));
+ auto audio_time = separate_audio_time ? sd::Tensor<float>::from_vector(base_times) : sd::Tensor<float>{};
+ auto target_time = std::isfinite(step.target_sigma) ? sd::Tensor<float>::from_vector({step.target_sigma}) : sd::Tensor<float>{};
+ auto lookahead_time = std::isfinite(step.lookahead_sigma) ? sd::Tensor<float>::from_vector({step.lookahead_sigma}) : sd::Tensor<float>{};
+ auto guidance = sd::Tensor<float>::from_vector({options.guidance});
+ auto present = [](const auto& tensor) { return tensor.empty() ? nullptr : &tensor; };
+ auto first = [](const auto& tensors) { return tensors.empty() ? nullptr : &tensors.front(); };
+
+ DiffusionParams params;
+ params.x = &input;
+ params.timesteps = ×teps;
+ params.context = present(condition.c_crossattn);
+ params.y = present(condition.c_vector);
+ params.c_concat = present(condition.c_concat);
+ params.ref_latents = references.empty() ? &condition.c_ref_images : &references;
+ params.ref_index_mode = options.ref_index_mode;
+ params.force_ref_timestep_zero = options.force_ref_timestep_zero;
+ UNetControls unet_controls;
+ std::vector<ZImageControlInput> z_image_controls;
+ if (sd_version_is_unet(version())) {
+ if (step.compute_controls)
+ unet_controls = step.compute_controls(input, timesteps, condition);
+ params.extra = UNetDiffusionExtra{-1, &unet_controls.residuals, unet_controls.strength,
+ nullptr, 1.f, step.attn2_processor, step.unet_block_processor};
+ } else if (sd_version_is_sd3(version()) || version() == VERSION_MOSS_IMAGE) {
+ params.extra = SkipLayerDiffusionExtra{present(options.skip_layers)};
+ } else if (sd_version_is_flux(version()) || sd_version_is_flux2(version()) || sd_version_is_longcat(version()) || sd_version_is_sefi_image(version())) {
+ params.extra = FluxDiffusionExtra{&guidance, present(options.skip_layers), nullptr, 1.f, step.flux_image_processor};
+ } else if (version() == VERSION_FLUX3) {
+ params.extra = Flux3DiffusionExtra{present(target_time)};
+ } else if (version() == VERSION_QWEN_IMAGE_2_1) {
+ params.extra = QwenImage21DiffusionExtra{&condition.c_token_types};
+ } else if (sd_version_is_anima(version())) {
+ params.extra = AnimaDiffusionExtra{present(condition.c_t5_ids), present(condition.c_t5_weights)};
+ } else if (sd_version_is_wan(version())) {
+ params.extra = WanDiffusionExtra{present(options.vace_context), options.vace_strength, present(options.audio_embedding)};
+ } else if (sd_version_is_hunyuan_video(version())) {
+ params.extra = HunyuanVideoDiffusionExtra{&guidance, first(condition.extra_c_crossattns), params.y, present(lookahead_time)};
+ } else if (version() == VERSION_HIDREAM_O1) {
+ params.extra = HiDreamO1DiffusionExtra{present(condition.c_input_ids), present(condition.c_position_ids),
+ present(condition.c_token_types), present(condition.c_vinput_mask),
+ present(condition.c_image_embeds)};
+ } else if (version() == VERSION_MING_IMAGE) {
+ params.extra = MingImageDiffusionExtra{first(condition.extra_c_crossattns)};
+ } else if (sd_version_is_llada_image(version())) {
+ params.extra = LLaDAImageDiffusionExtra{first(condition.extra_c_crossattns)};
+ } else if (sd_version_is_minimax_h3(version())) {
+ params.extra = MiniMaxH3DiffusionExtra{present(condition.c_token_types), present(condition.c_position_ids),
+ present(condition.c_ref_audios), present(condition.c_reference_blocks),
+ options.audio_length, options.video_sigma_shift, options.audio_sigma_shift, step.target_sigma};
+ } else if (sd_version_is_ltxav(version())) {
+ params.extra = LTXAVDiffusionExtra{nullptr, present(audio_time), options.audio_length, options.fps, present(options.video_positions), first(condition.extra_c_crossattns)};
+ } else if (sd_version_is_minit2i(version())) {
+ params.extra = MiniT2IDiffusionExtra{params.y};
+ } else if (sd_version_is_sensenova_u1(version())) {
+ params.extra = SenseNovaU1DiffusionExtra{present(condition.c_input_ids)};
+ } else if (sd_version_is_z_image(version()) && !step.controls.empty()) {
+ z_image_controls.reserve(step.controls.size());
+ for (const auto* binding : step.controls)
+ z_image_controls.push_back(binding->model->z_image_input(*binding, input));
+ params.extra = ZImageDiffusionExtra{nullptr, 1.f, &z_image_controls};
+ }
+ auto result = diffusion_model->compute(n_threads, params);
+ if (result.shape() != shape) {
+ throw std::runtime_error("diffusion output shape " + sd::tensor_shape_to_string(result.shape()) +
+ " does not match input " + sd::tensor_shape_to_string(shape));
+ }
+ if (step.cache) {
+ step.cache->after_condition(&condition, input, result);
+ }
+ return {std::move(result), scaling[0], scaling[1]};
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/dit.h b/src/dgl/models/dit.h
new file mode 100644
index 0000000..5060161
--- /dev/null
+++ b/src/dgl/models/dit.h
@@ -0,0 +1,58 @@
+#ifndef __SD_DGL_MODELS_DIT_H__
+#define __SD_DGL_MODELS_DIT_H__
+
+#include <cmath>
+#include <map>
+#include <mutex>
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+struct SDCondition;
+
+namespace sd::dgl::models {
+ class DenoisingParameterization;
+
+ struct DiTLoadParams {
+ ModelLoadParams model;
+ };
+
+ class DiT final : public Model {
+ ModelRuntime runtime_;
+ prediction_t default_prediction_ = PREDICTION_COUNT;
+ std::recursive_mutex prediction_mutex_;
+ std::map<prediction_t, std::shared_ptr<const DenoisingParameterization>> parameterizations_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+
+ std::shared_ptr<DiffusionModelRunner> high_noise_diffusion_model;
+ std::vector<float> file_alphas_cumprod;
+ bool reuse_file_alphas_ = false;
+ bool build_variant(const Model& base, ModelLoader loader) override;
+ bool build_runners();
+ bool is_using_v_parameterization_for_sd2(bool is_inpaint);
+ bool set_sage_attention_enabled(bool enabled);
+ void load_alphas_cumprod();
+ int get_diffusion_model_down_factor();
+ std::vector<float> process_timesteps(const std::vector<float>& timesteps, const Tensor<float>& init_latent, const Tensor<float>& denoise_mask);
+ std::vector<float> process_ltxav_video_timesteps(const std::vector<float>& timesteps,
+ const Tensor<float>& init_latent,
+ const Tensor<float>& denoise_mask);
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ int get_latent_channel() const;
+ bool circular_x = false;
+ bool circular_y = false;
+ std::shared_ptr<DiffusionModelRunner> diffusion_model;
+
+ DiT(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(DiTLoadParams params);
+ void apply_circular_axes(bool x, bool y);
+ std::shared_ptr<const DenoisingParameterization> make_parameterization(prediction_t prediction = PREDICTION_COUNT);
+ DiffusionPrediction predict_diffusion(const Tensor<float>& x, const DenoisingParameterization& parameterization, float sigma, const SDCondition& condition, const std::vector<Tensor<float>>& references, const DiffusionPredictionOptions& options, const DiffusionPredictionStep& step);
+ };
+}
+
+#endif // __SD_DGL_MODELS_DIT_H__
diff --git a/src/dgl/models/fwd.h b/src/dgl/models/fwd.h
new file mode 100644
index 0000000..b34d07a
--- /dev/null
+++ b/src/dgl/models/fwd.h
@@ -0,0 +1,49 @@
+#ifndef __SD_DGL_MODELS_FWD_H__
+#define __SD_DGL_MODELS_FWD_H__
+
+namespace sd::dgl::models {
+ class AnimateDiff;
+ class AudioEncoder;
+ class AudioVAE;
+ class CircularAxesScope;
+ struct ControlBinding;
+ struct ControlBindings;
+ class ControlNet;
+ struct DiffusionModel;
+ struct DetailDetection;
+ struct DetailDetector;
+ struct DetailDetectorParams;
+ struct DetectionOptions;
+ class DiT;
+ class DenoisingParameterization;
+ struct DiTLoadParams;
+ struct DiffusionPrediction;
+ struct DiffusionPredictionOptions;
+ struct DiffusionPredictionStep;
+ class IPAdapter;
+ class ImageUpscaleModel;
+ struct ImageUpscaler;
+ struct ImageUpscalerParams;
+ class LatentUpscaler;
+ struct LoraBinding;
+ struct LoraConfig;
+ class LoraExecution;
+ struct LoraFile;
+ class Model;
+ struct ModelExecutionOptions;
+ struct ModelLoadParams;
+ class ModelRuntime;
+ enum class OperatorMode;
+ class PhotoMaker;
+ struct PhotoMakerPrompt;
+ struct PredictionParameters;
+ class PuLID;
+ class TextEncoder;
+ struct TextEncoderLoadParams;
+ struct UNetControls;
+ class VAE;
+ struct VAELoadParams;
+ class VisionEncoder;
+}
+
+#endif
diff --git a/src/pipeline/image_upscale.cpp b/src/dgl/models/image_upscale.cpp
similarity index 88%
rename from src/pipeline/image_upscale.cpp
rename to src/dgl/models/image_upscale.cpp
index 1ccb7b5..3a4a8a4 100644
--- a/src/pipeline/image_upscale.cpp
+++ b/src/dgl/models/image_upscale.cpp
@@ -1,14 +1,15 @@
-#include "pipeline/image_upscale.h"
+#include "dgl/models/image_upscale.h"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "model/upscaler/esrgan.hpp"
#include "model/upscaler/mosr.hpp"
+#include "model/upscaler/scale.hpp"
#include "model_manager.h"
#include "runtime/tiling.h"
-namespace sd::pipeline {
- ImageUpscaleModel::ImageUpscaleModel(const std::shared_ptr<ModelManager>& manager)
+namespace sd::dgl::models {
+ ImageUpscaleModel::ImageUpscaleModel(const std::shared_ptr<ModelManager::ResourceHandle>& manager)
: manager_(manager) {}
ImageUpscaleModel::~ImageUpscaleModel() {
@@ -34,11 +35,11 @@ namespace sd::pipeline {
return esrgan_ != nullptr ? esrgan_->config.scale : 1;
}
- std::unique_ptr<ImageUpscaleModel> load_image_upscale_model(
- const std::shared_ptr<ModelManager>& manager,
+ std::unique_ptr<ImageUpscaleModel> build_image_upscale_model(
+ const std::shared_ptr<ModelManager::ResourceHandle>& manager,
SDBackendManager& backends,
+ ModelLoader loader,
const std::string& path,
- size_t max_vram_bytes,
bool conv_direct,
ggml_type weight_type) {
if (manager == nullptr) {
@@ -50,20 +51,19 @@ namespace sd::pipeline {
LOG_ERROR("failed to initialize upscaler backends");
return nullptr;
}
- ModelLoader loader;
- if (!loader.init_from_file_and_convert_name(path, "", VERSION_ESRGAN)) {
- LOG_ERROR("init upscaler model loader failed: '%s'", path.c_str());
- return nullptr;
- }
- loader.set_wtype_override(weight_type);
- const auto tensors = loader.get_tensor_storage_map();
auto model = std::unique_ptr<ImageUpscaleModel>(new ImageUpscaleModel(manager));
model->source_file_ = manager->register_model_source(std::move(loader));
if (model->source_file_ == 0) {
return nullptr;
}
+ const auto& tensors = manager->model_source(model->source_file_).get_tensor_storage_map();
+ const auto description = sd::upscaling::describe(tensors);
+ if (description.architecture == sd::upscaling::Architecture::Unknown) {
+ LOG_ERROR("unsupported image upscaler weights: '%s'", path.c_str());
+ return nullptr;
+ }
std::map<std::string, ggml_tensor*> weights;
- if (tensors.count("gblocks.0.weight") != 0) {
+ if (description.architecture == sd::upscaling::Architecture::MoSR) {
model->mosr_ = std::make_unique<MoSR>(backend, tensors, manager);
if (model->mosr_->mosr_net == nullptr) {
LOG_ERROR("unsupported MoSR weights: '%s'", path.c_str());
@@ -91,7 +91,6 @@ namespace sd::pipeline {
LOG_ERROR("register upscaler weights failed: '%s'", path.c_str());
return nullptr;
}
- model->runner()->set_max_graph_vram_bytes(max_vram_bytes);
model->runner()->set_conv2d_direct_enabled(conv_direct);
LOG_INFO("Upscaler architecture: %s, weight type: %s", model->runner()->get_desc().c_str(), ggml_type_name(weight_type));
return model;
diff --git a/src/pipeline/image_upscale.h b/src/dgl/models/image_upscale.h
similarity index 63%
rename from src/pipeline/image_upscale.h
rename to src/dgl/models/image_upscale.h
index 1de9ca0..2fe9bc3 100644
--- a/src/pipeline/image_upscale.h
+++ b/src/dgl/models/image_upscale.h
@@ -1,5 +1,5 @@
-#ifndef __SD_PIPELINE_IMAGE_UPSCALE_H__
-#define __SD_PIPELINE_IMAGE_UPSCALE_H__
+#ifndef __SD_DGL_MODELS_IMAGE_UPSCALE_H__
+#define __SD_DGL_MODELS_IMAGE_UPSCALE_H__
#include <cstdint>
#include <memory>
@@ -8,43 +8,44 @@
#include "core/tensor.hpp"
#include "ggml.h"
+#include "model_manager.h"
-class ModelManager;
class SDBackendManager;
struct GGMLRunner;
struct ESRGAN;
struct MoSR;
-namespace sd::pipeline {
+namespace sd::dgl::models {
class ImageUpscaleModel {
- friend std::unique_ptr<ImageUpscaleModel> load_image_upscale_model(
- const std::shared_ptr<ModelManager>&,
+ friend std::unique_ptr<ImageUpscaleModel> build_image_upscale_model(
+ const std::shared_ptr<ModelManager::ResourceHandle>&,
SDBackendManager&,
+ ModelLoader,
const std::string&,
- size_t,
bool,
ggml_type);
friend Tensor<float> upscale_image_tensor(ImageUpscaleModel&, const Tensor<float>&, int, int);
- std::weak_ptr<ModelManager> manager_;
+ std::weak_ptr<ModelManager::ResourceHandle> manager_;
std::unique_ptr<ESRGAN> esrgan_;
std::unique_ptr<MoSR> mosr_;
std::vector<ggml_tensor*> weights_;
uint64_t source_file_ = 0;
- explicit ImageUpscaleModel(const std::shared_ptr<ModelManager>& manager);
+ explicit ImageUpscaleModel(const std::shared_ptr<ModelManager::ResourceHandle>& manager);
public:
~ImageUpscaleModel();
GGMLRunner* runner() const;
int scale() const;
+ ModelLoader::FileId source_file() const { return source_file_; }
};
- std::unique_ptr<ImageUpscaleModel> load_image_upscale_model(
- const std::shared_ptr<ModelManager>& manager,
+ std::unique_ptr<ImageUpscaleModel> build_image_upscale_model(
+ const std::shared_ptr<ModelManager::ResourceHandle>& manager,
SDBackendManager& backends,
+ ModelLoader loader,
const std::string& path,
- size_t max_vram_bytes = 0,
bool conv_direct = false,
ggml_type weight_type = GGML_TYPE_F16);
@@ -54,4 +55,4 @@ namespace sd::pipeline {
int tile_size);
}
-#endif // __SD_PIPELINE_IMAGE_UPSCALE_H__
+#endif // __SD_DGL_MODELS_IMAGE_UPSCALE_H__
diff --git a/src/dgl/models/ip_adapter.cpp b/src/dgl/models/ip_adapter.cpp
new file mode 100644
index 0000000..62dfbd6
--- /dev/null
+++ b/src/dgl/models/ip_adapter.cpp
@@ -0,0 +1,59 @@
+#include "dgl/models/ip_adapter.h"
+
+#include <algorithm>
+
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/dit.h"
+#include "model/adapter/ip_adapter.hpp"
+#include "model/diffusion/model.hpp"
+
+namespace sd::dgl::models {
+ IPAdapter::IPAdapter(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::IPAdapter, SDBackendModule::DIFFUSION, [this] { return build_runners(); }) {}
+
+ bool IPAdapter::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::IPAdapter>);
+ }
+
+ bool IPAdapter::build_runners() {
+ ip_adapter = std::make_shared<::IPAdapter::IPAdapterRunner>(runtime_.backend_for(), model_weights->loader().get_tensor_storage_map(), "ip_adapter", model_weights, true);
+ ip_adapter->set_flash_attention_enabled(false);
+ return runtime_.register_runner_params(runtime_.component, ip_adapter);
+ }
+
+ Tensor<float> IPAdapter::project_image(const Tensor<float>& embedding) {
+ const auto width = ip_adapter->is_plus ? ip_adapter->resampler.embed_dim : ip_adapter->image_proj.clip_dim;
+ const auto min_rank = ip_adapter->is_plus ? 2 : 1;
+ if (embedding.empty() || embedding.dim() < min_rank || embedding.dim() > min_rank + 1 ||
+ embedding.shape()[0] != width) {
+ throw std::runtime_error(ip_adapter->is_plus ? "IP-Adapter Plus requires CLIP Vision hidden states"
+ : "IP-Adapter Classic requires pooled CLIP Vision embeddings");
+ }
+ return ip_adapter->compute(n_threads, embedding);
+ }
+
+ void IPAdapter::validate_tokens(const Tensor<float>& tokens) const {
+ if (tokens.empty() || tokens.dim() < 2 || tokens.dim() > 3 ||
+ tokens.shape()[0] != ip_adapter->token_width()) {
+ throw std::runtime_error("IP-Adapter tokens must have shape [projection width, tokens, optional batch]");
+ }
+ }
+
+ void IPAdapter::validate_target(const DiT& target) const {
+ if (!sd_version_is_unet(target.version()))
+ throw std::runtime_error("IP-Adapter requires a UNet predictor");
+ if (!(sd_version_is_sd1(version()) && sd_version_is_sd1(target.version())) &&
+ !(sd_version_is_sdxl(version()) && sd_version_is_sdxl(target.version()))) {
+ throw std::runtime_error("IP-Adapter and UNet must use the same SD 1.x or SDXL family");
+ }
+ std::map<std::string, ggml_tensor*> weights;
+ target.diffusion_model->get_param_tensors(weights);
+ for (const auto& [layer, projection] : ip_adapter->attention) {
+ const auto query = weights.find(layer + ".to_q.weight");
+ if (query == weights.end() || query->second->ne[1] != projection.width)
+ throw std::runtime_error("IP-Adapter is incompatible with UNet attention layer '" + layer + "'");
+ }
+ }
+}
diff --git a/src/dgl/models/ip_adapter.h b/src/dgl/models/ip_adapter.h
new file mode 100644
index 0000000..cd05aba
--- /dev/null
+++ b/src/dgl/models/ip_adapter.h
@@ -0,0 +1,32 @@
+#ifndef __SD_DGL_MODELS_IP_ADAPTER_H__
+#define __SD_DGL_MODELS_IP_ADAPTER_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+namespace sd::dgl::models {
+ class IPAdapter final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ return runtime_.build_variant(static_cast<const IPAdapter&>(source).runtime_, std::move(loader));
+ }
+
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ bool init(ModelLoadParams params);
+ std::shared_ptr<::IPAdapter::IPAdapterRunner> ip_adapter;
+
+ IPAdapter(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ Tensor<float> project_image(const Tensor<float>& embedding);
+ void validate_tokens(const Tensor<float>& tokens) const;
+ void validate_target(const DiT& target) const;
+ };
+}
+
+#endif // __SD_DGL_MODELS_IP_ADAPTER_H__
diff --git a/src/dgl/models/latent_upscaler.cpp b/src/dgl/models/latent_upscaler.cpp
new file mode 100644
index 0000000..08296c2
--- /dev/null
+++ b/src/dgl/models/latent_upscaler.cpp
@@ -0,0 +1,66 @@
+#include "dgl/models/latent_upscaler.h"
+
+#include "core/image_geometry.h"
+#include "dgl/models/component_sources.h"
+#include "model/upscaler/h3_latent_upscaler.hpp"
+#include "model/upscaler/ltx_latent_upscaler.hpp"
+
+namespace sd::dgl::models {
+ LatentUpscaler::LatentUpscaler(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::LatentUpsampler, SDBackendModule::DIFFUSION, [this] { return build_runners(); }) {}
+
+ bool LatentUpscaler::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::LatentUpsampler>);
+ }
+
+ bool LatentUpscaler::build_variant(const Model& source, ModelLoader loader) {
+ return runtime_.build_variant(static_cast<const LatentUpscaler&>(source).runtime_, std::move(loader));
+ }
+
+ bool LatentUpscaler::build_runners() {
+ const auto& weights = model_weights->loader().get_tensor_storage_map();
+ const auto backend = runtime_.backend_for();
+ const auto linear_scale = runtime_.params.linear_scale;
+ const auto attn_scale = runtime_.params.attn_scale;
+ if (sd_version_is_ltxav(runtime_.version)) {
+ ltx_ = std::make_shared<LTXVUpsampler::LatentUpsamplerRunner>(backend, weights, "ltx_latent_upsampler", model_weights);
+ if (!ltx_->model)
+ return false;
+ ltx_->set_conv2d_direct_enabled(runtime_.conv_direct_for(false));
+ ltx_->set_scale_overrides(linear_scale, attn_scale);
+ return runtime_.register_runner_params(runtime_.component, ltx_);
+ }
+
+ h3_ = std::make_shared<H3VUpsampler::H3LatentUpscalerRunner>(backend, weights, "h3_latent_upscaler", model_weights);
+ if (!h3_->model || h3_->config.in_channels != 24)
+ return false;
+ h3_->set_scale_overrides(linear_scale, attn_scale);
+ return runtime_.register_runner_params(runtime_.component, h3_);
+ }
+
+ Tensor<float> LatentUpscaler::upscale(const Tensor<float>& latent, int64_t width, int64_t height) {
+ const auto channels = ltx_ ? ltx_->config.in_channels : h3_->config.in_channels;
+ const auto rank = latent.dim();
+ if (latent.empty() || (rank != 4 && rank != 5) ||
+ (rank == 5 && latent.shape()[4] != 1) || latent.shape()[3] != channels) {
+ throw std::runtime_error("latent upscaling requires [W,H,T,C] or [W,H,T,C,1] with " + std::to_string(channels) + " channels");
+ }
+ if (ltx_) {
+ if (width || height)
+ throw std::runtime_error("LTX latent upscaler determines its scale from the checkpoint");
+ return ltx_->compute(n_threads, latent);
+ }
+
+ const auto aligned_width = sd::align_image_dimension(width, 2);
+ const auto aligned_height = sd::align_image_dimension(height, 2);
+ if (width != aligned_width || height != aligned_height)
+ LOG_VERBOSE("upscale_latent: aligning target from %lldx%lld to %lldx%lld", (long long)width, (long long)height,
+ (long long)aligned_width, (long long)aligned_height);
+ width = aligned_width;
+ height = aligned_height;
+ if (width == latent.shape()[0] && height == latent.shape()[1])
+ return latent;
+ return h3_->compute(n_threads, latent, width, height);
+ }
+}
diff --git a/src/dgl/models/latent_upscaler.h b/src/dgl/models/latent_upscaler.h
new file mode 100644
index 0000000..965ff92
--- /dev/null
+++ b/src/dgl/models/latent_upscaler.h
@@ -0,0 +1,32 @@
+#ifndef __SD_DGL_MODELS_LATENT_UPSCALER_H__
+#define __SD_DGL_MODELS_LATENT_UPSCALER_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/model_runtime.h"
+
+namespace H3VUpsampler {
+ struct H3LatentUpscalerRunner;
+}
+namespace LTXVUpsampler {
+ struct LatentUpsamplerRunner;
+}
+
+namespace sd::dgl::models {
+ class LatentUpscaler final : public Model {
+ ModelRuntime runtime_;
+ std::shared_ptr<LTXVUpsampler::LatentUpsamplerRunner> ltx_;
+ std::shared_ptr<H3VUpsampler::H3LatentUpscalerRunner> h3_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override;
+ bool build_runners();
+
+ public:
+ LatentUpscaler(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(ModelLoadParams params);
+ Tensor<float> upscale(const Tensor<float>& latent, int64_t width, int64_t height);
+ };
+}
+
+#endif // __SD_DGL_MODELS_LATENT_UPSCALER_H__
diff --git a/src/dgl/models/lora.cpp b/src/dgl/models/lora.cpp
new file mode 100644
index 0000000..9aa8320
--- /dev/null
+++ b/src/dgl/models/lora.cpp
@@ -0,0 +1,93 @@
+#include "dgl/models/lora.h"
+
+#include <string_view>
+
+#include "core/ggml_extend_backend.h"
+#include "model/adapter/lora.hpp"
+
+namespace sd::dgl::models {
+
+ static bool is_patch_tensor(std::string_view suffix) {
+ const auto index = suffix.find('.');
+ if (index != std::string_view::npos && index != 0 &&
+ suffix.substr(0, index).find_first_not_of("0123456789") == std::string_view::npos)
+ suffix.remove_prefix(index + 1);
+ return suffix == "diff" || suffix == "lora_up" || suffix == "lora_down" ||
+ suffix.substr(0, 6) == "hada_w" || suffix.substr(0, 6) == "lokr_w";
+ }
+
+ void LoraExecution::end() {
+ for (const auto& model : models_)
+ model->runner_end();
+ }
+
+ void LoraExecution::reset() {
+ end();
+ adapter_.reset();
+ models_.clear();
+ ready_ = false;
+ }
+
+ bool LoraExecution::prepare(const LoraConfig& config, const std::shared_ptr<ModelManager::ResourceHandle>& weights, SDVersion version, SDBackendManager& backends, SDBackendModule module, int n_threads, bool immediately) {
+ if (ready_)
+ return true;
+ std::vector<ModelManager::LoraSpec> specs;
+ for (const auto& binding : config.bindings) {
+ ModelManager::LoraSpec spec;
+ spec.path = binding.file->path;
+ spec.multiplier = binding.weight;
+ spec.required = true;
+ specs.push_back(std::move(spec));
+ }
+ if (!weights->prepare_lora_sources(specs))
+ return false;
+
+ const auto backend = backends.runtime_backend(module);
+ const auto params_backend = backends.params_backend(module);
+ if (!backend || !params_backend)
+ return false;
+ const auto residency = backends.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk
+ : ModelManager::ResidencyMode::ParamBackend;
+ const auto names = weights->tensor_names();
+ std::vector<std::shared_ptr<LoraModel>> models;
+ std::set<std::string> targets;
+ for (auto& spec : specs) {
+ auto model = std::make_shared<LoraModel>(spec.path, backend, params_backend, weights, spec.file_id,
+ version, residency, backends.params_backend_follows_runtime(module));
+ if (!model->init_params(n_threads))
+ return false;
+ model->preprocess_lora_tensors(names);
+ std::set<std::string_view> keys;
+ for (const auto& [name, _] : model->lora_tensors)
+ keys.insert(name);
+ bool matches = false;
+ for (const auto& name : names) {
+ const auto prefix = "lora." + name + ".";
+ for (auto key = keys.lower_bound(prefix);
+ key != keys.end() && key->substr(0, prefix.size()) == prefix; ++key) {
+ if (is_patch_tensor(key->substr(prefix.size()))) {
+ targets.insert(name);
+ matches = true;
+ break;
+ }
+ }
+ }
+ if (!matches)
+ throw std::runtime_error("LoRA '" + spec.path + "' has no matching model weights");
+ model->multiplier = spec.multiplier;
+ spec.prepared = model;
+ models.push_back(std::move(model));
+ }
+ if (!weights->set_loras(immediately ? std::move(specs) : std::vector<ModelManager::LoraSpec>{},
+ version, immediately ? std::move(targets) : std::set<std::string>{}))
+ return false;
+ if (!immediately && !models.empty())
+ adapter_ = std::make_shared<MultiLoraAdapter>(models);
+ if (!models.empty())
+ LOG_INFO("apply lora %s", immediately ? "immediately" : "at runtime");
+ models_ = std::move(models);
+ ready_ = true;
+ return true;
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/lora.h b/src/dgl/models/lora.h
new file mode 100644
index 0000000..2a6ed2e
--- /dev/null
+++ b/src/dgl/models/lora.h
@@ -0,0 +1,46 @@
+#ifndef __SD_DGL_MODELS_LORA_H__
+#define __SD_DGL_MODELS_LORA_H__
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/option_types.h"
+#include "model_manager.h"
+
+struct LoraModel;
+struct WeightAdapter;
+class SDBackendManager;
+enum class SDBackendModule;
+
+namespace sd::dgl::models {
+
+ struct LoraFile {
+ std::string path;
+ };
+
+ struct LoraBinding {
+ std::shared_ptr<const LoraFile> file;
+ float weight = 1.0f;
+ };
+
+ struct LoraConfig {
+ std::vector<LoraBinding> bindings;
+ lora_apply_mode_t mode = LORA_APPLY_AUTO;
+ };
+
+ class LoraExecution {
+ std::vector<std::shared_ptr<LoraModel>> models_;
+ std::shared_ptr<WeightAdapter> adapter_;
+ bool ready_ = false;
+
+ public:
+ void end();
+ void reset();
+ bool prepare(const LoraConfig& config, const std::shared_ptr<ModelManager::ResourceHandle>& weights, SDVersion version, SDBackendManager& backends, SDBackendModule module, int n_threads, bool immediately);
+ const std::shared_ptr<WeightAdapter>& adapter() const { return adapter_; }
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_LORA_H__
diff --git a/src/dgl/models/model_base.cpp b/src/dgl/models/model_base.cpp
new file mode 100644
index 0000000..e9e851b
--- /dev/null
+++ b/src/dgl/models/model_base.cpp
@@ -0,0 +1,168 @@
+#include "dgl/models/model_base.h"
+
+#include <stdexcept>
+#include <typeinfo>
+
+#include "core/probe/backend_probe.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/run_events.h"
+
+namespace sd::dgl::models {
+
+ Model::Model(std::shared_ptr<ModelManager> manager)
+ : backend_probes_(std::make_shared<BackendProbeCache>()),
+ family_probes_(std::make_shared<FamilyProbeCache>()),
+ model_manager(std::move(manager)),
+ model_weights(std::make_shared<ModelManager::ResourceHandle>(model_manager)) {}
+
+ Model::~Model() {
+ release_weights();
+ }
+
+ void Model::configure_execution(const ModelExecutionOptions& options) {
+ if (options.backend_probes)
+ backend_probes_ = options.backend_probes;
+ if (options.family_probes)
+ family_probes_ = options.family_probes;
+ tensor_executor = options.tensor_executor;
+ n_threads = tensor_executor->num_threads();
+ eager_load_ = options.eager_load;
+ defer_initialization_ = options.defer_initialization;
+ initialization_pending_ = options.defer_initialization;
+ model_weights->set_n_threads(n_threads);
+ model_weights->set_enable_mmap(options.enable_mmap);
+ model_weights->set_prefetch_disabled(options.disable_prefetch);
+ model_weights->set_segmented_compute_disabled(options.disable_segmented_compute);
+ }
+
+ bool Model::load_weights_eagerly() const {
+ if (defer_initialization_ || !eager_load_ || model_weights->load_all_params_eagerly())
+ return true;
+ LOG_ERROR("model params eager load failed");
+ return false;
+ }
+
+ Model::ExecutionScope::ExecutionScope(Model& model, bool use_tensor_executor)
+ : model_(model), lock_(model.execution_mutex_, std::try_to_lock) {
+ if (!lock_.owns_lock() || model_.executing_)
+ return;
+ model_.executing_ = acquired_ = true;
+ try {
+ if (model_.tensor_executor)
+ tensor_scope_.emplace(model_.tensor_executor.get());
+ report_run_progress("preparing");
+ ready_ = model_.refresh_weights() && model_.finish_initialization() && model_.apply_loras();
+ if (!use_tensor_executor)
+ tensor_scope_.reset();
+ } catch (...) {
+ model_.executing_ = false;
+ throw;
+ }
+ }
+
+ Model::ExecutionScope::~ExecutionScope() {
+ if (ready_) {
+ model_.end_runners();
+ model_.finish_execution();
+ }
+ if (acquired_)
+ model_.executing_ = false;
+ }
+
+ void Model::ExecutionScope::require_ready() const {
+ if (!ready_)
+ throw std::runtime_error("cannot access model component");
+ }
+
+ ModelLoader Model::base_loader() const {
+ return model_weights->base_loader();
+ }
+
+ bool Model::finish_initialization() {
+ if (!initialization_pending_)
+ return true;
+ defer_initialization_ = false;
+ if (eager_load_ && eager_load_supported_ && !load_weights_eagerly())
+ return false;
+ if (!finish_loading())
+ return false;
+ initialization_pending_ = false;
+ return true;
+ }
+
+ bool Model::place_params(const std::string& name, ggml_backend_t backend, bool disk) {
+ const auto current = model_weights->memory();
+ const auto mode = disk ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend;
+ if (current.params_backend == backend && current.mode == mode)
+ return true;
+ if (!model_weights->place_params(backend, disk ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend))
+ return false;
+ set_weight_adapter(nullptr);
+ reset_loras();
+ return !params_placement_ || params_placement_(name);
+ }
+
+ bool Model::init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent) {
+ if (typeid(*this) != typeid(base))
+ throw std::invalid_argument("LoRA variant must have the same model type as its base");
+ loras_ = std::move(loras);
+ persistent_loras_ = persistent;
+ family_dependencies_ = base.family_dependencies_;
+ return build_variant(base, base.base_loader()) && apply_loras();
+ }
+
+ ModelLoader::Dependencies Model::dependencies() const {
+ auto result = model_weights->dependencies();
+ result.insert(result.end(), family_dependencies_.begin(), family_dependencies_.end());
+ return result;
+ }
+
+ bool Model::has_loras() const {
+ return loras_ && !loras_->bindings.empty();
+ }
+
+ lora_apply_mode_t Model::lora_mode() const {
+ const auto mode = loras_ ? loras_->mode : LORA_APPLY_AUTO;
+ return persistent_loras_ && mode == LORA_APPLY_AUTO ? LORA_APPLY_AT_RUNTIME : mode;
+ }
+
+ bool Model::apply_loras() {
+ return !has_loras() || attach_loras();
+ }
+
+ bool Model::prepare_loras(SDVersion version, SDBackendManager& backends, SDBackendModule module, int n_threads, bool immediately) {
+ if (!lora_execution_.prepare(*loras_, model_weights, version, backends, module, n_threads, immediately))
+ return false;
+ set_weight_adapter(lora_execution_.adapter());
+ return true;
+ }
+
+ void Model::end_runners() {
+ for (const auto& runner : runners_)
+ runner.end();
+ lora_execution_.end();
+ }
+
+ void Model::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
+ for (const auto& runner : runners_) {
+ if (runner.set_adapter)
+ runner.set_adapter(adapter);
+ }
+ }
+
+ void Model::reset_loras() {
+ lora_execution_.reset();
+ }
+
+ void Model::release_weights() {
+ if (weights_released_)
+ return;
+ end_runners();
+ Model::set_weight_adapter(nullptr);
+ reset_loras();
+ GGML_ASSERT(model_weights->remove());
+ runners_.clear();
+ weights_released_ = true;
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/model_base.h b/src/dgl/models/model_base.h
new file mode 100644
index 0000000..0c98983
--- /dev/null
+++ b/src/dgl/models/model_base.h
@@ -0,0 +1,108 @@
+#ifndef __SD_DGL_MODELS_MODEL_BASE_H__
+#define __SD_DGL_MODELS_MODEL_BASE_H__
+
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include "core/parallel.h"
+#include "dgl/models/lora.h"
+
+class BackendProbeCache;
+
+namespace sd::dgl::models {
+
+ class FamilyProbeCache;
+
+ struct ModelExecutionOptions {
+ std::shared_ptr<sd::ParallelExecutor> tensor_executor;
+ bool enable_mmap = false;
+ bool eager_load = false;
+ bool disable_prefetch = false;
+ bool disable_segmented_compute = false;
+ std::shared_ptr<BackendProbeCache> backend_probes;
+ std::shared_ptr<FamilyProbeCache> family_probes;
+ bool defer_initialization = false;
+ };
+
+ class Model {
+ friend class ModelRuntime;
+
+ struct RunnerRegistration {
+ std::function<void()> end;
+ std::function<void(const std::shared_ptr<WeightAdapter>&)> set_adapter;
+ };
+
+ std::vector<RunnerRegistration> runners_;
+ std::recursive_mutex execution_mutex_;
+ std::shared_ptr<const LoraConfig> loras_;
+ LoraExecution lora_execution_;
+ bool executing_ = false;
+ bool persistent_loras_ = false;
+ bool weights_released_ = false;
+ bool eager_load_ = false;
+ ModelLoader::Dependencies family_dependencies_;
+ std::shared_ptr<BackendProbeCache> backend_probes_;
+ std::shared_ptr<FamilyProbeCache> family_probes_;
+ bool defer_initialization_ = false;
+ bool initialization_pending_ = false;
+ bool eager_load_supported_ = true;
+
+ void end_runners();
+
+ protected:
+ std::function<bool(const std::string&)> params_placement_;
+ explicit Model(std::shared_ptr<ModelManager> manager);
+ virtual ModelLoader base_loader() const;
+ virtual bool build_variant(const Model& base, ModelLoader loader) = 0;
+ virtual bool attach_loras() = 0;
+ virtual void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter);
+ virtual bool refresh_weights() { return true; }
+ virtual bool finish_loading() { return true; }
+ virtual void finish_execution() {}
+ virtual void invalidate_cached_state() {}
+
+ bool has_loras() const;
+ lora_apply_mode_t lora_mode() const;
+ bool apply_loras();
+ bool prepare_loras(SDVersion version, SDBackendManager& backends, SDBackendModule module, int n_threads, bool immediately);
+ void reset_loras();
+ bool load_weights_eagerly() const;
+ // Release the resource while registered tensors and their backends are alive.
+ void release_weights();
+
+ public:
+ const std::shared_ptr<ModelManager> model_manager;
+ const std::shared_ptr<ModelManager::ResourceHandle> model_weights;
+ int n_threads = -1;
+ std::shared_ptr<sd::ParallelExecutor> tensor_executor;
+
+ class ExecutionScope {
+ Model& model_;
+ std::unique_lock<std::recursive_mutex> lock_;
+ bool ready_ = false;
+ bool acquired_ = false;
+ std::optional<sd::ParallelScope> tensor_scope_;
+
+ public:
+ explicit ExecutionScope(Model& model, bool use_tensor_executor = true);
+ ~ExecutionScope();
+ void require_ready() const;
+ };
+
+ Model(const Model&) = delete;
+ Model& operator=(const Model&) = delete;
+ virtual ~Model();
+ void configure_execution(const ModelExecutionOptions& options);
+ bool finish_initialization();
+ bool place_params(const std::string& name, ggml_backend_t backend, bool disk);
+ void set_shared_weights(std::string key) { model_weights->set_shared_weights(std::move(key)); }
+ bool init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent);
+ ModelLoader::Dependencies dependencies() const;
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_MODEL_BASE_H__
diff --git a/src/dgl/models/model_load_params.h b/src/dgl/models/model_load_params.h
new file mode 100644
index 0000000..b4007f9
--- /dev/null
+++ b/src/dgl/models/model_load_params.h
@@ -0,0 +1,35 @@
+#ifndef __SD_DGL_MODELS_MODEL_LOAD_PARAMS_H__
+#define __SD_DGL_MODELS_MODEL_LOAD_PARAMS_H__
+
+#include <string>
+
+#include "model.h"
+#include "stable-diffusion.h"
+
+namespace sd::dgl::models {
+
+ enum class OperatorMode { Auto,
+ On,
+ Off };
+
+ struct ModelLoadParams {
+ std::string path;
+ SDVersion family = VERSION_COUNT;
+ std::string family_from;
+ std::string backend;
+ std::string params_backend;
+ std::string split_mode;
+ sd_type_t wtype = SD_TYPE_COUNT;
+ std::string tensor_type_rules;
+ std::string model_args;
+ OperatorMode flash_attn_mode = OperatorMode::Auto;
+ OperatorMode conv_direct_mode = OperatorMode::Auto;
+ bool sage_attn = false;
+ float conv_scale = 0.f;
+ float linear_scale = 0.f;
+ float attn_scale = 0.f;
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_MODEL_LOAD_PARAMS_H__
diff --git a/src/dgl/models/model_runtime.cpp b/src/dgl/models/model_runtime.cpp
new file mode 100644
index 0000000..3bed540
--- /dev/null
+++ b/src/dgl/models/model_runtime.cpp
@@ -0,0 +1,546 @@
+#include "dgl/models/model_runtime.h"
+
+#include <algorithm>
+#include <cmath>
+#include <type_traits>
+
+#include "conditioning/conditioner.hpp"
+#include "core/ggml_runner.h"
+#include "core/ggml_tensor_utils.h"
+#include "core/layer_split_partition.h"
+#include "core/probe/backend_probe.h"
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/control.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/vae.h"
+#include "model/adapter/ip_adapter.hpp"
+#include "model/adapter/pmid.hpp"
+#include "model/adapter/pulid.hpp"
+#include "model/audio/wav2vec2.hpp"
+#include "model/diffusion/animatediff.hpp"
+#include "model/diffusion/control.hpp"
+#include "model/diffusion/z_image.hpp"
+#include "model/upscaler/h3_latent_upscaler.hpp"
+#include "model/upscaler/ltx_latent_upscaler.hpp"
+#include "model/vae/audio_vae.hpp"
+#include "model/vae/vae.hpp"
+#include "model_definition.h"
+#include "name_conversion.h"
+
+namespace sd::dgl::models {
+
+ template <typename T, typename = void>
+ struct has_set_runtime_backends : std::false_type {};
+ template <typename T>
+ struct has_set_runtime_backends<T,
+ std::void_t<decltype(std::declval<T&>().set_runtime_backends(
+ std::declval<const std::vector<ggml_backend_t>&>()))>> : std::true_type {};
+
+ ModelRuntime::ModelRuntime(Model& model, std::shared_ptr<SDBackendPool> backends, ModelComponent component, SDBackendModule module, std::function<bool()> build_runners)
+ : model_(model),
+ build_runners_(std::move(build_runners)),
+ component(component),
+ module(module),
+ backend_manager(std::move(backends)) {
+ model_.params_placement_ = [this](const std::string& name) { return set_params_placement(name); };
+ }
+
+ ModelRuntime::~ModelRuntime() {
+ model_.release_weights();
+ }
+
+ bool ModelRuntime::refresh_weights() {
+ bool changed;
+ if (!model_.model_weights->loader().files_changed(changed, false))
+ return false;
+ if (!changed && ready_ && catalog_revision_ == model_.model_weights->loader().revision())
+ return true;
+
+ ModelLoader candidate = model_.model_weights->loader();
+ if (!candidate.refresh_files(false))
+ return false;
+ ready_ = false;
+ model_.end_runners();
+ model_.set_weight_adapter(nullptr);
+ model_.reset_loras();
+ model_.invalidate_cached_state();
+ for (auto registered : registered_components_) {
+ if (!model_.model_weights->unregister_param_tensors(registered))
+ return false;
+ }
+ registered_components_.clear();
+ model_.runners_.clear();
+ return model_.model_weights->set_loader(std::move(candidate)) && rebuild();
+ }
+
+ ggml_backend_t ModelRuntime::backend_for() {
+ ggml_backend_t module_backend = backend_manager.runtime_backend(module);
+ if (module_backend == nullptr) {
+ LOG_ERROR("failed to initialize %s backend", sd_backend_module_name(module));
+ }
+ return module_backend;
+ }
+
+ ggml_backend_t ModelRuntime::params_backend_for() {
+ ggml_backend_t module_backend = backend_manager.params_backend(module);
+ if (module_backend == nullptr) {
+ LOG_ERROR("failed to initialize %s params backend", sd_backend_module_name(module));
+ }
+ return module_backend;
+ }
+
+ bool ModelRuntime::ensure_backend_pair() {
+ if (backend_for() == nullptr) {
+ return false;
+ }
+ return params_backend_for() != nullptr;
+ }
+
+ template <typename T>
+ bool ModelRuntime::register_runner_params(ModelComponent component,
+ const std::shared_ptr<T>& model,
+ size_t* params_mem_size) {
+ if (model == nullptr) {
+ return true;
+ }
+ // Keep tensor metadata alive until ModelManager unregisters the resource.
+ Model::RunnerRegistration registration{[model] { model->runner_end(); }, {}};
+ if constexpr (std::is_base_of_v<GGMLRunner, T> || std::is_same_v<T, Conditioner> ||
+ std::is_same_v<T, DiffusionModelRunner>) {
+ registration.set_adapter = [model](std::shared_ptr<WeightAdapter> adapter) {
+ model->set_weight_adapter(std::move(adapter));
+ };
+ }
+ model_.runners_.push_back(std::move(registration));
+ registered_components_.insert(component);
+ std::map<std::string, ggml_tensor*> group_tensors;
+ std::map<ggml_tensor*, enum ggml_op> tensor_ops;
+ model->get_param_tensors(group_tensors);
+ if constexpr (std::is_base_of_v<Conditioner, T>) {
+ model->get_param_tensor_ops(tensor_ops);
+ }
+ ModelManager::ResidencyMode residency_mode =
+ backend_manager.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend;
+
+ std::vector<ggml_backend_t> module_backends = backend_manager.runtime_backends(module);
+ if (module_backends.size() > 1) {
+ if constexpr (has_set_runtime_backends<T>::value) {
+ if (module == SDBackendModule::DIFFUSION || module == SDBackendModule::TE) {
+ if (backend_manager.split_mode(module) == SDSplitMode::ROW) {
+ return register_row_split_runner_params(component,
+ model,
+ module_backends,
+ std::move(group_tensors),
+ tensor_ops,
+ residency_mode,
+ params_mem_size);
+ }
+ return register_layer_split_runner_params(component,
+ model,
+ module_backends,
+ std::move(group_tensors),
+ tensor_ops,
+ residency_mode,
+ params_mem_size);
+ }
+ }
+ LOG_WARN("%s module does not support multiple runtime backends; using %s",
+ sd_backend_module_name(module),
+ sd::layer_split_backend_device_display_name(module_backends[0]).c_str());
+ }
+ return model_.model_weights->register_param_tensors(component,
+ std::move(group_tensors),
+ residency_mode,
+ backend_for(),
+ params_backend_for(),
+ params_mem_size,
+ false,
+ false,
+ &tensor_ops);
+ }
+
+ template <typename T>
+ bool ModelRuntime::register_row_split_runner_params(ModelComponent component,
+ const std::shared_ptr<T>& model,
+ const std::vector<ggml_backend_t>& module_backends,
+ std::map<std::string, ggml_tensor*> group_tensors,
+ const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
+ ModelManager::ResidencyMode residency_mode,
+ size_t* params_mem_size) {
+ ggml_backend_t main_backend = module_backends[0];
+
+ auto fall_back_to_layer_split = [&](const char* reason) {
+ LOG_WARN("%s: row split unavailable (%s); falling back to layer split", model_component_name(component), reason);
+ return register_layer_split_runner_params(component,
+ model,
+ module_backends,
+ std::move(group_tensors),
+ tensor_ops,
+ residency_mode,
+ params_mem_size);
+ };
+
+ ggml_backend_dev_t main_dev = ggml_backend_get_device(main_backend);
+ ggml_backend_reg_t reg = main_dev != nullptr ? ggml_backend_dev_backend_reg(main_dev) : nullptr;
+ if (reg == nullptr) {
+ return fall_back_to_layer_split("no backend registry");
+ }
+ const size_t reg_dev_count = ggml_backend_reg_dev_count(reg);
+ std::vector<float> tensor_split(reg_dev_count, 0.0f);
+ constexpr int64_t compute_headroom_bytes = 2ll * 1024 * 1024 * 1024;
+ for (ggml_backend_t backend : module_backends) {
+ ggml_backend_dev_t dev = ggml_backend_get_device(backend);
+ int reg_index = -1;
+ for (size_t i = 0; i < reg_dev_count; i++) {
+ if (ggml_backend_reg_dev_get(reg, i) == dev) {
+ reg_index = (int)i;
+ break;
+ }
+ }
+ if (reg_index < 0) {
+ return fall_back_to_layer_split("devices span different backend registries");
+ }
+ size_t free_bytes = 0, total_bytes = 0;
+ ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
+ int64_t usable_bytes = std::max<int64_t>((int64_t)free_bytes - compute_headroom_bytes,
+ (int64_t)free_bytes / 8);
+ tensor_split[reg_index] = usable_bytes > 0 ? (float)((double)usable_bytes / (1024.0 * 1024.0)) : 1.0f;
+ }
+
+ ggml_backend_buffer_type_t split_buft = backend_manager.split_buffer_type(main_backend, tensor_split);
+ if (split_buft == nullptr) {
+ return fall_back_to_layer_split("backend has no split buffer type");
+ }
+ std::vector<std::pair<ggml_backend_t, size_t>> split_device_limits;
+ for (auto backend : module_backends) {
+ split_device_limits.emplace_back(backend, 0);
+ }
+ model_.model_weights->set_split_buffer_type(main_backend, split_buft, split_device_limits);
+
+ std::map<std::string, ggml_tensor*> split_tensors;
+ if constexpr (std::is_base_of_v<Conditioner, T>) {
+ model->get_layer_split_param_tensors(split_tensors);
+ } else {
+ split_tensors = group_tensors;
+ }
+
+ std::map<std::string, ggml_tensor*> row_split_map;
+ std::map<std::string, ggml_tensor*> regular_map;
+ size_t row_split_bytes = 0;
+ for (const auto& kv : group_tensors) {
+ if (split_tensors.count(kv.first) != 0 &&
+ sd::layer_split_tensor_block_index(kv.first) >= 0 &&
+ ModelManager::tensor_shape_supports_split_buffer(kv.second)) {
+ row_split_map[kv.first] = kv.second;
+ row_split_bytes += ggml_nbytes(kv.second);
+ } else {
+ regular_map[kv.first] = kv.second;
+ }
+ }
+ if (row_split_map.empty()) {
+ return fall_back_to_layer_split("no row-splittable transformer block weights found");
+ }
+
+ LOG_INFO("%s row split: %zu tensors (%.1f MB) split across %zu devices (main %s)",
+ model_component_name(component),
+ row_split_map.size(),
+ row_split_bytes / (1024.f * 1024.f),
+ module_backends.size(),
+ sd::layer_split_backend_device_display_name(main_backend).c_str());
+
+ if (!model_.model_weights->register_param_tensors(component,
+ std::move(row_split_map),
+ residency_mode,
+ main_backend,
+ params_backend_for(),
+ params_mem_size,
+ /*allow_split_buffer=*/true,
+ false,
+ &tensor_ops)) {
+ return false;
+ }
+ return model_.model_weights->register_param_tensors(component,
+ std::move(regular_map),
+ residency_mode,
+ main_backend,
+ params_backend_for(),
+ params_mem_size,
+ false,
+ false,
+ &tensor_ops);
+ }
+
+ template <typename T>
+ bool ModelRuntime::register_layer_split_runner_params(ModelComponent component,
+ const std::shared_ptr<T>& model,
+ const std::vector<ggml_backend_t>& module_backends,
+ std::map<std::string, ggml_tensor*> group_tensors,
+ const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
+ ModelManager::ResidencyMode residency_mode,
+ size_t* params_mem_size) {
+ bool has_cpu_device = false;
+ for (ggml_backend_t backend : module_backends) {
+ has_cpu_device = has_cpu_device || sd_backend_is_cpu(backend);
+ }
+ if (has_cpu_device) {
+ // The scheduler reserves the CPU slot for its fallback backend, and
+ // CPU weight participation is what --params-backend <module>=cpu is
+ // for; a CPU device in a split list is almost certainly a mistake.
+ LOG_WARN(
+ "%s: layer split across a CPU device is not supported; using %s "
+ "(use --params-backend %s=cpu to keep weights in RAM)",
+ model_component_name(component),
+ sd::layer_split_backend_device_display_name(module_backends[0]).c_str(),
+ sd_backend_module_name(module));
+ return model_.model_weights->register_param_tensors(component,
+ std::move(group_tensors),
+ residency_mode,
+ module_backends[0],
+ params_backend_for(),
+ params_mem_size,
+ false,
+ false,
+ &tensor_ops);
+ }
+
+ model->set_runtime_backends(module_backends);
+ model->set_graph_cut_layer_split_enabled(true);
+ model_.model_weights->set_layer_split(true);
+ const bool params_follow_runtime = backend_manager.params_backend_follows_runtime(module) ||
+ backend_manager.params_backend_is_disk(module);
+ ggml_backend_t initial_params_backend = params_follow_runtime ? module_backends[0] : params_backend_for();
+ if (initial_params_backend == nullptr) {
+ return false;
+ }
+
+ LOG_INFO("%s graph-cut layer split: deferring %zu tensors across %zu runtime backends until first graph",
+ model_component_name(component),
+ group_tensors.size(),
+ module_backends.size());
+
+ return model_.model_weights->register_param_tensors(component,
+ std::move(group_tensors),
+ residency_mode,
+ module_backends[0],
+ initial_params_backend,
+ params_mem_size,
+ false,
+ params_follow_runtime,
+ &tensor_ops);
+ }
+
+ bool ModelRuntime::split_active(SDSplitMode mode) {
+ return (module == SDBackendModule::DIFFUSION || module == SDBackendModule::TE) &&
+ backend_manager.split_mode(module) == mode && backend_manager.runtime_backends(module).size() > 1;
+ }
+
+ bool ModelRuntime::configure_backends() {
+ std::string error;
+ if (!backend_manager.init(backend_spec_.c_str(), params_backend_spec_.c_str(), split_mode_.c_str(), &error)) {
+ LOG_ERROR("backend config failed: %s", error.c_str());
+ return false;
+ }
+ return ensure_backend_pair();
+ }
+
+ bool ModelRuntime::init(ModelLoadParams load_params, const LoadCatalog& load_catalog) {
+ auto family = model_.family_probes_->resolve(load_params.family, load_params.family_from, model_.family_dependencies_);
+ params = std::move(load_params);
+#ifdef SD_USE_UPSTREAM_GGML
+ LOG_WARN(
+ "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are "
+ "converted to F16 at load time. Some operators may be unsupported and performance "
+ "may be lower than with patched GGML.");
+#endif
+ if (!validate_tensor_types(params.wtype, params.tensor_type_rules.c_str()))
+ return false;
+ for (float scale : {params.conv_scale, params.linear_scale, params.attn_scale}) {
+ if (!std::isfinite(scale) || scale < 0.f || (scale > 0.f && !std::isfinite(1.f / scale))) {
+ LOG_ERROR("scale overrides must be finite positive values, or 0 to keep model defaults");
+ return false;
+ }
+ }
+ split_mode_ = trim(params.split_mode);
+ backend_spec_ = trim(params.backend);
+ params_backend_spec_ = trim(params.params_backend);
+ ggml_log_set(sd_ggml_log_callback, nullptr);
+
+ sd::ParallelScope tensor_scope(model_.tensor_executor.get());
+ ModelLoader loader;
+ {
+ if (!load_catalog(loader, params, family, *model_.family_probes_))
+ return false;
+ version = family;
+ if (version == VERSION_COUNT) {
+ LOG_ERROR("get sd version from file failed: '%s'", params.path.c_str());
+ return false;
+ }
+ LOG_DEBUG("%s family: %s", model_component_name(component), sd_version_name(version));
+ }
+
+ if (!configure_backends())
+ return false;
+ return model_.model_weights->set_loader(std::move(loader)) && rebuild();
+ }
+
+ bool ModelRuntime::build_variant(const ModelRuntime& base, ModelLoader loader) {
+ params = base.params;
+ version = base.version;
+ backend_spec_ = base.backend_spec_;
+ params_backend_spec_ = base.params_backend_spec_;
+ split_mode_ = base.split_mode_;
+ sd::ParallelScope tensor_scope(model_.tensor_executor.get());
+ return configure_backends() && model_.model_weights->set_loader(std::move(loader)) && rebuild();
+ }
+
+ bool ModelRuntime::set_params_placement(const std::string& name) {
+ std::string error;
+ if (!backend_manager.init(backend_spec_.c_str(), name.c_str(), split_mode_.c_str(), &error))
+ throw std::runtime_error(error);
+ params_backend_spec_ = name;
+ if (!ensure_backend_pair())
+ return false;
+ configure_weight_loading();
+ return true;
+ }
+
+ void ModelRuntime::configure_weight_loading() {
+ model_.model_weights->prepare_file_io();
+ if (!model_.has_loras())
+ return;
+ const auto mode = model_.lora_mode();
+ const auto& model_loader = model_.model_weights->loader();
+ const auto wtype_stat = model_loader.get_wtype_stat();
+ bool have_int8_tensorwise = false;
+ bool have_quant_type_rule = false;
+ for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) {
+ if (tensor_storage.is_int8_tensorwise || tensor_storage.w4_convrot_kind) {
+ have_int8_tensorwise = true;
+ }
+ if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) {
+ have_quant_type_rule = true;
+ }
+ if (have_int8_tensorwise && have_quant_type_rule) {
+ break;
+ }
+ }
+
+ if (mode == LORA_APPLY_AUTO) {
+ bool have_quantized_weight = have_int8_tensorwise;
+ for (const auto& [type, _] : wtype_stat) {
+ if (ggml_is_quantized(type)) {
+ have_quantized_weight = true;
+ break;
+ }
+ }
+ // Avoid full-model LoRA merge buffers on constrained setups.
+ const bool params_offloaded = params_backend_for() != backend_for();
+ const bool streaming_constrained = params_offloaded ||
+ backend_manager.params_backend_is_disk(module);
+
+ if (have_quant_type_rule || have_quantized_weight || streaming_constrained || split_active(SDSplitMode::ROW)) {
+ apply_lora_immediately_ = false;
+ } else {
+ apply_lora_immediately_ = true;
+ }
+ } else if (mode == LORA_APPLY_IMMEDIATELY) {
+ apply_lora_immediately_ = !have_int8_tensorwise && !split_active(SDSplitMode::ROW);
+ } else {
+ apply_lora_immediately_ = false;
+ }
+ }
+
+ sd::model_builders::Context ModelRuntime::model_build_context() {
+ sd::model_builders::ModelBuildParams build_params;
+ build_params.model_args = params.model_args;
+ build_params.linear_scale = params.linear_scale;
+ build_params.attn_scale = params.attn_scale;
+ return {std::move(build_params), version, model_.model_weights->loader().get_tensor_storage_map(), backend_manager, model_.model_weights};
+ }
+
+ bool ModelRuntime::flash_attention_for() {
+ return params.flash_attn_mode == OperatorMode::Auto ? supports_operation(BackendProbeOperation::FlashAttention) : params.flash_attn_mode == OperatorMode::On;
+ }
+
+ bool ModelRuntime::conv_direct_for(bool conv3d) {
+ return params.conv_direct_mode == OperatorMode::Auto ? supports_operation(conv3d ? BackendProbeOperation::Conv3D : BackendProbeOperation::Conv2D) : params.conv_direct_mode == OperatorMode::On;
+ }
+
+ bool ModelRuntime::supports_operation(BackendProbeOperation operation) {
+ const auto devices = backend_manager.runtime_backends(module);
+ bool enabled = !devices.empty();
+ bool any_enabled = false;
+ for (auto backend : devices) {
+ const bool supported = model_.backend_probes_->supports(operation, backend, model_.n_threads);
+ enabled = enabled && supported;
+ any_enabled = any_enabled || supported;
+ }
+ if (any_enabled && !enabled) {
+ static constexpr const char* names[] = {"flash attention", "conv2d direct", "conv3d direct"};
+ LOG_INFO("%s auto: %s -> off (compute devices disagree)", names[static_cast<size_t>(operation)], sd_backend_module_name(module));
+ }
+ return enabled;
+ }
+
+ void ModelRuntime::configure_runner(GGMLRunner& runner, bool conv_direct) {
+ runner.set_scale_overrides(params.linear_scale, params.attn_scale, params.conv_scale);
+ runner.set_flash_attention_enabled(flash_attention_for());
+ if (conv_direct) {
+ runner.set_conv2d_direct_enabled(conv_direct_for(false));
+ runner.set_conv3d_direct_enabled(conv_direct_for(true));
+ }
+ }
+
+ bool ModelRuntime::rebuild() {
+ model_.model_weights->set_layer_split(false);
+ configure_weight_loading();
+ if (!build_runners_())
+ return false;
+
+ if (!model_.model_weights->validate_registered_tensors()) {
+ LOG_ERROR("model metadata validation failed");
+ return false;
+ }
+ model_.eager_load_supported_ = !split_active(SDSplitMode::LAYER);
+ model_.initialization_pending_ = true;
+ if (!model_.defer_initialization_ && !model_.finish_initialization()) {
+ return false;
+ }
+ const auto bytes = model_.model_weights->registered_params_size(registered_components_);
+ LOG_INFO("%s weights: %.2f MiB", model_component_name(component), bytes / (1024.0 * 1024.0));
+ catalog_revision_ = model_.model_weights->loader().revision();
+ ready_ = true;
+ return true;
+ }
+
+ bool ModelRuntime::attach_loras() {
+ if (model_.lora_mode() == LORA_APPLY_IMMEDIATELY && !apply_lora_immediately_)
+ throw std::runtime_error("immediately is unsupported by this model's weight layout");
+ if (!apply_lora_immediately_ &&
+ std::none_of(model_.runners_.begin(), model_.runners_.end(), [](const auto& runner) { return bool(runner.set_adapter); }))
+ throw std::runtime_error("at_runtime is unsupported by this model runner");
+
+ sd::ParallelScope tensor_scope(model_.tensor_executor.get());
+ if (!model_.prepare_loras(version, backend_manager, module, model_.n_threads, apply_lora_immediately_))
+ return false;
+ catalog_revision_ = model_.model_weights->loader().revision();
+ return true;
+ }
+
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<Conditioner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<DiffusionModelRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<::VAE>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<::ControlNet>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<ZImage::ControlNetRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<PhotoMakerIDEncoder>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<PuLIDRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<::AnimateDiff::MotionRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<H3VUpsampler::H3LatentUpscalerRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<LTXVUpsampler::LatentUpsamplerRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<::IPAdapter::IPAdapterRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<FrozenCLIPVisionEmbedder>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<Wav2Vec2::Wav2Vec2ModelRunner>&, size_t*);
+ template bool ModelRuntime::register_runner_params(ModelComponent, const std::shared_ptr<AudioVAERunner>&, size_t*);
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/model_runtime.h b/src/dgl/models/model_runtime.h
new file mode 100644
index 0000000..2b67943
--- /dev/null
+++ b/src/dgl/models/model_runtime.h
@@ -0,0 +1,73 @@
+#ifndef __SD_DGL_MODELS_MODEL_RUNTIME_H__
+#define __SD_DGL_MODELS_MODEL_RUNTIME_H__
+
+#include <functional>
+#include <optional>
+#include <set>
+
+#include "core/ggml_extend_backend.h"
+#include "dgl/models/model_base.h"
+#include "dgl/models/model_load_params.h"
+#include "model_build_context.h"
+
+struct GGMLRunner;
+enum class BackendProbeOperation;
+
+namespace sd::dgl::models {
+
+ class ModelRuntime {
+ using LoadCatalog = std::function<bool(ModelLoader&, const ModelLoadParams&, SDVersion&, FamilyProbeCache&)>;
+
+ Model& model_;
+ const std::function<bool()> build_runners_;
+ std::set<ModelComponent> registered_components_;
+ std::string backend_spec_;
+ std::string params_backend_spec_;
+ std::string split_mode_;
+ bool ready_ = false;
+ bool apply_lora_immediately_ = false;
+ uint64_t catalog_revision_ = 0;
+
+ bool rebuild();
+ void configure_weight_loading();
+ bool configure_backends();
+ bool split_active(SDSplitMode mode);
+ bool supports_operation(BackendProbeOperation operation);
+
+ template <typename T>
+ bool register_row_split_runner_params(ModelComponent component, const std::shared_ptr<T>& runner, const std::vector<ggml_backend_t>& backends, std::map<std::string, ggml_tensor*> tensors, const std::map<ggml_tensor*, enum ggml_op>& ops, ModelManager::ResidencyMode mode, size_t* size);
+
+ template <typename T>
+ bool register_layer_split_runner_params(ModelComponent component, const std::shared_ptr<T>& runner, const std::vector<ggml_backend_t>& backends, std::map<std::string, ggml_tensor*> tensors, const std::map<ggml_tensor*, enum ggml_op>& ops, ModelManager::ResidencyMode mode, size_t* size);
+
+ public:
+ const ModelComponent component;
+ const SDBackendModule module;
+ SDVersion version = VERSION_COUNT;
+ ModelLoadParams params;
+ SDBackendManager backend_manager;
+
+ ModelRuntime(Model& model, std::shared_ptr<SDBackendPool> backends, ModelComponent component, SDBackendModule module, std::function<bool()> build_runners);
+ ~ModelRuntime();
+
+ bool init(ModelLoadParams params, const LoadCatalog& load_catalog);
+ bool build_variant(const ModelRuntime& base, ModelLoader loader);
+ bool refresh_weights();
+ bool attach_loras();
+ bool set_params_placement(const std::string& name);
+
+ sd::model_builders::Context model_build_context();
+ bool flash_attention_for();
+ bool conv_direct_for(bool conv3d);
+ void configure_runner(GGMLRunner& runner, bool conv_direct = false);
+ bool ensure_backend_pair();
+ ggml_backend_t backend_for();
+ ggml_backend_t params_backend_for();
+
+ template <typename T>
+ bool register_runner_params(ModelComponent component, const std::shared_ptr<T>& runner, size_t* size = nullptr);
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_MODEL_RUNTIME_H__
diff --git a/src/dgl/models/photomaker.cpp b/src/dgl/models/photomaker.cpp
new file mode 100644
index 0000000..c833552
--- /dev/null
+++ b/src/dgl/models/photomaker.cpp
@@ -0,0 +1,71 @@
+#include "dgl/models/photomaker.h"
+
+#include <algorithm>
+
+#include "core/image_tensor.h"
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+#include "extensions/photomaker_embedding.h"
+#include "model/adapter/pmid.hpp"
+
+namespace sd::dgl::models {
+ PhotoMaker::PhotoMaker(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::PhotoMaker, SDBackendModule::PHOTOMAKER, [this] { return build_runners(); }) {}
+
+ bool PhotoMaker::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::PhotoMaker>);
+ }
+
+ bool PhotoMaker::build_runners() {
+ if (!sd_version_is_sdxl(version()))
+ throw std::runtime_error("PhotoMaker requires the SDXL family");
+ const auto& weights = model_weights->loader().get_tensor_storage_map();
+ const bool v2 = std::any_of(weights.begin(), weights.end(), [](const auto& entry) {
+ return starts_with(entry.first, "pmid.qformer_perceiver.");
+ });
+ runner_ = std::make_shared<PhotoMakerIDEncoder>(runtime_.backend_for(), weights, "pmid", version(),
+ v2 ? PM_VERSION_2 : PM_VERSION_1, 0.f, model_weights);
+ runner_->set_scale_overrides(runtime_.params.linear_scale,
+ runtime_.params.attn_scale);
+ return runtime_.register_runner_params(runtime_.component, runner_);
+ }
+
+ Tensor<float> PhotoMaker::project_images(const std::vector<Tensor<float>>& images, const Tensor<float>* embedding) {
+ const bool v2 = runner_->get_version() == PM_VERSION_2;
+ const int64_t count = images.size();
+ if (count == 0 || count > 75 / (v2 ? 2 : 1))
+ throw std::runtime_error("PhotoMaker image count exceeds the available prompt slots");
+ if (v2 && (!embedding || embedding->shape() != std::vector<int64_t>{sd::photomaker::id_embedding_width, count}))
+ throw std::runtime_error("PhotoMaker v2 requires a float32 embedding with 512 values per image");
+ if (!v2 && embedding)
+ throw std::runtime_error("PhotoMaker identity embeddings require v2");
+
+ constexpr int64_t image_size = 224;
+ Tensor<float> pixels({image_size, image_size, 3, count});
+ auto target = pixels.values().begin();
+ for (const auto& image : images) {
+ if (image.dim() != 4 || image.shape()[3] != 1 || image.shape()[2] == 2)
+ throw std::runtime_error("PhotoMaker expects individual grayscale, RGB or RGBA images");
+ const auto preprocessed = clip_preprocess(image.shape()[2] == 3 ? image : sd::ensure_image_tensor_channels(image, 3),
+ image_size, image_size);
+ target = std::copy(preprocessed.values().begin(), preprocessed.values().end(), target);
+ }
+ static const Tensor<float> empty;
+ return runner_->compute_tokens(n_threads, pixels, embedding ? *embedding : empty);
+ }
+
+ Tensor<float> PhotoMaker::fuse_prompt(const Tensor<float>& prompt, const Tensor<float>& tokens, const Tensor<uint8_t>& mask) {
+ const bool single_prompt = prompt.dim() == 2 || (prompt.dim() == 3 && prompt.shape()[2] == 1);
+ if (tokens.dim() != 2 || tokens.shape()[0] != 2048 || tokens.empty() ||
+ !single_prompt || prompt.shape()[0] != 2048 || mask.dim() != 1 || prompt.shape()[1] != mask.numel())
+ throw std::runtime_error("PhotoMaker requires SDXL text and identity tokens and one mask entry per text token");
+ auto begin = std::find(mask.values().begin(), mask.values().end(), uint8_t{1});
+ const auto slots = tokens.shape()[1];
+ if (mask.values().end() - begin < slots ||
+ !std::all_of(begin, begin + slots, [](uint8_t value) { return value == 1; }) || !std::all_of(mask.values().begin(), begin, [](uint8_t value) { return value == 0; }) || !std::all_of(begin + slots, mask.values().end(), [](uint8_t value) { return value == 0; }))
+ throw std::runtime_error("PhotoMaker mask requires one contiguous slot for each identity token");
+ std::vector<bool> selected(mask.values().begin(), mask.values().end());
+ return runner_->compute(n_threads, {}, prompt, tokens, selected);
+ }
+}
diff --git a/src/dgl/models/photomaker.h b/src/dgl/models/photomaker.h
new file mode 100644
index 0000000..c3b857e
--- /dev/null
+++ b/src/dgl/models/photomaker.h
@@ -0,0 +1,32 @@
+#ifndef __SD_DGL_MODELS_PHOTOMAKER_H__
+#define __SD_DGL_MODELS_PHOTOMAKER_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+struct PhotoMakerIDEncoder;
+
+namespace sd::dgl::models {
+ class PhotoMaker final : public Model {
+ ModelRuntime runtime_;
+ std::shared_ptr<PhotoMakerIDEncoder> runner_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ return runtime_.build_variant(static_cast<const PhotoMaker&>(source).runtime_, std::move(loader));
+ }
+
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ PhotoMaker(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(ModelLoadParams params);
+ Tensor<float> project_images(const std::vector<Tensor<float>>& images, const Tensor<float>* embedding);
+ Tensor<float> fuse_prompt(const Tensor<float>& prompt, const Tensor<float>& tokens, const Tensor<uint8_t>& mask);
+ };
+}
+
+#endif // __SD_DGL_MODELS_PHOTOMAKER_H__
diff --git a/src/dgl/models/photomaker_prompt.h b/src/dgl/models/photomaker_prompt.h
new file mode 100644
index 0000000..ad55692
--- /dev/null
+++ b/src/dgl/models/photomaker_prompt.h
@@ -0,0 +1,14 @@
+#ifndef __SD_DGL_MODELS_PHOTOMAKER_PROMPT_H__
+#define __SD_DGL_MODELS_PHOTOMAKER_PROMPT_H__
+
+#include "conditioning/conditioner.hpp"
+
+namespace sd::dgl::models {
+ struct PhotoMakerPrompt {
+ SDCondition plain;
+ SDCondition condition;
+ sd::Tensor<uint8_t> mask;
+ };
+}
+
+#endif // __SD_DGL_MODELS_PHOTOMAKER_PROMPT_H__
diff --git a/src/dgl/models/pulid.cpp b/src/dgl/models/pulid.cpp
new file mode 100644
index 0000000..a8180f0
--- /dev/null
+++ b/src/dgl/models/pulid.cpp
@@ -0,0 +1,57 @@
+#include "dgl/models/pulid.h"
+
+#include "dgl/models/component_sources.h"
+#include "dgl/models/dit.h"
+#include "model/adapter/pulid.hpp"
+#include "model/diffusion/flux.hpp"
+
+namespace sd::dgl::models {
+ PuLID::PuLID(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::PuLID, SDBackendModule::DIFFUSION, [this] { return build_runners(); }) {}
+
+ bool PuLID::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::PuLID>);
+ }
+
+ bool PuLID::build_variant(const Model& source, ModelLoader loader) {
+ return runtime_.build_variant(static_cast<const PuLID&>(source).runtime_, std::move(loader));
+ }
+
+ bool PuLID::build_runners() {
+ if (runtime_.version != VERSION_FLUX)
+ throw std::runtime_error("PuLID requires family = \"flux\"");
+ runner_ = std::make_shared<PuLIDRunner>(runtime_.backend_for(), model_weights->loader().get_tensor_storage_map(), model_weights);
+ runner_->set_scale_overrides(runtime_.params.linear_scale,
+ runtime_.params.attn_scale);
+ return runtime_.register_runner_params(runtime_.component, runner_);
+ }
+
+ void PuLID::validate_target(const DiT& target) const {
+ const auto* flux = dynamic_cast<const Flux::FluxRunner*>(target.diffusion_model.get());
+ if (target.version() != VERSION_FLUX || !flux ||
+ flux->config.depth != PuLIDRunner::double_blocks || flux->config.depth_single_blocks != PuLIDRunner::single_blocks ||
+ flux->config.hidden_size != PuLIDPerceiverAttentionCA::DEFAULT_DIM)
+ throw std::runtime_error("PuLID requires a matching FLUX.1 denoiser");
+ if (runner_->compute_backend() != flux->compute_backend())
+ throw std::runtime_error("PuLID and the denoiser require the same compute backend");
+ }
+
+ void PuLID::validate_embedding(const Tensor<float>& embedding) const {
+ if (embedding.empty() || embedding.dim() != 2 || embedding.shape()[0] != PuLIDPerceiverAttentionCA::DEFAULT_KV_DIM)
+ throw std::runtime_error("PuLID identity must have shape [2048, tokens]");
+ }
+
+ std::vector<Tensor<float>> PuLID::compute_kv(const Tensor<float>& embedding) {
+ return runner_->compute_kv(n_threads, embedding);
+ }
+
+ size_t PuLID::extra_graph_size() const {
+ return runner_->extra_graph_size();
+ }
+
+ ggml_tensor* PuLID::attend(GGMLRunnerContext& ctx, size_t layer, const Tensor<float>& kv, ggml_tensor* image) const {
+ return runner_->attend(ctx, layer, kv, image);
+ }
+
+}
diff --git a/src/dgl/models/pulid.h b/src/dgl/models/pulid.h
new file mode 100644
index 0000000..9ba4d2e
--- /dev/null
+++ b/src/dgl/models/pulid.h
@@ -0,0 +1,32 @@
+#ifndef __SD_DGL_MODELS_PULID_H__
+#define __SD_DGL_MODELS_PULID_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+class PuLIDRunner;
+struct GGMLRunnerContext;
+
+namespace sd::dgl::models {
+ class PuLID final : public Model {
+ ModelRuntime runtime_;
+ std::shared_ptr<PuLIDRunner> runner_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override;
+ bool build_runners();
+
+ public:
+ PuLID(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(ModelLoadParams params);
+ void validate_target(const DiT& target) const;
+ void validate_embedding(const Tensor<float>& embedding) const;
+ std::vector<Tensor<float>> compute_kv(const Tensor<float>& embedding);
+ size_t extra_graph_size() const;
+ ggml_tensor* attend(GGMLRunnerContext& ctx, size_t layer, const Tensor<float>& kv, ggml_tensor* image) const;
+ };
+}
+
+#endif // __SD_DGL_MODELS_PULID_H__
diff --git a/src/dgl/models/relay_vsr.cpp b/src/dgl/models/relay_vsr.cpp
new file mode 100644
index 0000000..b306b59
--- /dev/null
+++ b/src/dgl/models/relay_vsr.cpp
@@ -0,0 +1,20 @@
+#include "dgl/models/relay_vsr.h"
+
+#include "dgl/models/vae.h"
+#include "json.hpp"
+#include "model/diffusion/relay_vsr.hpp"
+#include "model/upscaler/flash_vsr.hpp"
+#include "model_loader.h"
+
+namespace sd::dgl::models {
+ std::shared_ptr<::VAE> build_relay_vsr_vae(const model_builders::Context& ctx, const ModelLoader& loader) {
+ const auto& metadata = loader.get_metadata();
+ const auto config = metadata.find("config");
+ if (config == metadata.end())
+ throw std::runtime_error("RelayVSR Flash weights require upscale_factor metadata");
+ const auto scale = nlohmann::json::parse(config->second).at("upscale_factor").get<int>();
+ return std::make_shared<RelayVSR::FlashVSRRunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
+ ctx.tensor_storage_map, scale, ctx.weight_manager);
+ }
+
+}
diff --git a/src/dgl/models/relay_vsr.h b/src/dgl/models/relay_vsr.h
new file mode 100644
index 0000000..d1a928a
--- /dev/null
+++ b/src/dgl/models/relay_vsr.h
@@ -0,0 +1,13 @@
+#ifndef __SD_DGL_MODELS_RELAY_VSR_H__
+#define __SD_DGL_MODELS_RELAY_VSR_H__
+
+#include "core/tensor.hpp"
+#include "model_build_context.h"
+#include "model_component.h"
+#include "model_manager.h"
+
+namespace sd::dgl::models {
+ std::shared_ptr<::VAE> build_relay_vsr_vae(const model_builders::Context& ctx, const ModelLoader& loader);
+}
+
+#endif
diff --git a/src/dgl/models/text_encoder.cpp b/src/dgl/models/text_encoder.cpp
new file mode 100644
index 0000000..fd19ef0
--- /dev/null
+++ b/src/dgl/models/text_encoder.cpp
@@ -0,0 +1,112 @@
+#include "dgl/models/text_encoder.h"
+
+#include <algorithm>
+
+#include "conditioning/conditioner.hpp"
+#include "conditioning/conditioning_cache.h"
+#include "conditioning/photomaker_prompt.h"
+#include "core/util.h"
+#include "dgl/models/component_builders.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/models/photomaker_prompt.h"
+
+namespace sd::dgl::models {
+ TextEncoder::TextEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::Conditioner, SDBackendModule::TE, [this] { return build_runners(); }),
+ conditioning_cache_(std::make_unique<ConditioningCache>()) {}
+
+ TextEncoder::~TextEncoder() = default;
+
+ bool TextEncoder::init(TextEncoderLoadParams params) {
+ tokenizer_ = std::move(params.tokenizer);
+ WeightSources sources{
+ {params.connectors_path, "embeddings_connectors."},
+ {params.vision_path, "text_encoders.llm.visual."},
+ {params.projection_path, ""},
+ {params.t5_path, "text_encoders.t5xxl.transformer."},
+ {params.clip_l_path, "clip_l."},
+ {params.clip_g_path, "clip_g."},
+ };
+ return runtime_.init(std::move(params.model),
+ [&sources](ModelLoader& loader, const ModelLoadParams& model, SDVersion& family, FamilyProbeCache& family_probes) {
+ return init_component_loader(loader, model, ModelComponent::Conditioner, family, sources, &family_probes);
+ });
+ }
+
+ bool TextEncoder::build_runners() {
+ auto context = runtime_.model_build_context();
+ context.build_conditioner = true;
+ context.build_diffusion = false;
+ context.build_auxiliary = false;
+ sd::model_builders::CoreRunners runners;
+ if (!component_builders::build_family_runners(context, runners, tokenizer_) || !runners.conditioner) {
+ return false;
+ }
+ cond_stage_model = std::move(runners.conditioner);
+ cond_stage_model->set_flash_attention_enabled(runtime_.flash_attention_for());
+ return runtime_.register_runner_params(runtime_.component, cond_stage_model);
+ }
+
+ void TextEncoder::invalidate_cached_state() {
+ conditioning_cache_->clear();
+ }
+
+ void TextEncoder::set_conditioning_cache_size(size_t capacity) {
+ conditioning_cache_->set_capacity(capacity);
+ }
+
+ SDCondition TextEncoder::get_learned_condition(const ConditionerParams& params) {
+ if (params.raw && params.system_prompt)
+ throw std::runtime_error("raw prompt encoding cannot be combined with system");
+ if (params.raw && !cond_stage_model->supports_prompt_formatting())
+ throw std::runtime_error("this text encoder does not accept raw prompt encoding");
+ if (params.system_prompt && !cond_stage_model->supports_prompt_formatting())
+ throw std::runtime_error("this text encoder does not accept a system prompt");
+ auto resolved = params;
+ resolved.clip_skip = cond_stage_model->resolve_clip_skip(params.clip_skip);
+ // Wan attends all UMT5 slots, so padding must contain zero embeddings.
+ if (sd_version_is_wan(version()) || sd_version_is_lingbot_video(version()))
+ resolved.zero_out_masked = true;
+ return conditioning_cache_->get(*cond_stage_model, n_threads, resolved);
+ }
+
+ bool TextEncoder::supports_textual_inversion() const {
+ return dynamic_cast<FrozenCLIPEmbedderWithCustomWords*>(cond_stage_model.get()) != nullptr;
+ }
+
+ bool TextEncoder::supports_prompt_images() const {
+ return cond_stage_model->supports_prompt_images();
+ }
+
+ size_t TextEncoder::max_reference_images() const {
+ return cond_stage_model->max_reference_images();
+ }
+
+ bool TextEncoder::supports_multimodal_prompt() const {
+ return cond_stage_model->supports_multimodal_prompt();
+ }
+
+ PhotoMakerPrompt TextEncoder::encode_photomaker_prompt(ConditionerParams params, const Tensor<float>& tokens) {
+ if (tokens.empty() || tokens.dim() != 2 || tokens.shape()[0] != 2048 || tokens.shape()[1] > 75)
+ throw std::runtime_error("PhotoMaker tokens must have shape [2048, slots] with up to 75 slots");
+ Model::ExecutionScope scope(*this);
+ scope.require_ready();
+ auto* clip = dynamic_cast<FrozenCLIPEmbedderWithCustomWords*>(cond_stage_model.get());
+ if (!sd_version_is_sdxl(version()) || !clip)
+ throw std::runtime_error("PhotoMaker requires an SDXL CLIP text encoder");
+ params.clip_skip = clip->resolve_clip_skip(params.clip_skip);
+ auto [encoded, slots] = sd::photomaker::encode_prompt(*clip, n_threads, params, "img", int(tokens.shape()[1]));
+ if (encoded.empty() || std::count(slots.begin(), slots.end(), true) != tokens.shape()[1])
+ throw std::runtime_error("PhotoMaker prompt requires an 'img' trigger after a class word and enough slots for all identity tokens");
+ std::string plain_text;
+ if (!sd::photomaker::remove_trigger(*clip, params.text, "img", plain_text))
+ throw std::runtime_error("cannot remove the PhotoMaker prompt trigger");
+ params.text = std::move(plain_text);
+ auto plain = get_learned_condition(params);
+ if (plain.empty())
+ throw std::runtime_error("PhotoMaker plain prompt encoding failed");
+ auto mask = sd::Tensor<uint8_t>::from_vector(std::vector<uint8_t>(slots.begin(), slots.end()));
+ return {std::move(plain), std::move(encoded), std::move(mask)};
+ }
+}
diff --git a/src/dgl/models/text_encoder.h b/src/dgl/models/text_encoder.h
new file mode 100644
index 0000000..487d6e2
--- /dev/null
+++ b/src/dgl/models/text_encoder.h
@@ -0,0 +1,59 @@
+#ifndef __SD_DGL_MODELS_TEXT_ENCODER_H__
+#define __SD_DGL_MODELS_TEXT_ENCODER_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+#include "tokenizers/tokenizer_config.h"
+
+class ConditioningCache;
+struct ConditionerParams;
+struct SDCondition;
+
+namespace sd::dgl::models {
+ struct TextEncoderLoadParams {
+ ModelLoadParams model;
+ TokenizerConfig tokenizer;
+ std::string vision_path;
+ std::string projection_path;
+ std::string connectors_path;
+ std::string t5_path;
+ std::string clip_l_path;
+ std::string clip_g_path;
+ };
+
+ class TextEncoder final : public Model {
+ ModelRuntime runtime_;
+ TokenizerConfig tokenizer_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ const auto& base = static_cast<const TextEncoder&>(source);
+ tokenizer_ = base.tokenizer_;
+ return runtime_.build_variant(base.runtime_, std::move(loader));
+ }
+
+ std::unique_ptr<ConditioningCache> conditioning_cache_;
+
+ bool build_runners();
+ void invalidate_cached_state() override;
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ std::shared_ptr<Conditioner> cond_stage_model;
+
+ TextEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ ~TextEncoder() override;
+ bool init(TextEncoderLoadParams params);
+ void set_conditioning_cache_size(size_t capacity);
+ bool supports_prompt_images() const;
+ size_t max_reference_images() const;
+ bool supports_multimodal_prompt() const;
+ bool supports_textual_inversion() const;
+ SDCondition get_learned_condition(const ConditionerParams& params);
+ PhotoMakerPrompt encode_photomaker_prompt(ConditionerParams params, const Tensor<float>& tokens);
+ };
+}
+
+#endif // __SD_DGL_MODELS_TEXT_ENCODER_H__
diff --git a/src/dgl/models/upscaler.cpp b/src/dgl/models/upscaler.cpp
new file mode 100644
index 0000000..39ba9a6
--- /dev/null
+++ b/src/dgl/models/upscaler.cpp
@@ -0,0 +1,111 @@
+#include "dgl/models/upscaler.h"
+
+#include <cstdlib>
+#include <utility>
+
+#include "core/ggml_extend_backend.h"
+#include "core/util.h"
+#include "model/upscaler/esrgan.hpp"
+#include "model/upscaler/mosr.hpp"
+#include "model_loader.h"
+#include "stable-diffusion.h"
+
+namespace sd::dgl::models {
+
+ ImageUpscaler::ImageUpscaler(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)), backend_manager(std::move(backends)) {
+ params_placement_ = [this](const std::string& name) {
+ std::string error;
+ if (!backend_manager.init(params_.backend.c_str(), name.c_str(), nullptr, &error))
+ throw std::runtime_error(error);
+ params_.params_backend = name;
+ return true;
+ };
+ }
+
+ ImageUpscaler::~ImageUpscaler() {
+ model.reset();
+ release_weights();
+ }
+
+ int ImageUpscaler::get_scale() {
+ return model != nullptr ? model->scale() : 1;
+ }
+
+ bool ImageUpscaler::init(ImageUpscalerParams params) {
+ params_ = std::move(params);
+ ggml_log_set(sd_ggml_log_callback, nullptr);
+ ModelLoader loader;
+ if (!loader.init_from_file_and_convert_name(params_.path, "", VERSION_ESRGAN)) {
+ LOG_ERROR("init upscaler model loader failed: '%s'", params_.path.c_str());
+ return false;
+ }
+ loader.set_wtype_override(model_data_type);
+ return build(std::move(loader));
+ }
+
+ ModelLoader ImageUpscaler::base_loader() const {
+ return model_weights->model_source(model->source_file());
+ }
+
+ bool ImageUpscaler::build_variant(const Model& source, ModelLoader loader) {
+ const auto& base = static_cast<const ImageUpscaler&>(source);
+ params_ = base.params_;
+ model_data_type = base.model_data_type;
+ return build(std::move(loader));
+ }
+
+ bool ImageUpscaler::build(ModelLoader loader) {
+ sd::ParallelScope tensor_scope(tensor_executor.get());
+ std::string error;
+ if (!backend_manager.init(params_.backend.c_str(), params_.params_backend.c_str(), nullptr, &error)) {
+ LOG_ERROR("upscaler backend config failed: %s", error.c_str());
+ return false;
+ }
+ model = build_image_upscale_model(model_weights, backend_manager, std::move(loader), params_.path,
+ params_.direct, model_data_type);
+ return model != nullptr && load_weights_eagerly();
+ }
+
+ sd::Tensor<float> ImageUpscaler::upscale_tensor(const sd::Tensor<float>& input_tensor) {
+ Model::ExecutionScope scope(*this);
+ scope.require_ready();
+ if (model == nullptr) {
+ return {};
+ }
+ return upscale_image_tensor(*model, input_tensor, n_threads, params_.tile_size);
+ }
+
+ bool ImageUpscaler::attach_loras() {
+ return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::UPSCALER,
+ n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+ }
+
+ void ImageUpscaler::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
+ model->runner()->set_weight_adapter(adapter);
+ }
+
+ sd::Image ImageUpscaler::upscale(sd::Image input_image) {
+ sd::ParallelScope tensor_scope(tensor_executor.get());
+ sd::Image upscaled_image = {0, 0, 0, nullptr};
+ const int scale = get_scale();
+ int output_width = (int)input_image.width * scale;
+ int output_height = (int)input_image.height * scale;
+ LOG_INFO("upscaling from (%i x %i) to (%i x %i)",
+ input_image.width, input_image.height, output_width, output_height);
+
+ sd::Tensor<float> input_tensor = sd_image_to_tensor(input_image);
+ sd::Tensor<float> upscaled;
+ int64_t t0 = ggml_time_ms();
+ upscaled = upscale_tensor(input_tensor);
+ if (upscaled.empty()) {
+ return upscaled_image;
+ }
+ sd::Image upscaled_data = tensor_to_sd_image(upscaled);
+ int64_t t3 = ggml_time_ms();
+ LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f);
+ upscaled_image = upscaled_data;
+ return upscaled_image;
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/upscaler.h b/src/dgl/models/upscaler.h
new file mode 100644
index 0000000..5e7e7d0
--- /dev/null
+++ b/src/dgl/models/upscaler.h
@@ -0,0 +1,48 @@
+#ifndef __SD_DGL_MODELS_UPSCALER_H__
+#define __SD_DGL_MODELS_UPSCALER_H__
+
+#include "core/ggml_extend_backend.h"
+#include "core/media.h"
+#include "core/tensor.hpp"
+#include "dgl/models/image_upscale.h"
+#include "dgl/models/model_base.h"
+#include "stable-diffusion.h"
+
+#include <memory>
+#include <string>
+
+namespace sd::dgl::models {
+
+ struct ImageUpscalerParams {
+ std::string path;
+ std::string backend;
+ std::string params_backend;
+ bool direct;
+ int tile_size;
+ };
+
+ struct ImageUpscaler final : public Model {
+ SDBackendManager backend_manager;
+ ggml_type model_data_type = GGML_TYPE_F16;
+ std::unique_ptr<ImageUpscaleModel> model;
+
+ ImageUpscaler(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ ~ImageUpscaler() override;
+
+ bool init(ImageUpscalerParams params);
+ int get_scale();
+ sd::Tensor<float> upscale_tensor(const sd::Tensor<float>& input_tensor);
+ sd::Image upscale(sd::Image input_image);
+
+ private:
+ ImageUpscalerParams params_;
+ bool build(ModelLoader loader);
+ ModelLoader base_loader() const override;
+ bool build_variant(const Model& base, ModelLoader loader) override;
+ bool attach_loras() override;
+ void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override;
+ };
+
+} // namespace sd::dgl::models
+
+#endif // __SD_DGL_MODELS_UPSCALER_H__
diff --git a/src/dgl/models/vae.cpp b/src/dgl/models/vae.cpp
new file mode 100644
index 0000000..86726ab
--- /dev/null
+++ b/src/dgl/models/vae.cpp
@@ -0,0 +1,203 @@
+#include "dgl/models/vae.h"
+
+#include "model/vae/geometry.hpp"
+
+#include "core/backend_fit.h"
+#include "core/ggml_tensor_utils.h"
+#include "core/image_geometry.h"
+#include "core/util.h"
+#include "dgl/models/component_builders.h"
+#include "dgl/models/component_sources.h"
+#include "dgl/run_events.h"
+#include "model/vae/auto_encoder_kl.hpp"
+#include "model/vae/ltx_vae.hpp"
+#include "model/vae/vae.hpp"
+#include "model_definition.h"
+
+namespace sd::dgl::models {
+
+ VAE::VAE(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::VAE, SDBackendModule::VAE, [this] { return build_runners(); }) {}
+
+ int VAE::get_latent_channel() const {
+ return find_model_definition(version())->geometry.latent_channels;
+ }
+
+ void VAE::apply_circular_axes(bool x, bool y) {
+ circular_x = x;
+ circular_y = y;
+ first_stage_model->set_circular_axes(x, y);
+ if (x || y)
+ LOG_DEBUG("Using circular padding for convolutions (x=%s, y=%s)", x ? "true" : "false", y ? "true" : "false");
+ }
+
+ bool VAE::init(VAELoadParams params) {
+ format_ = params.format;
+ return runtime_.init(std::move(params.model), load_component_catalog<ModelComponent::VAE>);
+ }
+
+ static bool load_tae_catalog(ModelLoader& loader, const ModelLoadParams& params, SDVersion& family, FamilyProbeCache&) {
+ if (family == VERSION_COUNT)
+ throw std::runtime_error("load_tae requires family or family_from");
+ if (!loader.init_from_file_and_convert_name(params.path, "tae.", family))
+ return false;
+ loader.set_wtype_override(sd_type_to_ggml_type(params.wtype), params.tensor_type_rules.c_str());
+ return true;
+ }
+
+ bool VAE::init_tae(ModelLoadParams params, bool decode_only) {
+ tae_ = true;
+ decode_only_ = decode_only;
+ return runtime_.init(std::move(params), load_tae_catalog);
+ }
+
+ bool VAE::build_variant(const Model& source, ModelLoader loader) {
+ const auto& base = static_cast<const VAE&>(source);
+ format_ = base.format_;
+ tae_ = base.tae_;
+ decode_only_ = base.decode_only_;
+ return runtime_.build_variant(base.runtime_, std::move(loader));
+ }
+
+ bool VAE::build_runners() {
+ auto ctx = runtime_.model_build_context();
+ ctx.params.vae_format = format_;
+ first_stage_model = uses_tae() ? component_builders::build_tae_runner(ctx, decode_only_)
+ : component_builders::build_vae_runner(ctx, model_weights->loader());
+ if (!first_stage_model)
+ return false;
+ runtime_.configure_runner(*first_stage_model, true);
+ return runtime_.register_runner_params(runtime_.component, first_stage_model);
+ }
+
+ bool VAE::uses_tae() const {
+ return tae_ || version() == VERSION_SDXS_512_DS || version() == VERSION_SDXS_09;
+ }
+
+ int VAE::get_vae_scale_factor() {
+ if (sd_version_is_pid(version())) {
+ return 1;
+ }
+ return first_stage_model->get_scale_factor();
+ }
+
+ int VAE::get_image_channels() const {
+ return find_model_definition(version())->geometry.image_channels;
+ }
+
+ int VAE::video_frames_to_latent_frames(int frames) {
+ int latent_frames = frames;
+ if (sd_version_is_ltxav(version())) {
+ latent_frames = ((frames - 1) / 8) + 1;
+ } else if (sd_version_is_minimax_h3(version())) {
+ latent_frames = frames <= 5 ? 2 : ((frames - 5) / 17) * 5 + 2;
+ } else if (sd_version_is_wan(version()) || sd_version_is_lingbot_video(version()) || sd_version_is_hunyuan_video(version()) || version() == VERSION_SEEDVR2) {
+ latent_frames = ((frames - 1) / 4) + 1;
+ }
+ return latent_frames;
+ }
+
+ int VAE::align_video_frames(int frames) {
+ return sd::vae::aligned_frames(version(), frames);
+ }
+
+ std::vector<int64_t> VAE::latent_shape(int64_t width, int64_t height, int frames, int batch) {
+ if (frames < 0 || batch <= 0)
+ throw std::invalid_argument("invalid latent frame or batch count");
+ const auto scale = get_vae_scale_factor();
+ const auto multiple = scale * find_model_definition(version())->geometry.diffusion_down_factor;
+ const auto aligned_width = sd::align_image_dimension(width, multiple);
+ const auto aligned_height = sd::align_image_dimension(height, multiple);
+ if (width != aligned_width || height != aligned_height)
+ LOG_VERBOSE("latent_shape: aligning pixel dimensions from %lldx%lld to %lldx%lld", (long long)width, (long long)height,
+ (long long)aligned_width, (long long)aligned_height);
+ std::vector<int64_t> shape{aligned_width / scale, aligned_height / scale};
+ if (frames) {
+ const auto aligned_frames = align_video_frames(frames);
+ if (aligned_frames != frames)
+ LOG_VERBOSE("latent_shape: aligning frame count from %d to %d", frames, aligned_frames);
+ shape.push_back(video_frames_to_latent_frames(aligned_frames));
+ }
+ shape.push_back(get_latent_channel());
+ shape.push_back(batch);
+ return shape;
+ }
+
+ void VAE::validate_vae_latent(const Tensor<float>& latent) const {
+ const auto* kl = dynamic_cast<const AutoEncoderKL*>(first_stage_model.get());
+ const auto* ltx = dynamic_cast<const LTXVideoVAE*>(first_stage_model.get());
+ const auto rank = latent.dim();
+ const auto channels = kl ? kl->ae.get_latent_channels() : get_latent_channel();
+ if ((rank != 4 && rank != 5) || (kl && rank != 4) || (ltx && rank != 5) ||
+ latent.shape()[rank == 5 ? 3 : 2] != channels ||
+ (rank == 5 && latent.shape().back() != 1)) {
+ throw std::runtime_error("latent shape does not match the VAE");
+ }
+ }
+
+ sd::Tensor<float> VAE::encode_vae_latents(const sd::Tensor<float>& x, bool posterior_mode, const std::shared_ptr<RNG>& rng) {
+ if (decode_only_)
+ throw std::runtime_error("this TAE was loaded with decode_only and cannot encode");
+ report_run_progress("vae_encode");
+ // Qwen 2.1 expands each vision slot into four reference latent tokens.
+ const auto alignment = version() == VERSION_QWEN_IMAGE_2_1 ? 32 : get_vae_scale_factor();
+ auto adjusted = sd::fit_image_tensor(x, alignment, "vae_encode");
+ const auto* input = adjusted.empty() ? &x : &adjusted;
+ if (input->dim() == 5 && input->shape()[2] > 1 && !sd_version_is_minimax_h3_image(version())) {
+ const auto frames = align_video_frames(int(input->shape()[2]));
+ if (frames != input->shape()[2]) {
+ LOG_VERBOSE("vae_encode: padding video from %lld to %d frames by repeating the last frame", (long long)input->shape()[2], frames);
+ adjusted = sd::pad_video_tensor(*input, frames);
+ input = &adjusted;
+ }
+ }
+ auto latents = first_stage_model->encode(n_threads, *input, vae_tiling_params, circular_x, circular_y);
+ if (latents.empty()) {
+ return {};
+ }
+ return first_stage_model->vae_output_to_latents(latents, rng,
+ posterior_mode ? VAEPosterior::Mode : VAEPosterior::Sample,
+ true);
+ }
+
+ sd::Tensor<float> VAE::vae_to_dit_latents(const sd::Tensor<float>& latent) {
+ auto* ltx = dynamic_cast<LTXVideoVAE*>(first_stage_model.get());
+ return ltx ? ltx->normalize_latents(n_threads, latent)
+ : first_stage_model->vae_to_diffusion_latents(latent);
+ }
+
+ sd::Tensor<float> VAE::dit_to_vae_latents(const sd::Tensor<float>& latent) {
+ auto* ltx = dynamic_cast<LTXVideoVAE*>(first_stage_model.get());
+ return ltx ? ltx->un_normalize_latents(n_threads, latent)
+ : first_stage_model->diffusion_to_vae_latents(latent);
+ }
+
+ sd::Tensor<float> VAE::decode_vae_latents(const sd::Tensor<float>& latents, bool decode_video) {
+ report_run_progress("vae_decode");
+ auto tiling_params = first_stage_model->resolve_tiling_params(vae_tiling_params);
+ const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 &&
+ first_stage_model->can_temporal_tile_decode();
+ for (;;) {
+ int tile_size_w = static_cast<int>(latents.shape()[0]);
+ int tile_size_h = static_cast<int>(latents.shape()[1]);
+ float tile_overlap;
+ if (tiling_params.enabled &&
+ !first_stage_model->get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params,
+ latents.shape()[0], latents.shape()[1])) {
+ return {};
+ }
+ auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y);
+ if (sd_version_is_minimax_h3_image(version()) && !decoded.empty() && decoded.dim() == 5) {
+ decoded = decoded.squeeze(2);
+ }
+ if (!decoded.empty() ||
+ !sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling,
+ first_stage_model->last_compute_status(),
+ tile_size_w, tile_size_h, first_stage_model->get_scale_factor())) {
+ return decoded;
+ }
+ }
+ }
+
+} // namespace sd::dgl::models
diff --git a/src/dgl/models/vae.h b/src/dgl/models/vae.h
new file mode 100644
index 0000000..8ffbb47
--- /dev/null
+++ b/src/dgl/models/vae.h
@@ -0,0 +1,55 @@
+#ifndef __SD_DGL_MODELS_VAE_H__
+#define __SD_DGL_MODELS_VAE_H__
+
+#include "config.h"
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+class RNG;
+
+namespace sd::dgl::models {
+ struct VAELoadParams {
+ ModelLoadParams model;
+ sd_vae_format_t format = SD_VAE_FORMAT_AUTO;
+ };
+
+ class VAE final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override;
+
+ bool build_runners();
+ bool uses_tae() const;
+ bool tae_ = false;
+ bool decode_only_ = false;
+ sd_vae_format_t format_ = SD_VAE_FORMAT_AUTO;
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ int get_latent_channel() const;
+ bool circular_x = false;
+ bool circular_y = false;
+ void apply_circular_axes(bool x, bool y);
+ std::shared_ptr<::VAE> first_stage_model;
+ sd::TilingConfig vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
+
+ VAE(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ bool init(VAELoadParams params);
+ bool init_tae(ModelLoadParams params, bool decode_only);
+ int get_image_channels() const;
+ int get_vae_scale_factor();
+ std::vector<int64_t> latent_shape(int64_t width, int64_t height, int frames = 0, int batch = 1);
+ int video_frames_to_latent_frames(int frames);
+ int align_video_frames(int frames);
+ void validate_vae_latent(const Tensor<float>& latent) const;
+ Tensor<float> encode_vae_latents(const Tensor<float>& input, bool posterior_mode, const std::shared_ptr<RNG>& rng);
+ Tensor<float> vae_to_dit_latents(const Tensor<float>& latent);
+ Tensor<float> dit_to_vae_latents(const Tensor<float>& latent);
+ Tensor<float> decode_vae_latents(const Tensor<float>& latent, bool decode_video);
+ };
+}
+
+#endif // __SD_DGL_MODELS_VAE_H__
diff --git a/src/dgl/models/vision_encoder.cpp b/src/dgl/models/vision_encoder.cpp
new file mode 100644
index 0000000..e8dcfb8
--- /dev/null
+++ b/src/dgl/models/vision_encoder.cpp
@@ -0,0 +1,49 @@
+#include "dgl/models/vision_encoder.h"
+
+#include <algorithm>
+
+#include "conditioning/conditioner.hpp"
+#include "core/util.h"
+#include "dgl/models/component_sources.h"
+
+namespace sd::dgl::models {
+ VisionEncoder::VisionEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends)
+ : Model(std::move(manager)),
+ runtime_(*this, std::move(backends), ModelComponent::CLIPVision, SDBackendModule::CLIP_VISION, [this] { return build_runners(); }) {}
+
+ bool VisionEncoder::init(ModelLoadParams params) {
+ return runtime_.init(std::move(params), load_component_catalog<ModelComponent::CLIPVision>);
+ }
+
+ bool VisionEncoder::build_runners() {
+ clip_vision = std::make_shared<FrozenCLIPVisionEmbedder>(runtime_.backend_for(), model_weights->loader().get_tensor_storage_map(), model_weights);
+ runtime_.configure_runner(*clip_vision);
+ return runtime_.register_runner_params(runtime_.component, clip_vision);
+ }
+
+ sd::Tensor<float> VisionEncoder::get_clip_vision_output(const sd::Tensor<float>& image,
+ bool return_pooled,
+ int clip_skip,
+ bool zero_out_masked,
+ bool zero_input) {
+ sd::Tensor<float> output;
+ if (zero_out_masked) {
+ if (return_pooled) {
+ output = sd::zeros<float>({clip_vision->vision_model.projection_dim});
+ } else {
+ output = sd::zeros<float>({clip_vision->vision_model.hidden_size, 257});
+ }
+ } else {
+ auto pixel_values = clip_preprocess(image, clip_vision->vision_model.image_size, clip_vision->vision_model.image_size);
+ if (zero_input)
+ pixel_values.fill_(0.f);
+ auto output_opt = clip_vision->compute(n_threads, pixel_values, return_pooled, clip_skip);
+ if (output_opt.empty()) {
+ LOG_ERROR("clip_vision compute failed");
+ return {};
+ }
+ output = std::move(output_opt);
+ }
+ return output;
+ }
+}
diff --git a/src/dgl/models/vision_encoder.h b/src/dgl/models/vision_encoder.h
new file mode 100644
index 0000000..8c808c9
--- /dev/null
+++ b/src/dgl/models/vision_encoder.h
@@ -0,0 +1,29 @@
+#ifndef __SD_DGL_MODELS_VISION_ENCODER_H__
+#define __SD_DGL_MODELS_VISION_ENCODER_H__
+
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+#include "dgl/models/model_runtime.h"
+
+namespace sd::dgl::models {
+ class VisionEncoder final : public Model {
+ ModelRuntime runtime_;
+
+ bool refresh_weights() override { return runtime_.refresh_weights(); }
+ bool attach_loras() override { return runtime_.attach_loras(); }
+ bool build_variant(const Model& source, ModelLoader loader) override {
+ return runtime_.build_variant(static_cast<const VisionEncoder&>(source).runtime_, std::move(loader));
+ }
+
+ std::shared_ptr<FrozenCLIPVisionEmbedder> clip_vision;
+ bool build_runners();
+
+ public:
+ SDVersion version() const { return runtime_.version; }
+ bool init(ModelLoadParams params);
+ VisionEncoder(std::shared_ptr<ModelManager> manager, std::shared_ptr<SDBackendPool> backends);
+ Tensor<float> get_clip_vision_output(const Tensor<float>& image, bool return_pooled, int clip_skip, bool zero_out_masked, bool zero_input);
+ };
+}
+
+#endif // __SD_DGL_MODELS_VISION_ENCODER_H__
diff --git a/src/dgl/ops/audio.cpp b/src/dgl/ops/audio.cpp
new file mode 100644
index 0000000..b4055c0
--- /dev/null
+++ b/src/dgl/ops/audio.cpp
@@ -0,0 +1,181 @@
+#include "dgl/ops/audio.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cmath>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+
+#include "core/file_access.h"
+
+namespace sd::dgl::ops {
+ sd::Audio audio_view(const sd_tensor_t& tensor, int64_t sample_rate) {
+ if (tensor.type != SD_TENSOR_F32 || tensor.shape().size() != 2 ||
+ tensor.shape()[0] < 1 || tensor.shape()[0] > 32 || tensor.shape()[1] < 1 || sample_rate < 1000 || sample_rate > 384000)
+ throw std::runtime_error("audio requires F32 [channels, samples] and a sample rate in [1000,384000]");
+ const auto* samples = static_cast<const float*>(tensor.data());
+ if (!std::all_of(samples, samples + tensor.bytes / sizeof(float), [](float x) { return std::isfinite(x); }))
+ throw std::runtime_error("audio samples must be finite");
+ sd::Audio view{};
+ view.data = const_cast<float*>(samples);
+ view.channels = uint32_t(tensor.shape()[0]);
+ view.sample_count = uint64_t(tensor.shape()[1]);
+ view.sample_rate = uint32_t(sample_rate);
+ return view;
+ }
+
+ AudioReadResult copy_audio(const sd::Audio& view) {
+ if (!view.data || !view.channels || view.channels > 32 || !view.sample_count || view.sample_count > 256 * 1024 * 1024 / (view.channels * sizeof(float)))
+ throw std::runtime_error("invalid audio or audio exceeds 256 MiB");
+ AudioReadResult audio;
+ audio.sample_rate = view.sample_rate;
+ audio.samples = make_tensor(SD_TENSOR_F32, std::vector<int64_t>{view.channels, int64_t(view.sample_count)}, view.data, size_t(view.sample_count) * view.channels * sizeof(float));
+ audio_view(*audio.samples, audio.sample_rate);
+ return audio;
+ }
+
+ static uint32_t audio_u32(const uint8_t* data) {
+ return uint32_t(data[0]) | (uint32_t(data[1]) << 8) | (uint32_t(data[2]) << 16) | (uint32_t(data[3]) << 24);
+ }
+ static uint16_t audio_u16(const uint8_t* data) {
+ return uint16_t(data[0]) | (uint16_t(data[1]) << 8);
+ }
+
+ AudioReadResult read_wav(const std::filesystem::path& path) {
+ require_file_access(path.u8string(), SD_DGL_FILE_READ);
+ std::ifstream file(path, std::ios::binary | std::ios::ate);
+ if (!file || file.tellg() < 12 || uint64_t(file.tellg()) > 256 * 1024 * 1024)
+ throw std::runtime_error("cannot read WAV file or file exceeds 256 MiB");
+ std::vector<uint8_t> bytes(size_t(file.tellg()));
+ file.seekg(0);
+ if (!file.read(reinterpret_cast<char*>(bytes.data()), std::streamsize(bytes.size())))
+ throw std::runtime_error("cannot read WAV bytes");
+ if (std::memcmp(bytes.data(), "RIFF", 4) || std::memcmp(bytes.data() + 8, "WAVE", 4) || uint64_t(audio_u32(bytes.data() + 4)) + 8 != bytes.size())
+ throw std::runtime_error("audio input requires a RIFF WAV file");
+ const uint8_t* format = nullptr;
+ const uint8_t* data = nullptr;
+ size_t count = 0;
+ for (size_t offset = 12; offset < bytes.size();) {
+ if (bytes.size() - offset < 8)
+ throw std::runtime_error("truncated WAV chunk");
+ uint64_t size = audio_u32(bytes.data() + offset + 4), padded = size + (size & 1);
+ if (padded > bytes.size() - offset - 8)
+ throw std::runtime_error("WAV chunk exceeds its container");
+ if (!std::memcmp(bytes.data() + offset, "fmt ", 4)) {
+ if (format || size < 16)
+ throw std::runtime_error("invalid WAV format chunk");
+ format = bytes.data() + offset + 8;
+ } else if (!std::memcmp(bytes.data() + offset, "data", 4)) {
+ if (data)
+ throw std::runtime_error("multiple WAV data chunks are unsupported");
+ data = bytes.data() + offset + 8;
+ count = size_t(size);
+ }
+ offset += 8 + size_t(padded);
+ }
+ if (!format || !data || !count)
+ throw std::runtime_error("WAV has no format or samples");
+ int encoding = audio_u16(format), channels = audio_u16(format + 2), bits = audio_u16(format + 14), block = audio_u16(format + 12);
+ uint32_t rate = audio_u32(format + 4);
+ if (channels < 1 || channels > 32 || rate < 1000 || rate > 384000 ||
+ !((encoding == 1 && (bits == 8 || bits == 16 || bits == 24 || bits == 32)) || (encoding == 3 && bits == 32)) || block != channels * (bits / 8) || count % block)
+ throw std::runtime_error("WAV requires interleaved PCM8/16/24/32 or float32 audio");
+ return decode_pcm_audio(data, count, encoding, bits, channels, rate);
+ }
+
+ AudioReadResult decode_pcm_audio(const uint8_t* data, size_t bytes, int encoding, int bits, int channels, int64_t rate) {
+ if (!data || !bytes || channels < 1 || channels > 32 || rate < 1000 || rate > 384000 ||
+ !((encoding == 1 && (bits == 8 || bits == 16 || bits == 24 || bits == 32)) || (encoding == 3 && bits == 32)) || bytes % (channels * (bits / 8)))
+ throw std::runtime_error("invalid PCM/float audio format or samples");
+ size_t samples = bytes / size_t(bits / 8);
+ if (samples > 256 * 1024 * 1024 / sizeof(float))
+ throw std::runtime_error("decoded WAV exceeds 256 MiB");
+ std::vector<float> decoded(samples);
+ for (size_t i = 0; i < samples; ++i) {
+ const uint8_t* src = data + i * size_t(bits / 8);
+ if (encoding == 3) {
+ uint32_t word = audio_u32(src);
+ std::memcpy(&decoded[i], &word, 4);
+ } else if (bits == 8)
+ decoded[i] = (int(src[0]) - 128) / 128.f;
+ else {
+ uint32_t word = 0;
+ for (int j = 0; j < bits / 8; ++j)
+ word |= uint32_t(src[j]) << (j * 8);
+ int64_t signed_value = word;
+ if (word & (uint32_t(1) << (bits - 1)))
+ signed_value -= int64_t(1) << bits;
+ decoded[i] = float(double(signed_value) / double(int64_t(1) << (bits - 1)));
+ }
+ }
+ sd::Audio view{};
+ view.data = decoded.data();
+ view.channels = channels;
+ view.sample_count = samples / channels;
+ view.sample_rate = rate;
+ return copy_audio(view);
+ }
+
+ std::vector<uint8_t> encode_pcm_audio(const sd::Audio& view, bool pcm16, uint64_t first, uint64_t count) {
+ if (first > view.sample_count || count > view.sample_count - first)
+ throw std::runtime_error("audio packet exceeds samples");
+ std::vector<uint8_t> bytes(size_t(count) * view.channels * (pcm16 ? 2 : 4));
+ for (uint64_t i = 0; i < count * view.channels; ++i) {
+ float value = view.data[first * view.channels + i];
+ uint32_t word = 0;
+ if (pcm16)
+ word = uint16_t(std::clamp(int(std::lround(std::clamp(value, -1.f, 1.f) * 32768)), -32768, 32767));
+ else
+ std::memcpy(&word, &value, 4);
+ int width = pcm16 ? 2 : 4;
+ for (int j = 0; j < width; ++j)
+ bytes[size_t(i) * width + j] = uint8_t(word >> (8 * j));
+ }
+ return bytes;
+ }
+
+ void write_wav(const sd_tensor_t& samples, int64_t sample_rate, const std::filesystem::path& path, bool pcm16) {
+ require_file_access(path.u8string(), SD_DGL_FILE_WRITE);
+ auto view = audio_view(samples, sample_rate);
+ uint64_t data_size = view.sample_count * view.channels * (pcm16 ? 2 : 4);
+ if (data_size > UINT32_MAX - 48)
+ throw std::runtime_error("WAV exceeds RIFF size limit");
+ std::ofstream file(path, std::ios::binary | std::ios::trunc);
+ if (!file)
+ throw std::runtime_error("cannot open WAV output");
+ auto u32 = [&](uint32_t value) {uint8_t data[]={uint8_t(value),uint8_t(value>>8),uint8_t(value>>16),uint8_t(value>>24)};file.write(reinterpret_cast<char*>(data),4); };
+ auto u16 = [&](uint16_t value) {uint8_t data[]={uint8_t(value),uint8_t(value>>8)};file.write(reinterpret_cast<char*>(data),2); };
+ file.write("RIFF", 4);
+ u32(uint32_t(data_size) + (pcm16 ? 36 : 48));
+ file.write("WAVEfmt ", 8);
+ u32(16);
+ u16(pcm16 ? 1 : 3);
+ u16(uint16_t(view.channels));
+ u32(view.sample_rate);
+ u32(view.sample_rate * view.channels * (pcm16 ? 2 : 4));
+ u16(uint16_t(view.channels * (pcm16 ? 2 : 4)));
+ u16(pcm16 ? 16 : 32);
+ if (!pcm16) {
+ file.write("fact", 4);
+ u32(4);
+ u32(uint32_t(view.sample_count));
+ }
+ file.write("data", 4);
+ u32(uint32_t(data_size));
+ for (uint64_t i = 0; i < view.sample_count * view.channels; ++i) {
+ if (pcm16) {
+ int value = int(std::lround(std::clamp(view.data[i], -1.f, 1.f) * 32768));
+ u16(uint16_t(std::clamp(value, -32768, 32767)));
+ } else {
+ uint32_t word;
+ std::memcpy(&word, &view.data[i], 4);
+ u32(word);
+ }
+ }
+ file.close();
+ if (!file)
+ throw std::runtime_error("cannot write WAV output");
+ }
+
+}
diff --git a/src/dgl/ops/audio.h b/src/dgl/ops/audio.h
new file mode 100644
index 0000000..f5a31d3
--- /dev/null
+++ b/src/dgl/ops/audio.h
@@ -0,0 +1,22 @@
+#ifndef __SD_DGL_OPS_AUDIO_H__
+#define __SD_DGL_OPS_AUDIO_H__
+
+#include <filesystem>
+#include "core/media.h"
+#include "dgl/tensor.h"
+
+namespace sd::dgl::ops {
+ struct AudioReadResult {
+ std::shared_ptr<sd_tensor_t> samples;
+ int64_t sample_rate = 0;
+ };
+
+ sd::Audio audio_view(const sd_tensor_t& samples, int64_t sample_rate);
+ AudioReadResult copy_audio(const sd::Audio& audio);
+ AudioReadResult read_wav(const std::filesystem::path& path);
+ void write_wav(const sd_tensor_t& samples, int64_t sample_rate, const std::filesystem::path& path, bool pcm16);
+ AudioReadResult decode_pcm_audio(const uint8_t* data, size_t bytes, int encoding, int bits, int channels, int64_t rate);
+ std::vector<uint8_t> encode_pcm_audio(const sd::Audio& audio, bool pcm16, uint64_t first, uint64_t count);
+}
+
+#endif
diff --git a/src/dgl/ops/control.cpp b/src/dgl/ops/control.cpp
new file mode 100644
index 0000000..2479193
--- /dev/null
+++ b/src/dgl/ops/control.cpp
@@ -0,0 +1,35 @@
+#include "dgl/ops/control.h"
+
+#include "conditioning/conditioner.hpp"
+#include "dgl/models/fwd.h"
+
+namespace sd::dgl::ops {
+ models::UNetControls compute_unet_controls(const std::vector<const models::ControlBinding*>& bindings,
+ const Tensor<float>& input,
+ const Tensor<float>& timesteps,
+ const SDCondition& condition) {
+ if (bindings.size() == 1) {
+ const auto& binding = *bindings.front();
+ return {binding.model->compute_sample_controls(binding, input, timesteps, condition), binding.strength};
+ }
+
+ models::UNetControls controls;
+ for (const auto* binding : bindings) {
+ auto residuals = binding->model->compute_sample_controls(*binding, input, timesteps, condition);
+ if (controls.residuals.empty()) {
+ controls.residuals = std::move(residuals);
+ for (auto& residual : controls.residuals)
+ residual *= binding->strength;
+ continue;
+ }
+ if (controls.residuals.size() != residuals.size())
+ throw std::runtime_error("ControlNet residual counts differ");
+ for (size_t i = 0; i < controls.residuals.size(); ++i) {
+ if (controls.residuals[i].shape() != residuals[i].shape())
+ throw std::runtime_error("ControlNet residual shapes differ");
+ controls.residuals[i] += residuals[i] * binding->strength;
+ }
+ }
+ return controls;
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/control.h b/src/dgl/ops/control.h
new file mode 100644
index 0000000..1889026
--- /dev/null
+++ b/src/dgl/ops/control.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_OPS_CONTROL_H__
+#define __SD_DGL_OPS_CONTROL_H__
+
+#include "dgl/models/fwd.h"
+
+#include "dgl/models/control.h"
+
+namespace sd::dgl::ops {
+ models::UNetControls compute_unet_controls(const std::vector<const models::ControlBinding*>& bindings, const Tensor<float>& input, const Tensor<float>& timesteps, const SDCondition& condition);
+}
+
+#endif // __SD_DGL_OPS_CONTROL_H__
diff --git a/src/dgl/ops/detailer.cpp b/src/dgl/ops/detailer.cpp
new file mode 100644
index 0000000..42b6c9a
--- /dev/null
+++ b/src/dgl/ops/detailer.cpp
@@ -0,0 +1,99 @@
+#include "dgl/ops/detailer.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "dgl/models/fwd.h"
+
+namespace sd::dgl::ops {
+ static float detection_area(const models::DetailDetection& detection) {
+ return std::max(0.f, detection.x2 - detection.x1) * std::max(0.f, detection.y2 - detection.y1);
+ }
+
+ std::vector<int64_t> filter_detection_indices(const std::vector<models::DetailDetection>& detections,
+ int width,
+ int height,
+ const DetectionFilter& filter) {
+ const float image_area = static_cast<float>(width) * height;
+ std::vector<int64_t> indices;
+ for (size_t i = 0; i < detections.size(); ++i) {
+ const float ratio = detection_area(detections[i]) / image_area;
+ if (ratio >= filter.min_area_ratio && ratio <= filter.max_area_ratio)
+ indices.push_back(int64_t(i));
+ }
+ if (filter.top_k > 0 && static_cast<int>(indices.size()) > filter.top_k) {
+ std::partial_sort(indices.begin(), indices.begin() + filter.top_k, indices.end(), [&](int64_t a, int64_t b) {
+ return filter.top_k_by_confidence ? detections[a].confidence > detections[b].confidence
+ : detection_area(detections[a]) > detection_area(detections[b]);
+ });
+ indices.resize(filter.top_k);
+ }
+ if (filter.sort_by == DetectionSort::LeftToRight) {
+ std::sort(indices.begin(), indices.end(), [&](int64_t a, int64_t b) { return detections[a].x1 < detections[b].x1; });
+ } else if (filter.sort_by == DetectionSort::CenterToEdge) {
+ const float cx = width * 0.5f, cy = height * 0.5f;
+ std::sort(indices.begin(), indices.end(), [&](int64_t a, int64_t b) {
+ const auto& first = detections[a];
+ const auto& second = detections[b];
+ const float ax = (first.x1 + first.x2) * 0.5f - cx, ay = (first.y1 + first.y2) * 0.5f - cy;
+ const float bx = (second.x1 + second.x2) * 0.5f - cx, by = (second.y1 + second.y2) * 0.5f - cy;
+ return ax * ax + ay * ay < bx * bx + by * by;
+ });
+ } else if (filter.sort_by == DetectionSort::Area) {
+ std::sort(indices.begin(), indices.end(), [&](int64_t a, int64_t b) { return detection_area(detections[a]) > detection_area(detections[b]); });
+ }
+ return indices;
+ }
+
+ DetailMask box_mask(const models::DetailDetection& detection, int width, int height) {
+ DetailMask mask;
+ mask.width = width;
+ mask.height = height;
+ mask.data.assign(static_cast<size_t>(width) * height, 0);
+ int x1 = std::clamp(static_cast<int>(std::floor(detection.x1)), 0, width);
+ int y1 = std::clamp(static_cast<int>(std::floor(detection.y1)), 0, height);
+ int x2 = std::clamp(static_cast<int>(std::ceil(detection.x2)), 0, width);
+ int y2 = std::clamp(static_cast<int>(std::ceil(detection.y2)), 0, height);
+ for (int y = y1; y < y2; ++y) {
+ std::fill(mask.data.begin() + static_cast<size_t>(y) * width + x1,
+ mask.data.begin() + static_cast<size_t>(y) * width + x2,
+ 255);
+ }
+ return mask;
+ }
+
+ bool detail_crop_box(const sd::Image& mask, float aspect, int padding, bool only_masked, DetailCrop& crop) {
+ const int canvas_width = int(mask.width), canvas_height = int(mask.height);
+ int x1 = canvas_width, y1 = canvas_height, x2 = 0, y2 = 0;
+ for (int y = 0; y < canvas_height; ++y) {
+ for (int x = 0; x < canvas_width; ++x) {
+ if (mask.data[size_t(y) * canvas_width + x]) {
+ x1 = std::min(x1, x);
+ y1 = std::min(y1, y);
+ x2 = std::max(x2, x + 1);
+ y2 = std::max(y2, y + 1);
+ }
+ }
+ }
+ if (x2 <= x1 || y2 <= y1)
+ return false;
+ if (!only_masked) {
+ crop = {0, 0, canvas_width, canvas_height};
+ return true;
+ }
+ x1 = std::max(0, x1 - padding);
+ y1 = std::max(0, y1 - padding);
+ x2 = std::min(canvas_width, x2 + padding);
+ y2 = std::min(canvas_height, y2 + padding);
+ int width = x2 - x1, height = y2 - y1;
+ if (float(width) / height < aspect)
+ width = std::min(canvas_width, int(std::ceil(height * aspect)));
+ else
+ height = std::min(canvas_height, int(std::ceil(width / aspect)));
+ const int x = std::clamp((x1 + x2) / 2 - width / 2, 0, canvas_width - width);
+ const int y = std::clamp((y1 + y2) / 2 - height / 2, 0, canvas_height - height);
+ crop = {x, y, x + width, y + height};
+ return true;
+ }
+
+}
diff --git a/src/dgl/ops/detailer.h b/src/dgl/ops/detailer.h
new file mode 100644
index 0000000..65e2df3
--- /dev/null
+++ b/src/dgl/ops/detailer.h
@@ -0,0 +1,40 @@
+#ifndef __SD_DGL_OPS_DETAILER_H__
+#define __SD_DGL_OPS_DETAILER_H__
+
+#include "dgl/models/fwd.h"
+
+#include "dgl/models/detail_detector.h"
+
+namespace sd::dgl::ops {
+ enum class DetectionSort {
+ None,
+ LeftToRight,
+ CenterToEdge,
+ Area,
+ };
+
+ struct DetectionFilter {
+ int top_k = 0;
+ bool top_k_by_confidence = false;
+ float min_area_ratio = 0.f;
+ float max_area_ratio = 1.f;
+ DetectionSort sort_by = DetectionSort::None;
+ };
+
+ struct DetailMask {
+ int width = 0;
+ int height = 0;
+ std::vector<uint8_t> data;
+ };
+
+ struct DetailCrop {
+ int x1, y1, x2, y2;
+ };
+
+ std::vector<int64_t> filter_detection_indices(const std::vector<models::DetailDetection>& detections, int width, int height, const DetectionFilter& filter);
+ DetailMask box_mask(const models::DetailDetection& detection, int width, int height);
+
+ bool detail_crop_box(const sd::Image& mask, float aspect, int padding, bool only_masked, DetailCrop& crop);
+}
+
+#endif
diff --git a/src/dgl/ops/fwd.h b/src/dgl/ops/fwd.h
new file mode 100644
index 0000000..e96985b
--- /dev/null
+++ b/src/dgl/ops/fwd.h
@@ -0,0 +1,37 @@
+#ifndef __SD_DGL_OPS_FWD_H__
+#define __SD_DGL_OPS_FWD_H__
+
+#include "dgl/models/fwd.h"
+
+namespace sd::dgl::ops {
+ struct ApgPredictor;
+ struct CfgPredictor;
+ struct DetailCrop;
+ struct DetailMask;
+ struct DetectionFilter;
+ enum class DetectionSort;
+ struct GuidanceBranches;
+ struct IPAdapterBinding;
+ struct IPAdapterBindings;
+ struct ModelPredictor;
+ class PredictionExecution;
+ struct PredictionExecutionOptions;
+ struct Predictor;
+ class PreviewObserver;
+ class PuLIDAttention;
+ struct PuLIDBinding;
+ struct PuLIDBindings;
+ class SamplingObservation;
+ class SamplingObserver;
+ class SamplingObserverSession;
+ struct SamplingOptions;
+ struct SamplingRun;
+ struct SamplingSolver;
+ struct SamplingStep;
+ struct SigmaWindow;
+ struct SkipLayerPredictor;
+ struct StepWindow;
+ struct SwitchPredictor;
+}
+
+#endif
diff --git a/src/dgl/ops/image.cpp b/src/dgl/ops/image.cpp
new file mode 100644
index 0000000..f7f334b
--- /dev/null
+++ b/src/dgl/ops/image.cpp
@@ -0,0 +1,306 @@
+#include "dgl/ops/image.h"
+#include "api/metadata.h"
+#include "core/file_access.h"
+#include "core/ggml_tensor_utils.h"
+#include "dgl/ops/tensor.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cmath>
+#include <cstring>
+#include <fstream>
+
+#define STB_IMAGE_STATIC
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_WINDOWS_UTF8
+#include "stb_image.h"
+#define STB_IMAGE_WRITE_STATIC
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#define STBIW_WINDOWS_UTF8
+#include "stb_image_write.h"
+
+namespace sd::dgl::ops {
+
+ static void validate_image_shape(int64_t width, int64_t height, int64_t channels) {
+ if (width <= 0 || height <= 0 || width > 16384 || height > 16384 ||
+ width * height > 64 * 1024 * 1024 || (channels != 1 && channels != 3 && channels != 4))
+ throw std::runtime_error("invalid image dimensions or channels (expected 1, 3 or 4)");
+ }
+
+ std::shared_ptr<sd_tensor_t> allocate_image(int width, int height, int channels) {
+ validate_image_shape(width, height, channels);
+ return make_tensor(SD_TENSOR_U8, std::vector<int64_t>{channels, width, height});
+ }
+
+ std::shared_ptr<sd_tensor_t> copy_image(const sd::Image& image) {
+ validate_image_shape(image.width, image.height, image.channel);
+ return make_tensor(SD_TENSOR_U8, std::vector<int64_t>{image.channel, image.width, image.height},
+ image.data, size_t(image.width) * image.height * image.channel);
+ }
+
+ sd::Image image_view(const sd_tensor_t& tensor) {
+ const auto& shape = tensor.shape();
+ if (tensor.type != SD_TENSOR_U8 || shape.size() != 3)
+ throw std::runtime_error("image requires a U8 tensor with shape [channels, width, height]");
+ validate_image_shape(shape[1], shape[2], shape[0]);
+ return {uint32_t(shape[1]), uint32_t(shape[2]), uint32_t(shape[0]), const_cast<uint8_t*>(static_cast<const uint8_t*>(tensor.data()))};
+ }
+
+ std::shared_ptr<sd_tensor_t> decode_video_jpeg(const std::vector<uint8_t>& bytes, int width, int height) {
+ int w = 0, h = 0, channels = 0;
+ if (bytes.size() < 2 || bytes[0] != 0xff || bytes[1] != 0xd8 ||
+ !stbi_info_from_memory(bytes.data(), int(bytes.size()), &w, &h, &channels) || w != width || h != height)
+ throw std::runtime_error("invalid MJPEG frame dimensions");
+ validate_image_shape(width, height, 3);
+ std::unique_ptr<stbi_uc, decltype(&stbi_image_free)> pixels(stbi_load_from_memory(bytes.data(), int(bytes.size()), &w, &h, &channels, 3), stbi_image_free);
+ if (!pixels || w != width || h != height)
+ throw std::runtime_error("cannot decode MJPEG frame");
+ return copy_image({uint32_t(width), uint32_t(height), 3, pixels.get()});
+ }
+
+ std::vector<uint8_t> encode_video_jpeg(const sd::Image& image, int quality) {
+ std::vector<uint8_t> bytes;
+ auto append = [](void* context, void* data, int size) {
+ auto& output = *static_cast<std::vector<uint8_t>*>(context);
+ auto first = static_cast<const uint8_t*>(data);
+ output.insert(output.end(), first, first + size);
+ };
+ // Keep stb's subsampled JPEG path for compatibility with MJPEG players.
+ if (!stbi_write_jpg_to_func(append, &bytes, image.width, image.height, 3, image.data, std::min(quality, 90)))
+ throw std::runtime_error("cannot encode MJPEG frame");
+ return bytes;
+ }
+
+ static const char* base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+ void decode_base64(const std::string& text, uint8_t* bytes, size_t size) {
+ if (text.size() != (size + 2) / 3 * 4)
+ throw std::runtime_error("base64 length does not match tensor dimensions");
+ size_t output = 0;
+ for (size_t i = 0; i < text.size(); i += 4) {
+ uint32_t n = 0;
+ for (size_t j = 0; j < 4; ++j) {
+ bool padding = i + 4 == text.size() && ((j == 3 && size % 3 != 0) || (j == 2 && size % 3 == 1));
+ if (padding) {
+ if (text[i + j] != '=')
+ throw std::runtime_error("invalid base64 padding");
+ n <<= 6;
+ } else {
+ const char* p = text[i + j] == '\0' ? nullptr : std::strchr(base64_alphabet, text[i + j]);
+ if (!p)
+ throw std::runtime_error("invalid base64 character");
+ n = (n << 6) | uint32_t(p - base64_alphabet);
+ }
+ }
+ if (i + 4 == text.size() && ((size % 3 == 1 && (n & 0xffff)) || (size % 3 == 2 && (n & 0xff))))
+ throw std::runtime_error("nonzero base64 padding bits");
+ if (output < size)
+ bytes[output++] = uint8_t(n >> 16);
+ if (output < size)
+ bytes[output++] = uint8_t(n >> 8);
+ if (output < size)
+ bytes[output++] = uint8_t(n);
+ }
+ }
+
+ struct Region {
+ int x, y, width, height;
+ };
+
+ static Region region_at(const sd::Tensor<int64_t>& boxes, int64_t index) {
+ const auto* box = boxes.data() + index * 4;
+ return {int(box[0]), int(box[1]), int(box[2]), int(box[3])};
+ }
+
+ void validate_region_boxes(const sd::Tensor<int64_t>& boxes, int64_t width, int64_t height) {
+ if (width < 1 || width > 16384 || height < 1 || height > 16384)
+ throw std::runtime_error("region canvas width and height must be in [1, 16384]");
+ if (boxes.dim() != 2 || boxes.shape()[0] != 4 || boxes.shape()[1] > 10000)
+ throw std::runtime_error("region boxes require I64 [4,N] with at most 10000 boxes");
+ for (int64_t i = 0; i < boxes.shape()[1]; ++i) {
+ const auto* box = boxes.data() + i * 4;
+ const auto label = "region box " + std::to_string(i);
+ if (box[0] < 0 || box[1] < 0)
+ throw std::runtime_error(label + " has negative coordinates");
+ if (box[2] <= 0 || box[3] <= 0)
+ throw std::runtime_error(label + " must have positive width and height");
+ if (box[0] >= width || box[1] >= height || box[2] > width - box[0] || box[3] > height - box[1])
+ throw std::runtime_error(label + " lies outside the canvas");
+ }
+ }
+
+ sd::Tensor<int64_t> boxes_from_detections(const sd::Tensor<float>& xyxy, int64_t width, int64_t height) {
+ if (width < 1 || width > 16384 || height < 1 || height > 16384)
+ throw std::runtime_error("region canvas width and height must be in [1, 16384]");
+ if (xyxy.dim() != 2 || xyxy.shape()[0] != 4 || xyxy.shape()[1] > 4096)
+ throw std::runtime_error("detections.xyxy requires F32 [4,N] with at most 4096 boxes");
+ std::vector<int64_t> boxes;
+ boxes.reserve(size_t(xyxy.shape()[1]) * 4);
+ for (int64_t i = 0; i < xyxy.shape()[1]; ++i) {
+ const auto* d = xyxy.data() + i * 4;
+ for (int j = 0; j < 4; ++j)
+ if (!std::isfinite(d[j]))
+ throw std::runtime_error("detection " + std::to_string(i) + " coordinates must be finite");
+ const auto x = int64_t(std::clamp(std::floor(d[0]), 0.f, float(width)));
+ const auto y = int64_t(std::clamp(std::floor(d[1]), 0.f, float(height)));
+ const auto right = int64_t(std::clamp(std::ceil(d[2]), 0.f, float(width)));
+ const auto bottom = int64_t(std::clamp(std::ceil(d[3]), 0.f, float(height)));
+ if (right > x && bottom > y)
+ boxes.insert(boxes.end(), {x, y, right - x, bottom - y});
+ }
+ const auto count = int64_t(boxes.size() / 4);
+ return sd::Tensor<int64_t>({4, count}, std::move(boxes));
+ }
+
+ std::shared_ptr<sd_tensor_t> read_image_tensor(const std::string& path, int requested_channels) {
+ require_file_access(path, SD_DGL_FILE_READ);
+ int width, height, channels;
+ if (!stbi_info(path.c_str(), &width, &height, &channels))
+ throw std::runtime_error("cannot read image '" + path + "'");
+ const int output_channels = requested_channels ? requested_channels : (channels == 2 ? 4 : channels);
+ validate_image_shape(width, height, output_channels);
+ std::unique_ptr<stbi_uc, decltype(&stbi_image_free)> pixels(stbi_load(path.c_str(), &width, &height, &channels, output_channels), stbi_image_free);
+ if (!pixels)
+ throw std::runtime_error("image decode failed");
+ return copy_image({uint32_t(width), uint32_t(height), uint32_t(output_channels), pixels.get()});
+ }
+
+ std::vector<std::shared_ptr<sd_tensor_t>> read_image_directory(const std::filesystem::path& path, size_t max_images, int channels) {
+ require_file_access(path.u8string(), SD_DGL_FILE_READ);
+ std::vector<std::filesystem::path> paths;
+ for (const auto& entry : std::filesystem::directory_iterator(path)) {
+ auto extension = entry.path().extension().u8string();
+ std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ if (entry.is_regular_file() && (extension == ".png" || extension == ".jpg" || extension == ".jpeg" || extension == ".bmp"))
+ paths.push_back(entry.path());
+ }
+ std::sort(paths.begin(), paths.end());
+ if (max_images && paths.size() > max_images)
+ paths.resize(max_images);
+ if (paths.empty())
+ throw std::runtime_error("image directory has no supported images");
+ std::vector<std::shared_ptr<sd_tensor_t>> images;
+ size_t bytes = 0;
+ for (const auto& file : paths) {
+ auto image = read_image_tensor(file.u8string(), channels);
+ if (image->bytes > 256 * 1024 * 1024 - bytes)
+ throw std::runtime_error("image directory exceeds 256 MiB");
+ bytes += image->bytes;
+ images.push_back(std::move(image));
+ }
+ return images;
+ }
+
+ std::vector<std::shared_ptr<sd_tensor_t>> crop_regions(const sd_tensor_t& input, const sd::Tensor<int64_t>& boxes) {
+ const auto image = image_view(input);
+ validate_region_boxes(boxes, image.width, image.height);
+ std::vector<std::shared_ptr<sd_tensor_t>> crops;
+ crops.reserve(size_t(boxes.shape()[1]));
+ for (int64_t i = 0; i < boxes.shape()[1]; ++i) {
+ const auto r = region_at(boxes, i);
+ crops.push_back(crop_tensor(input, {1, 2}, {r.x, r.y}, {r.width, r.height}));
+ }
+ return crops;
+ }
+
+ std::shared_ptr<sd_tensor_t> paste_regions(const sd_tensor_t& background, const sd::Tensor<int64_t>& boxes, const std::vector<std::shared_ptr<sd_tensor_t>>& items) {
+ const auto source = image_view(background);
+ validate_region_boxes(boxes, source.width, source.height);
+ if (items.size() != size_t(boxes.shape()[1]))
+ throw std::runtime_error("foreground count does not match region boxes");
+ auto result = copy_image(source);
+ const auto image = image_view(*result);
+ for (size_t i = 0; i < items.size(); ++i) {
+ const auto r = region_at(boxes, int64_t(i));
+ auto patch = items[i];
+ if (!patch || image_view(*patch).channel != image.channel)
+ throw std::runtime_error("region foreground channels differ from the background");
+ auto resized = interpolate_tensor(*patch, {r.width, r.height}, {1, 2}, "bilinear", false);
+ const auto view = image_view(*resized);
+ for (int y = 0; y < r.height; ++y)
+ std::memcpy(image.data + (size_t(y + r.y) * image.width + r.x) * image.channel,
+ view.data + size_t(y) * r.width * image.channel, size_t(r.width) * image.channel);
+ }
+ return result;
+ }
+
+ std::vector<uint8_t> encode_image(const sd_tensor_t& input, const std::string& metadata) {
+ const auto image = image_view(input);
+ std::vector<uint8_t> bytes;
+ auto append = [](void* context, void* data, int size) {
+ auto& output = *static_cast<std::vector<uint8_t>*>(context);
+ const auto* first = static_cast<const uint8_t*>(data);
+ output.insert(output.end(), first, first + size);
+ };
+ if (!stbi_write_png_to_func(append, &bytes, image.width, image.height, image.channel, image.data, image.width * image.channel) ||
+ !sd::metadata::embed_png_metadata(bytes, metadata))
+ throw std::runtime_error("cannot encode image or generation metadata");
+ return bytes;
+ }
+
+ void save_image(const sd_tensor_t& input, const std::string& path, const std::string& metadata) {
+ if (path.size() < 4 || path.substr(path.size() - 4) != ".png")
+ throw std::runtime_error("image::save requires a .png path");
+ require_file_access(path, SD_DGL_FILE_WRITE);
+ const auto bytes = encode_image(input, metadata);
+ std::ofstream file(std::filesystem::u8path(path), std::ios::binary | std::ios::trunc);
+ file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
+ file.close();
+ if (!file)
+ throw std::runtime_error("cannot save image '" + path + "'");
+ }
+
+ void copy_video_frame(sd::Tensor<float>& video, const sd::Tensor<float>& frame, size_t index) {
+ const auto& shape = video.shape();
+ const auto plane_size = size_t(shape[0]) * size_t(shape[1]);
+ const auto frame_count = size_t(shape[2]);
+ for (size_t channel = 0; channel < size_t(shape[3]); ++channel) {
+ const auto offset = (channel * frame_count + index) * plane_size;
+ std::copy_n(frame.data() + channel * plane_size, plane_size, video.data() + offset);
+ }
+ }
+ sd::Tensor<float> image_to_f32(const sd_tensor_t& tensor) {
+ const auto& s = tensor.shape();
+ if (tensor.type != SD_TENSOR_U8 || (s.size() != 3 && s.size() != 4) || s[0] < 1 || s[0] > 4 ||
+ s[1] < 1 || s[1] > 16384 || s[2] < 1 || s[2] > 16384 || (s.size() == 4 && s[3] < 1)) {
+ throw std::runtime_error("expected a U8 tensor with channels, width, height and optional time");
+ }
+ auto* data = const_cast<uint8_t*>(static_cast<const uint8_t*>(tensor.data()));
+ const auto stride = s[0] * s[1] * s[2];
+ auto frame = [&](int64_t i) {
+ return sd_image_to_tensor({uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), data + i * stride});
+ };
+ if (s.size() == 3) {
+ return frame(0);
+ }
+ const std::vector<int64_t> shape{s[1], s[2], s[3], s[0], 1};
+ if (tensor.bytes > std::vector<float>().max_size()) {
+ throw std::runtime_error("video pixel shape exceeds addressable float storage");
+ }
+ sd::Tensor<float> output(shape);
+ for (int64_t i = 0; i < s[3]; ++i) {
+ copy_video_frame(output, frame(i), size_t(i));
+ }
+ return output;
+ }
+ std::shared_ptr<sd_tensor_t> image_to_u8(const sd::Tensor<float>& tensor) {
+ if (tensor.empty() || (tensor.dim() != 4 && tensor.dim() != 5)) {
+ throw std::runtime_error("expected nonempty float pixels with width, height, optional time, channels and batch");
+ }
+ const auto& s = tensor.shape();
+ const auto channels = s[tensor.dim() == 5 ? 3 : 2];
+ const auto frames = tensor.dim() == 5 ? s[2] : 1;
+ std::vector<int64_t> shape{channels, s[0], s[1]};
+ if (tensor.dim() == 5) {
+ shape.push_back(frames);
+ }
+ auto pixels = make_tensor(SD_TENSOR_U8, std::move(shape));
+ auto* data = static_cast<uint8_t*>(pixels->mutable_data());
+ const auto stride = pixels->bytes / size_t(frames);
+ for (int64_t i = 0; i < frames; ++i) {
+ std::unique_ptr<uint8_t, decltype(&std::free)> frame(tensor_to_sd_image(tensor, int(i)).data, std::free);
+ std::memcpy(data + stride * size_t(i), frame.get(), stride);
+ }
+ return pixels;
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/image.h b/src/dgl/ops/image.h
new file mode 100644
index 0000000..53bc6c5
--- /dev/null
+++ b/src/dgl/ops/image.h
@@ -0,0 +1,29 @@
+#ifndef __SD_DGL_OPS_IMAGE_H__
+#define __SD_DGL_OPS_IMAGE_H__
+
+#include <filesystem>
+#include "core/media.h"
+#include "dgl/tensor.h"
+
+namespace sd::dgl::ops {
+ void copy_video_frame(sd::Tensor<float>& video, const sd::Tensor<float>& frame, size_t index);
+ std::shared_ptr<sd_tensor_t> allocate_image(int width, int height, int channels);
+ std::shared_ptr<sd_tensor_t> copy_image(const sd::Image& image);
+ sd::Image image_view(const sd_tensor_t& tensor);
+ std::shared_ptr<sd_tensor_t> read_image_tensor(const std::string& path, int requested_channels = 0);
+ std::vector<std::shared_ptr<sd_tensor_t>> read_image_directory(const std::filesystem::path& path, size_t max_images, int channels);
+ void save_image(const sd_tensor_t& input, const std::string& path, const std::string& metadata = {});
+ std::vector<uint8_t> encode_image(const sd_tensor_t& input, const std::string& metadata = {});
+ std::shared_ptr<sd_tensor_t> decode_video_jpeg(const std::vector<uint8_t>& bytes, int width, int height);
+ std::vector<uint8_t> encode_video_jpeg(const sd::Image& image, int quality);
+ void decode_base64(const std::string& text, uint8_t* bytes, size_t size);
+ void validate_region_boxes(const sd::Tensor<int64_t>& boxes, int64_t width, int64_t height);
+ sd::Tensor<int64_t> boxes_from_detections(const sd::Tensor<float>& detections, int64_t width, int64_t height);
+ std::vector<std::shared_ptr<sd_tensor_t>> crop_regions(const sd_tensor_t& input, const sd::Tensor<int64_t>& boxes);
+ std::shared_ptr<sd_tensor_t> paste_regions(const sd_tensor_t& background, const sd::Tensor<int64_t>& boxes, const std::vector<std::shared_ptr<sd_tensor_t>>& foregrounds);
+ sd::Tensor<float> image_to_f32(const sd_tensor_t& tensor);
+ std::shared_ptr<sd_tensor_t> image_to_u8(const sd::Tensor<float>& tensor);
+
+}
+
+#endif
diff --git a/src/dgl/ops/ip_adapter.h b/src/dgl/ops/ip_adapter.h
new file mode 100644
index 0000000..c1fef04
--- /dev/null
+++ b/src/dgl/ops/ip_adapter.h
@@ -0,0 +1,25 @@
+#ifndef __SD_DGL_OPS_IP_ADAPTER_H__
+#define __SD_DGL_OPS_IP_ADAPTER_H__
+
+#include "dgl/models/fwd.h"
+
+#include "model/adapter/ip_adapter.hpp"
+
+namespace sd::dgl::ops {
+
+ struct IPAdapterBinding {
+ std::shared_ptr<models::IPAdapter> model;
+ std::shared_ptr<const Tensor<float>> tokens;
+ float strength;
+ float start_sigma;
+ float end_sigma;
+ };
+
+ struct IPAdapterBindings {
+ std::vector<IPAdapterBinding> entries;
+ };
+
+ using IPAdapterCache = std::map<const models::IPAdapter*, std::map<const Tensor<float>*, ::IPAdapter::AttentionCache>>;
+}
+
+#endif // __SD_DGL_OPS_IP_ADAPTER_H__
diff --git a/src/dgl/ops/media.cpp b/src/dgl/ops/media.cpp
new file mode 100644
index 0000000..2226d3a
--- /dev/null
+++ b/src/dgl/ops/media.cpp
@@ -0,0 +1,199 @@
+#include "dgl/ops/media.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+
+#include "conditioning/wan_audio.h"
+#include "dgl/models/audio_vae.h"
+#include "dgl/models/fwd.h"
+#include "model/vae/audio_vae.hpp"
+#include "model_definition.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl::ops {
+
+ sd::Audio* waveform_to_sd_audio(const models::AudioVAE* sd,
+ const sd::Tensor<float>& waveform) {
+ if (sd == nullptr || waveform.empty()) {
+ return nullptr;
+ }
+
+ int64_t sample_count = waveform.shape()[0];
+ int64_t channels = waveform.shape().size() > 1 ? waveform.shape()[1] : 1;
+ if (sample_count <= 0 || channels <= 0) {
+ return nullptr;
+ }
+
+ sd::Audio* audio = (sd::Audio*)malloc(sizeof(sd::Audio));
+ if (audio == nullptr) {
+ return nullptr;
+ }
+
+ audio->sample_rate = static_cast<uint32_t>(sd->audio_vae_model != nullptr ? sd->audio_vae_model->output_sample_rate() : 0);
+ audio->channels = static_cast<uint32_t>(channels);
+ audio->sample_count = static_cast<uint64_t>(sample_count);
+ size_t sample_bytes = waveform.numel() * sizeof(float);
+ audio->data = (float*)malloc(sample_bytes);
+ if (audio->data == nullptr) {
+ free(audio);
+ return nullptr;
+ }
+
+ auto wavaform_t = waveform.permute({1, 0, 2, 3});
+ std::memcpy(audio->data, wavaform_t.data(), sample_bytes);
+
+ return audio;
+ }
+
+ static float ltxv_latent_corner_to_pixel_frame(int64_t corner_index,
+ int temporal_scale,
+ bool causal_temporal_positioning) {
+ float pixel_t = static_cast<float>(corner_index * temporal_scale);
+ if (causal_temporal_positioning) {
+ pixel_t = std::max(0.f, pixel_t + 1.f - static_cast<float>(temporal_scale));
+ }
+ return pixel_t;
+ }
+
+ static void set_ltxv_video_position(sd::Tensor<float>* positions,
+ int64_t token,
+ float t_start,
+ float t_end,
+ float h_start,
+ float h_end,
+ float w_start,
+ float w_end) {
+ positions->index(0, 0, token, 0) = t_start;
+ positions->index(1, 0, token, 0) = t_end;
+ positions->index(0, 1, token, 0) = h_start;
+ positions->index(1, 1, token, 0) = h_end;
+ positions->index(0, 2, token, 0) = w_start;
+ positions->index(1, 2, token, 0) = w_end;
+ }
+
+ sd::Tensor<float> build_ltxv_video_positions(int64_t width,
+ int64_t height,
+ int64_t target_latent_frames,
+ int64_t keyframe_latent_frames,
+ int keyframe_frame_idx,
+ int keyframe_pixel_frames,
+ int fps,
+ int spatial_scale,
+ int temporal_scale,
+ bool causal_temporal_positioning) {
+ GGML_ASSERT(width > 0 && height > 0 && target_latent_frames > 0);
+ GGML_ASSERT(keyframe_latent_frames > 0);
+ GGML_ASSERT(fps > 0);
+
+ int64_t total_tokens = width * height * (target_latent_frames + keyframe_latent_frames);
+ sd::Tensor<float> positions({2, 3, total_tokens, 1});
+ int64_t token = 0;
+
+ for (int64_t t = 0; t < target_latent_frames; t++) {
+ float t_start = ltxv_latent_corner_to_pixel_frame(t, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
+ float t_end = ltxv_latent_corner_to_pixel_frame(t + 1, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
+ for (int64_t h = 0; h < height; h++) {
+ float h_start = static_cast<float>(h * spatial_scale);
+ float h_end = static_cast<float>((h + 1) * spatial_scale);
+ for (int64_t w = 0; w < width; w++) {
+ float w_start = static_cast<float>(w * spatial_scale);
+ float w_end = static_cast<float>((w + 1) * spatial_scale);
+ set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
+ }
+ }
+ }
+
+ for (int64_t t = 0; t < keyframe_latent_frames; t++) {
+ float t_start = static_cast<float>(keyframe_frame_idx + t * temporal_scale);
+ float t_end = static_cast<float>(keyframe_frame_idx + (t + 1) * temporal_scale);
+ if (keyframe_pixel_frames == 1) {
+ t_end = t_start + 1.f;
+ }
+ t_start /= static_cast<float>(fps);
+ t_end /= static_cast<float>(fps);
+ for (int64_t h = 0; h < height; h++) {
+ float h_start = static_cast<float>(h * spatial_scale);
+ float h_end = static_cast<float>((h + 1) * spatial_scale);
+ for (int64_t w = 0; w < width; w++) {
+ float w_start = static_cast<float>(w * spatial_scale);
+ float w_end = static_cast<float>((w + 1) * spatial_scale);
+ set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
+ }
+ }
+ }
+
+ return positions;
+ }
+
+ sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd::Audio& audio,
+ int target_sample_rate) {
+ if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) {
+ return {};
+ }
+ uint64_t output_samples = static_cast<uint64_t>(std::llround(
+ static_cast<long double>(audio.sample_count) * target_sample_rate / audio.sample_rate));
+ output_samples = std::max<uint64_t>(1, output_samples);
+ uint64_t padded_samples = (output_samples + 799) / 800 * 800;
+ // Keep stereo streams planar for the mono-per-stream audio encoder:
+ // [samples, 1, stereo, batch]. This avoids flattening interleaved L/R
+ // storage into alternating samples when the encoder folds streams into
+ // its batch dimension.
+ sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 1, 2, 1});
+
+ for (uint64_t i = 0; i < output_samples; ++i) {
+ long double source_pos = static_cast<long double>(i) * audio.sample_rate / target_sample_rate;
+ uint64_t source0 = std::min<uint64_t>(static_cast<uint64_t>(source_pos), audio.sample_count - 1);
+ uint64_t source1 = std::min<uint64_t>(source0 + 1, audio.sample_count - 1);
+ float fraction = static_cast<float>(source_pos - source0);
+ for (uint32_t channel = 0; channel < 2; ++channel) {
+ uint32_t source_channel = audio.channels == 1 ? 0 : std::min<uint32_t>(channel, audio.channels - 1);
+ float a = audio.data[source0 * audio.channels + source_channel];
+ float b = audio.data[source1 * audio.channels + source_channel];
+ waveform.index(static_cast<int64_t>(i), 0, channel, 0) =
+ std::clamp(a + (b - a) * fraction, -1.f, 1.f);
+ }
+ }
+ return waveform;
+ }
+
+ sd::Tensor<float> build_s2v_audio_window(const sd::Tensor<float>& stacked, int64_t batch_frames) {
+ const int64_t embed_dim = stacked.shape()[0];
+ const int64_t in_frames = stacked.shape()[1];
+ const int64_t num_layers = stacked.shape()[2];
+ if (embed_dim <= 0 || in_frames <= 0 || num_layers <= 0 || batch_frames <= 0) {
+ return {};
+ }
+ std::vector<float> layer_first(static_cast<size_t>(num_layers) * in_frames * embed_dim);
+ for (int64_t l = 0; l < num_layers; ++l) {
+ for (int64_t f = 0; f < in_frames; ++f) {
+ const float* src = stacked.data() + l * embed_dim * in_frames + f * embed_dim;
+ std::copy_n(src,
+ static_cast<size_t>(embed_dim),
+ layer_first.data() + (static_cast<size_t>(l) * in_frames + f) * embed_dim);
+ }
+ }
+ sd::wan_audio::BucketPlan plan;
+ std::vector<float> buckets = sd::wan_audio::build_audio_buckets(layer_first.data(),
+ static_cast<int>(num_layers),
+ static_cast<int>(in_frames),
+ static_cast<int>(embed_dim),
+ static_cast<int>(batch_frames),
+ &plan);
+ if (buckets.empty() || plan.bucket_frames < batch_frames) {
+ return {};
+ }
+ // Reorder frame-major buckets into sd::Tensor's [dim, frame, layer] layout.
+ sd::Tensor<float> window({embed_dim, batch_frames, num_layers});
+ for (int64_t f = 0; f < batch_frames; ++f) {
+ for (int64_t l = 0; l < num_layers; ++l) {
+ const float* src = buckets.data() + (static_cast<size_t>(f) * num_layers + l) * embed_dim;
+ float* dst = window.data() + l * embed_dim * batch_frames + f * embed_dim;
+ std::copy_n(src, static_cast<size_t>(embed_dim), dst);
+ }
+ }
+ return window;
+ }
+
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/media.h b/src/dgl/ops/media.h
new file mode 100644
index 0000000..5206bca
--- /dev/null
+++ b/src/dgl/ops/media.h
@@ -0,0 +1,30 @@
+#ifndef __SD_DGL_OPS_MEDIA_H__
+#define __SD_DGL_OPS_MEDIA_H__
+
+#include "dgl/models/fwd.h"
+
+#include "core/image_tensor.h"
+#include "core/media.h"
+#include "core/option_types.h"
+#include "core/tensor.hpp"
+
+namespace sd::dgl::ops {
+
+ sd::Audio* waveform_to_sd_audio(const models::AudioVAE* sd,
+ const sd::Tensor<float>& waveform);
+ sd::Tensor<float> build_ltxv_video_positions(int64_t width,
+ int64_t height,
+ int64_t target_latent_frames,
+ int64_t keyframe_latent_frames,
+ int keyframe_frame_idx,
+ int keyframe_pixel_frames,
+ int fps,
+ int spatial_scale,
+ int temporal_scale,
+ bool causal_temporal_positioning);
+ sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd::Audio& audio,
+ int target_sample_rate);
+ sd::Tensor<float> build_s2v_audio_window(const sd::Tensor<float>& stacked, int64_t batch_frames);
+} // namespace sd::dgl::ops
+
+#endif // __SD_DGL_OPS_MEDIA_H__
diff --git a/src/dgl/ops/media_encoding.cpp b/src/dgl/ops/media_encoding.cpp
new file mode 100644
index 0000000..a02e07e
--- /dev/null
+++ b/src/dgl/ops/media_encoding.cpp
@@ -0,0 +1,29 @@
+#include "dgl/ops/media_encoding.h"
+
+#include <algorithm>
+#include <cctype>
+#include <fstream>
+#include <limits>
+#include <stdexcept>
+
+namespace sd::dgl::ops {
+ void save_media(const std::string& path, const std::vector<uint8_t>& data, const std::string& mime_type) {
+ const char* expected = mime_type == "image/png" ? ".png" : mime_type == "video/webm" ? ".webm"
+ : mime_type == "video/x-msvideo" ? ".avi"
+ : nullptr;
+ if (!expected)
+ throw std::runtime_error("media::save does not support MIME type '" + mime_type + "'");
+ const auto output = std::filesystem::u8path(path);
+ auto extension = output.extension().u8string();
+ std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ if (extension != expected)
+ throw std::runtime_error("media::save path extension must be '" + std::string(expected) + "' for " + mime_type);
+ if (data.empty() || data.size() > size_t(std::numeric_limits<std::streamsize>::max()))
+ throw std::runtime_error("media::save requires nonempty bytes that fit a file stream");
+ std::ofstream file(output, std::ios::binary | std::ios::trunc);
+ file.write(reinterpret_cast<const char*>(data.data()), std::streamsize(data.size()));
+ file.close();
+ if (!file)
+ throw std::runtime_error("cannot save media '" + path + "'");
+ }
+}
diff --git a/src/dgl/ops/media_encoding.h b/src/dgl/ops/media_encoding.h
new file mode 100644
index 0000000..4df6d7e
--- /dev/null
+++ b/src/dgl/ops/media_encoding.h
@@ -0,0 +1,11 @@
+#ifndef __SD_DGL_OPS_MEDIA_ENCODING_H__
+#define __SD_DGL_OPS_MEDIA_ENCODING_H__
+
+#include "dgl/ops/video.h"
+
+namespace sd::dgl::ops {
+ std::vector<uint8_t> encode_webm(const sd_tensor_t& frames, const VideoSaveOptions& options);
+ void save_media(const std::string& path, const std::vector<uint8_t>& data, const std::string& mime_type);
+}
+
+#endif
diff --git a/src/dgl/ops/predictor.cpp b/src/dgl/ops/predictor.cpp
new file mode 100644
index 0000000..0996877
--- /dev/null
+++ b/src/dgl/ops/predictor.cpp
@@ -0,0 +1,699 @@
+#include "dgl/ops/predictor.h"
+
+#include <algorithm>
+
+#include "dgl/models/animatediff.h"
+#include "dgl/models/circular_axes.h"
+#include "dgl/models/control.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/fwd.h"
+#include "dgl/models/ip_adapter.h"
+#include "dgl/models/pulid.h"
+#include "dgl/ops/spectrum_cache.h"
+#include "runtime/sample-cache.h"
+
+namespace sd::dgl::ops {
+ float GuidanceBranches::scale_at(int step) const {
+ const auto index = static_cast<size_t>(step - 1);
+ return index < scale_schedule.size() ? scale_schedule[index] : scale;
+ }
+
+ bool StepWindow::active(const models::DiffusionPredictionStep& step) const {
+ const auto size = static_cast<float>(step.schedule_size);
+ const auto start = static_cast<int>(start_fraction * size);
+ const auto stop = static_cast<int>(stop_fraction * size);
+ return step.index > start && step.index < stop;
+ }
+
+ bool SkipLayerPredictor::active(float sigma, const models::DiffusionPredictionStep& step) const {
+ return scale != 0.f && sigmas.active(sigma) && (!steps || steps->active(step));
+ }
+
+ static const GuidanceBranches* guidance_branches(const Predictor& predictor) {
+ if (const auto* cfg = std::get_if<CfgPredictor>(&predictor.node))
+ return &cfg->branches;
+ if (const auto* apg = std::get_if<ApgPredictor>(&predictor.node))
+ return &apg->branches;
+ return nullptr;
+ }
+
+ static bool provides_unconditional(const Predictor& predictor, int first_step, int last_step) {
+ if (const auto* branches = guidance_branches(predictor))
+ return branches->negative && !branches->image_only;
+ if (const auto* slg = std::get_if<SkipLayerPredictor>(&predictor.node))
+ return provides_unconditional(*slg->base, first_step, last_step);
+ if (const auto* transition = std::get_if<SwitchPredictor>(&predictor.node)) {
+ if (transition->after_steps < first_step)
+ return provides_unconditional(*transition->after, first_step, last_step);
+ if (transition->after_steps >= last_step)
+ return provides_unconditional(*transition->before, first_step, last_step);
+ return provides_unconditional(*transition->before, first_step, transition->after_steps) &&
+ provides_unconditional(*transition->after, transition->after_steps + 1, last_step);
+ }
+ return false;
+ }
+
+ static const Predictor& selected_predictor(const Predictor& predictor, float sigma, const models::DiffusionPredictionStep& step) {
+ if (const auto* transition = std::get_if<SwitchPredictor>(&predictor.node))
+ return selected_predictor(step.index > transition->after_steps ? *transition->after : *transition->before, sigma, step);
+ if (const auto* slg = std::get_if<SkipLayerPredictor>(&predictor.node); slg && !slg->active(sigma, step))
+ return selected_predictor(*slg->base, sigma, step);
+ return predictor;
+ }
+
+ static const Predictor* unconditional_predictor(const Predictor& predictor, float sigma, const models::DiffusionPredictionStep& step) {
+ const auto& selected = selected_predictor(predictor, sigma, step);
+ if (const auto* branches = guidance_branches(selected))
+ return branches->image_only ? nullptr : branches->negative.get();
+ if (const auto* slg = std::get_if<SkipLayerPredictor>(&selected.node))
+ return unconditional_predictor(*slg->base, sigma, step);
+ return nullptr;
+ }
+
+ static PredictorCacheKey active_prediction(const Predictor& root, float sigma, const models::DiffusionPredictionStep& step, bool need_unconditional) {
+ PredictorCacheKey active;
+ std::set<const Predictor*> visited;
+ const auto collect = [&](const auto& self, const Predictor& predictor) -> void {
+ if (!visited.insert(&predictor).second)
+ return;
+ const auto& selected = selected_predictor(predictor, sigma, step);
+ if (&selected != &predictor) {
+ if (std::holds_alternative<SwitchPredictor>(predictor.node))
+ active.switches.push_back(&selected);
+ self(self, selected);
+ } else if (const auto* model = std::get_if<ModelPredictor>(&predictor.node)) {
+ active.models.push_back(model);
+ } else if (const auto* guidance = guidance_branches(predictor)) {
+ const auto scale = guidance->scale_at(step.index);
+ active.guidance.emplace_back(&predictor, scale);
+ self(self, *guidance->positive);
+ if (scale != 1.f || guidance->image_only)
+ self(self, *guidance->negative);
+ if (guidance->image_only)
+ self(self, *guidance->image_only);
+ } else if (const auto* slg = std::get_if<SkipLayerPredictor>(&predictor.node)) {
+ self(self, *slg->base);
+ if (slg->active(sigma, step)) {
+ active.slg.emplace_back(&predictor, slg->scale);
+ self(self, *slg->positive);
+ self(self, *slg->skipped);
+ }
+ }
+ };
+ collect(collect, root);
+ if (need_unconditional)
+ if (const auto* unconditional = unconditional_predictor(root, sigma, step))
+ collect(collect, *unconditional);
+ return active;
+ }
+
+ template <class Binding>
+ struct ConditioningWindow {
+ const Binding* binding;
+ SigmaWindow range;
+ };
+
+ struct ConditioningWindows {
+ std::vector<ConditioningWindow<models::ControlBinding>> controls;
+ std::vector<ConditioningWindow<IPAdapterBinding>> ip_adapters;
+ std::vector<ConditioningWindow<PuLIDBinding>> pulid;
+ };
+
+ struct ActiveConditioning {
+ std::vector<const models::ControlBinding*> controls;
+ std::vector<const IPAdapterBinding*> ip_adapters;
+ std::vector<const PuLIDBinding*> pulid;
+ };
+
+ struct EvaluationState {
+ std::map<const models::DenoisingParameterization*, sd_sample::SampleStepCacheDispatcher> caches;
+ };
+
+ static bool compatible_condition_tensors(const Tensor<float>& positive, const Tensor<float>& negative, size_t rank) {
+ if (positive.empty() || negative.empty())
+ return positive.empty() && negative.empty();
+ if ((positive.dim() != rank && positive.dim() != rank - 1) ||
+ (negative.dim() != rank && negative.dim() != rank - 1))
+ return false;
+ auto positive_shape = positive.shape(), negative_shape = negative.shape();
+ positive_shape.resize(rank, 1);
+ negative_shape.resize(rank, 1);
+ return positive_shape == negative_shape && positive_shape.back() == 1;
+ }
+
+ static Tensor<float> batch_condition_tensor(const Tensor<float>& positive, const Tensor<float>& negative, size_t rank) {
+ if (positive.empty())
+ return {};
+ auto shape = positive.shape();
+ shape.resize(rank, 1);
+ shape.back() = 2;
+ Tensor<float> result(shape);
+ auto next = std::copy(positive.values().begin(), positive.values().end(), result.values().begin());
+ std::copy(negative.values().begin(), negative.values().end(), next);
+ return result;
+ }
+
+ struct PredictionExecution::Resources {
+ struct PreparedModel {
+ models::Model::ExecutionScope scope;
+
+ explicit PreparedModel(models::Model& model)
+ : scope(model, false) { scope.require_ready(); }
+ };
+
+ const ModelPredictor& primary;
+ const models::DenoisingParameterization& parameterization;
+ const std::vector<float>& sigmas;
+ const PredictionExecutionOptions options;
+ std::map<models::Model*, PreparedModel> prepared;
+ std::map<const ModelPredictor*, ConditioningWindows> windows;
+ std::map<const models::DenoisingParameterization*, sd_sample::ConditionCacheRuntime> caches;
+ std::map<const ModelPredictor*, ActiveConditioning> previous_conditioning;
+ IPAdapterCache ip_adapter_cache;
+ PuLIDCache pulid_cache;
+ SpectrumCache spectrum;
+ std::map<std::pair<const ModelPredictor*, const ModelPredictor*>, SDCondition> batched_conditions;
+
+ Resources(const ModelPredictor& primary, const std::vector<float>& sigmas, const PredictionExecutionOptions& options)
+ : primary(primary), parameterization(primary.resolved_parameterization()), sigmas(sigmas), options(options), spectrum(options.cache, sigmas.size() - 1) {}
+
+ void check_noise_convention(const ModelPredictor& leaf) const {
+ if (!parameterization.compatible_noise(leaf.resolved_parameterization()))
+ throw std::runtime_error("predictors in one sample must use compatible noise conventions");
+ }
+
+ void prepare_predictor(const ModelPredictor& leaf) {
+ auto& model = leaf.dit();
+ prepared.try_emplace(&model, model);
+ const auto* parameterization = &leaf.resolved_parameterization();
+ if (options.cache && options.cache->mode != SD_CACHE_DISABLED && options.cache->mode != SD_CACHE_SPECTRUM && !caches.count(parameterization)) {
+ auto runtime = sd_sample::init_condition_cache_runtime(model.version(), options.cache, parameterization->native(), sigmas);
+ caches.emplace(parameterization, std::move(runtime));
+ }
+ }
+
+ template <class Bindings, class Windows>
+ void add_windows(const Bindings& bindings, Windows& target) {
+ if (!bindings)
+ return;
+ for (const auto& binding : bindings->entries) {
+ if (binding.strength != 0.f)
+ target.push_back({&binding, {binding.start_sigma, binding.end_sigma}});
+ }
+ }
+
+ models::DiffusionPrediction predict_model(const ModelPredictor& leaf, const Tensor<float>& x, float sigma, models::DiffusionPredictionStep step, EvaluationState& state);
+ models::DiffusionPrediction predict_batch(const ModelPredictor& leaf, const SDCondition& condition, const Tensor<float>& x, float sigma, const models::DiffusionPredictionStep& step);
+ };
+
+ PredictionExecution::PredictionExecution(const Predictor& root, const std::vector<float>& sigmas, const PredictionExecutionOptions& options)
+ : root_(root) {
+ if (options.need_unconditional && !provides_unconditional(root, 1, static_cast<int>(sigmas.size() - 1)))
+ throw std::runtime_error("CFG++ requires a predictor with an unconditional branch");
+ resources_ = std::make_unique<Resources>(primary_predictor(root), sigmas, options);
+ auto& resources = *resources_;
+ std::set<const Predictor*> visited;
+ std::set<models::DiT*> models;
+ std::set<const models::DenoisingParameterization*> parameterizations;
+ const auto collect = [&](const auto& self, const Predictor& predictor) -> void {
+ if (!visited.insert(&predictor).second)
+ return;
+ if (const auto* model = std::get_if<ModelPredictor>(&predictor.node)) {
+ models.insert(&model->dit());
+ resources.check_noise_convention(*model);
+ parameterizations.insert(&model->resolved_parameterization());
+ if (model->controls || model->ip_adapters || model->pulid) {
+ auto& windows = resources.windows[model];
+ resources.add_windows(model->controls, windows.controls);
+ resources.add_windows(model->ip_adapters, windows.ip_adapters);
+ resources.add_windows(model->pulid, windows.pulid);
+ }
+ } else if (const auto* branches = guidance_branches(predictor)) {
+ self(self, *branches->positive);
+ self(self, *branches->negative);
+ if (branches->image_only)
+ self(self, *branches->image_only);
+ } else if (const auto* slg = std::get_if<SkipLayerPredictor>(&predictor.node)) {
+ self(self, *slg->base);
+ if (slg->scale != 0.f) {
+ self(self, *slg->positive);
+ self(self, *slg->skipped);
+ }
+ } else {
+ const auto& transition = std::get<SwitchPredictor>(predictor.node);
+ self(self, *transition.before);
+ self(self, *transition.after);
+ }
+ };
+ collect(collect, root_);
+ if (options.cache && options.cache->mode == SD_CACHE_SPECTRUM &&
+ (options.need_unconditional || models.size() != 1 || parameterizations.size() != 1 || options.dual_schedule))
+ throw std::runtime_error("Spectrum requires one denoiser, a single sigma schedule and a non-CFG++ sampler");
+ }
+
+ PredictionExecution::~PredictionExecution() = default;
+
+ models::DiT& PredictionExecution::dit() const {
+ return resources_->primary.dit();
+ }
+
+ const models::DenoisingParameterization& PredictionExecution::parameterization() const {
+ return resources_->parameterization;
+ }
+
+ class IPAdapterAttention final : public AttentionProcessor {
+ std::vector<std::pair<const ::IPAdapter::AttentionCache*, float>> adapters_;
+
+ public:
+ IPAdapterAttention(const std::vector<const IPAdapterBinding*>& bindings, IPAdapterCache& cache) {
+ adapters_.reserve(bindings.size());
+ for (const auto* binding : bindings) {
+ auto& projections = cache[binding->model.get()];
+ auto entry = projections.find(binding->tokens.get());
+ if (entry == projections.end()) {
+ auto& adapter = *binding->model;
+ models::Model::ExecutionScope scope(adapter, false);
+ scope.require_ready();
+ auto kv = adapter.ip_adapter->compute_attention(adapter.n_threads, *binding->tokens);
+ if (kv.empty())
+ throw std::runtime_error("IP-Adapter key/value projection failed");
+ entry = projections.emplace(binding->tokens.get(), std::move(kv)).first;
+ }
+ const auto projection = std::find_if(adapters_.begin(), adapters_.end(), [&](const auto& adapter) {
+ return adapter.first == &entry->second;
+ });
+ if (projection == adapters_.end()) {
+ adapters_.emplace_back(&entry->second, binding->strength);
+ } else {
+ projection->second += binding->strength;
+ }
+ }
+ }
+
+ ggml_tensor* apply(GGMLRunnerContext& ctx, const std::string& layer, ggml_tensor* query, ggml_tensor* output, int64_t heads) const override {
+ for (const auto& [cache, strength] : adapters_) {
+ const auto entry = cache->find(layer);
+ if (entry == cache->end())
+ continue;
+ const auto& kv = entry->second;
+ const auto input = [&](const Tensor<float>& tensor) {
+ auto* value = sd::make_ggml_tensor(ctx.ggml_ctx, tensor, false);
+ ctx.set_backend_tensor_data(value, tensor.data());
+ if (value->ne[2] == query->ne[2])
+ return value;
+ if (value->ne[2] != 1)
+ throw std::runtime_error("IP-Adapter token batch must be one or match the UNet batch");
+ return ggml_repeat_4d(ctx.ggml_ctx, value, value->ne[0], value->ne[1], query->ne[2], 1);
+ };
+ auto* key = input(kv.key);
+ auto* value = input(kv.value);
+ auto* image = ggml_ext_attention_ext(&ctx, query, key, value, heads, nullptr, false, ctx.flash_attn_enabled);
+ output = ggml_add(ctx.ggml_ctx, output, ggml_scale(ctx.ggml_ctx, image, strength));
+ }
+ return output;
+ }
+ };
+
+ const models::DenoisingParameterization& ModelPredictor::resolved_parameterization() const {
+ if (!parameterization)
+ parameterization = dit().make_parameterization();
+ return *parameterization;
+ }
+
+ const ModelPredictor& primary_predictor(const Predictor& predictor) {
+ if (const auto* model = std::get_if<ModelPredictor>(&predictor.node))
+ return *model;
+ if (const auto* branches = guidance_branches(predictor))
+ return primary_predictor(*branches->positive);
+ if (const auto* slg = std::get_if<SkipLayerPredictor>(&predictor.node))
+ return primary_predictor(*slg->base);
+ return primary_predictor(*std::get<SwitchPredictor>(predictor.node).before);
+ }
+
+ models::DiT& primary_dit(const Predictor& predictor) {
+ return primary_predictor(predictor).dit();
+ }
+
+ template <class Compatible>
+ static const models::DenoisingParameterization& common_parameterization(const Predictor& predictor, Compatible compatible, const char* error) {
+ const auto& parameterization = primary_predictor(predictor).resolved_parameterization();
+ std::set<const Predictor*> visited;
+ const auto check = [&](const auto& self, const Predictor& current) -> void {
+ if (!visited.insert(¤t).second)
+ return;
+ if (const auto* model = std::get_if<ModelPredictor>(¤t.node)) {
+ if (!compatible(parameterization, model->resolved_parameterization()))
+ throw std::runtime_error(error);
+ } else if (const auto* branches = guidance_branches(current)) {
+ self(self, *branches->positive);
+ self(self, *branches->negative);
+ if (branches->image_only)
+ self(self, *branches->image_only);
+ } else if (const auto* slg = std::get_if<SkipLayerPredictor>(¤t.node)) {
+ self(self, *slg->base);
+ if (slg->scale != 0.f) {
+ self(self, *slg->positive);
+ self(self, *slg->skipped);
+ }
+ } else {
+ const auto& transition = std::get<SwitchPredictor>(current.node);
+ self(self, *transition.before);
+ self(self, *transition.after);
+ }
+ };
+ check(check, predictor);
+ return parameterization;
+ }
+
+ const models::DenoisingParameterization& sampling_parameterization(const Predictor& predictor) {
+ return common_parameterization(
+ predictor, [](const auto& a, const auto& b) { return a.compatible_noise(b); },
+ "predictors must use compatible noise conventions");
+ }
+
+ const models::DenoisingParameterization& scaling_parameterization(const Predictor& predictor) {
+ return common_parameterization(
+ predictor, [](const auto& a, const auto& b) { return a.compatible_scalings(b); },
+ "denoiser_scalings requires matching scaling rules across all branches; pass a model predictor");
+ }
+
+ const ModelPredictor& schedule_predictor(const Predictor& predictor) {
+ common_parameterization(
+ predictor, [](const auto& a, const auto& b) { return &a == &b; },
+ "ambiguous schedule source: make_sigmas and t_to_sigma require one shared denoising parameterization; pass the model predictor that defines the schedule");
+ return primary_predictor(predictor);
+ }
+
+ bool provides_model_output(const Predictor& predictor) {
+ std::set<const Predictor*> visited;
+ const auto check = [&](const auto& self, const Predictor& branch) -> bool {
+ if (!visited.insert(&branch).second)
+ return true;
+ if (const auto* transition = std::get_if<SwitchPredictor>(&branch.node))
+ return self(self, *transition->before) && self(self, *transition->after);
+ return std::holds_alternative<ModelPredictor>(branch.node);
+ };
+ return check(check, predictor);
+ }
+
+ models::DiffusionPrediction PredictionExecution::Resources::predict_model(const ModelPredictor& leaf, const Tensor<float>& x, float sigma, models::DiffusionPredictionStep step, EvaluationState& state) {
+ auto& engine = leaf.dit();
+ prepare_predictor(leaf);
+ models::CircularAxesScope circular_axes(engine, leaf.circular_x, leaf.circular_y);
+ std::map<models::ControlNet*, models::CircularAxesScope> control_axes;
+ if (auto* motion = leaf.model->motion.get()) {
+ if (prepared.try_emplace(motion, *motion).second)
+ motion->validate_latent(x);
+ step.unet_block_processor = motion;
+ }
+ step.compute_controls = [&step](const Tensor<float>& input, const Tensor<float>& timesteps, const SDCondition& condition) {
+ return compute_unet_controls(step.controls, input, timesteps, condition);
+ };
+ std::vector<const IPAdapterBinding*> active_adapters;
+ std::vector<const PuLIDBinding*> active_pulid;
+ const auto bindings = windows.find(&leaf);
+ if (bindings != windows.end()) {
+ for (const auto& window : bindings->second.controls) {
+ if (window.range.active(sigma)) {
+ auto& model = *window.binding->model;
+ prepared.try_emplace(&model, model);
+ control_axes.try_emplace(&model, model, leaf.circular_x, leaf.circular_y);
+ step.controls.push_back(window.binding);
+ }
+ }
+ for (const auto& window : bindings->second.ip_adapters)
+ if (window.range.active(sigma))
+ active_adapters.push_back(window.binding);
+ for (const auto& window : bindings->second.pulid) {
+ if (window.range.active(sigma)) {
+ auto& model = *window.binding->model;
+ prepared.try_emplace(&model, model);
+ active_pulid.push_back(window.binding);
+ }
+ }
+ }
+ const auto cache = caches.find(&leaf.resolved_parameterization());
+ if (cache != caches.end()) {
+ step.cache = &state.caches.try_emplace(&leaf.resolved_parameterization(), cache->second, step.solver_index, sigma).first->second;
+ if (bindings != windows.end()) {
+ auto& previous = previous_conditioning[&leaf];
+ if (previous.controls != step.controls || previous.ip_adapters != active_adapters || previous.pulid != active_pulid) {
+ step.cache->invalidate_condition(&leaf.condition);
+ previous = {step.controls, active_adapters, active_pulid};
+ }
+ }
+ }
+ IPAdapterAttention attention(active_adapters, ip_adapter_cache);
+ if (!active_adapters.empty())
+ step.attn2_processor = &attention;
+ PuLIDAttention pulid(active_pulid, pulid_cache);
+ if (!active_pulid.empty())
+ step.flux_image_processor = &pulid;
+ sd::ParallelScope tensor_scope(engine.tensor_executor.get());
+ auto prediction = engine.predict_diffusion(x, leaf.resolved_parameterization(), sigma, leaf.condition, leaf.references,
+ leaf.options, step);
+ if (prediction.output.empty())
+ throw std::runtime_error("diffusion prediction failed");
+ return prediction;
+ }
+
+ models::DiffusionPrediction PredictionExecution::Resources::predict_batch(const ModelPredictor& leaf, const SDCondition& condition, const Tensor<float>& x, float sigma, const models::DiffusionPredictionStep& step) {
+ auto& model = leaf.dit();
+ prepare_predictor(leaf);
+ models::CircularAxesScope circular_axes(model, leaf.circular_x, leaf.circular_y);
+ sd::ParallelScope tensor_scope(model.tensor_executor.get());
+ auto prediction = model.predict_diffusion(sd::ops::concat(x, x, 3), leaf.resolved_parameterization(), sigma, condition, {},
+ leaf.options, step);
+ if (prediction.output.empty())
+ throw std::runtime_error("batched diffusion prediction failed");
+ return prediction;
+ }
+
+ class PredictionExecution::Evaluation {
+ struct Result {
+ models::DiffusionPrediction prediction;
+ Tensor<float> denoised;
+ };
+
+ PredictionExecution& execution_;
+ const Tensor<float>& x_;
+ float sigma_;
+ const models::DiffusionPredictionStep& step_;
+ EvaluationState& state_;
+ const Predictor* unconditional_;
+ std::map<const Predictor*, Result> results_;
+
+ const Predictor& active_branch(const Predictor& predictor) const {
+ return selected_predictor(predictor, sigma_, step_);
+ }
+
+ void batch_cfg(const GuidanceBranches& branches) {
+ auto& resources = *execution_.resources_;
+ if (!resources.options.batched_cfg || branches.image_only || x_.dim() != 4 || x_.shape().back() != 1)
+ return;
+ const auto& positive = active_branch(*branches.positive);
+ const auto& negative = active_branch(*branches.negative);
+ if (&positive == &negative || results_.count(&positive) || results_.count(&negative))
+ return;
+ const auto* pos = std::get_if<ModelPredictor>(&positive.node);
+ const auto* neg = std::get_if<ModelPredictor>(&negative.node);
+ if (!pos || !neg || &pos->dit() != &neg->dit() || !sd_version_is_unet(pos->dit().version()) ||
+ !pos->resolved_parameterization().compatible_forward(neg->resolved_parameterization()) ||
+ pos->circular_x != neg->circular_x || pos->circular_y != neg->circular_y ||
+ pos->model->motion || neg->model->motion || pos->controls || neg->controls ||
+ pos->ip_adapters || neg->ip_adapters || pos->pulid || neg->pulid ||
+ pos->options.transform || neg->options.transform ||
+ !pos->references.empty() || !neg->references.empty() ||
+ !pos->condition.c_ref_images.empty() || !neg->condition.c_ref_images.empty())
+ return;
+
+ if (pos && neg) {
+ resources.prepare_predictor(*pos);
+ resources.prepare_predictor(*neg);
+ for (const auto* model : {pos, neg}) {
+ const auto cache = resources.caches.find(&model->resolved_parameterization());
+ if (cache != resources.caches.end() && cache->second.enabled())
+ return;
+ }
+ }
+ const auto key = std::make_pair(pos, neg);
+ auto packed = resources.batched_conditions.find(key);
+ if (packed == resources.batched_conditions.end()) {
+ if (!compatible_condition_tensors(pos->condition.c_crossattn, neg->condition.c_crossattn, 3) ||
+ !compatible_condition_tensors(pos->condition.c_vector, neg->condition.c_vector, 2) ||
+ !compatible_condition_tensors(pos->condition.c_concat, neg->condition.c_concat, 4))
+ return;
+ for (const auto* condition : {&pos->condition, &neg->condition}) {
+ if (condition->c_concat.empty())
+ continue;
+ const auto& shape = condition->c_concat.shape();
+ if (shape.size() != 4 || shape[0] != x_.shape()[0] || shape[1] != x_.shape()[1])
+ return;
+ }
+ SDCondition condition{batch_condition_tensor(pos->condition.c_crossattn, neg->condition.c_crossattn, 3),
+ batch_condition_tensor(pos->condition.c_vector, neg->condition.c_vector, 2),
+ batch_condition_tensor(pos->condition.c_concat, neg->condition.c_concat, 4)};
+ packed = resources.batched_conditions.emplace(key, std::move(condition)).first;
+ LOG_VERBOSE("batching compatible positive and negative UNet branches");
+ }
+ auto prediction = resources.predict_batch(*pos, packed->second, x_, sigma_, step_);
+ auto parts = sd::ops::chunk(prediction.output, 2, 3);
+ results_.emplace(&positive, Result{{std::move(parts[0]), prediction.skip_scale, prediction.output_scale}});
+ results_.emplace(&negative, Result{{std::move(parts[1]), prediction.skip_scale, prediction.output_scale}});
+ }
+
+ Tensor<float> to_denoised(const models::DiffusionPrediction& prediction, const Tensor<float>& raw) const {
+ return x_ * prediction.skip_scale + raw * prediction.output_scale;
+ }
+
+ const Tensor<float>& denoised(Result& result) const {
+ if (result.denoised.empty())
+ result.denoised = to_denoised(result.prediction, result.prediction.output);
+ return result.denoised;
+ }
+
+ const Tensor<float>& denoised(const Predictor& predictor) {
+ return denoised(evaluate(predictor));
+ }
+
+ const Tensor<float>& matching_raw(const Predictor& predictor, const models::DiffusionPrediction& positive) {
+ const auto& prediction = evaluate(predictor).prediction;
+ if (prediction.output.empty())
+ throw std::runtime_error("APG requires model outputs from each branch");
+ if (prediction.skip_scale != positive.skip_scale || prediction.output_scale != positive.output_scale)
+ throw std::runtime_error("APG predictors must use matching prediction scalings");
+ return prediction.output;
+ }
+
+ Result cfg(const GuidanceBranches& branches) {
+ const auto scale = branches.scale_at(step_.index);
+ if (scale != 1.f || branches.negative.get() == unconditional_)
+ batch_cfg(branches);
+ const auto& pos = denoised(*branches.positive);
+ if (branches.image_only) {
+ const auto& neg = denoised(*branches.negative);
+ const auto& img = denoised(*branches.image_only);
+ return {{}, neg + (img - neg) * branches.image_scale + (pos - img) * scale};
+ }
+ if (scale == 1.f)
+ return {{}, pos};
+ const auto& neg = denoised(*branches.negative);
+ return {{}, neg + (pos - neg) * scale};
+ }
+
+ Result apg(const ApgPredictor& node) {
+ const auto& branches = node.branches;
+ const auto scale = branches.scale_at(step_.index);
+ auto& positive = evaluate(*branches.positive);
+ if (scale == 1.f && !branches.image_only)
+ return {{}, denoised(positive)};
+
+ const auto& prediction = positive.prediction;
+ if (prediction.output.empty())
+ throw std::runtime_error("APG requires model outputs from each branch");
+ const auto& neg = matching_raw(*branches.negative, prediction);
+ const auto* img = branches.image_only ? &matching_raw(*branches.image_only, prediction) : nullptr;
+
+ auto& guidance = execution_.apg_.try_emplace(&node, branches.scale, branches.image_scale, node.params).first->second;
+ sd::guidance::GuidanceInput input;
+ input.step = step_.index;
+ input.schedule_size = step_.schedule_size;
+ input.pred_cond = &prediction.output;
+ input.pred_uncond = img ? img : &neg;
+ input.pred_img_uncond = img ? &neg : nullptr;
+ guidance.set_guidance_scale(scale);
+ auto result = guidance.forward(input, {});
+ return {{}, to_denoised(prediction, result.pred)};
+ }
+
+ Result slg(const SkipLayerPredictor& node) {
+ auto& base = evaluate(*node.base);
+ const auto& pred = denoised(base);
+ const auto& pos = denoised(*node.positive);
+ const auto& skipped = denoised(*node.skipped);
+ return {{}, pred + (pos - skipped) * node.scale};
+ }
+
+ Result& evaluate(const Predictor& predictor) {
+ const auto& selected = selected_predictor(predictor, sigma_, step_);
+ if (&selected != &predictor)
+ return evaluate(selected);
+
+ auto found = results_.find(&predictor);
+ if (found != results_.end())
+ return found->second;
+
+ Result result;
+ if (const auto* model = std::get_if<ModelPredictor>(&predictor.node)) {
+ result.prediction = execution_.resources_->predict_model(*model, x_, sigma_, step_, state_);
+ } else if (const auto* node = std::get_if<CfgPredictor>(&predictor.node)) {
+ result = cfg(node->branches);
+ } else if (const auto* node = std::get_if<ApgPredictor>(&predictor.node)) {
+ result = apg(*node);
+ } else {
+ result = slg(std::get<SkipLayerPredictor>(predictor.node));
+ }
+ return results_.emplace(&predictor, std::move(result)).first->second;
+ }
+
+ public:
+ Evaluation(PredictionExecution& execution, const Tensor<float>& x, float sigma, const models::DiffusionPredictionStep& step, EvaluationState& state)
+ : execution_(execution), x_(x), sigma_(sigma), step_(step), state_(state), unconditional_(execution.resources_->options.need_unconditional ? unconditional_predictor(execution.root_, sigma, step) : nullptr) {}
+
+ sd::guidance::GuiderOutput run(bool need_unconditional) {
+ auto& result = evaluate(execution_.root_);
+ sd::guidance::GuiderOutput output;
+ if (need_unconditional) {
+ if (!unconditional_)
+ throw std::runtime_error("CFG++ requires a predictor with an unconditional branch");
+ output.pred_uncond = denoised(*unconditional_);
+ }
+ denoised(result);
+ output.pred = std::move(result.denoised);
+ return output;
+ }
+ };
+
+ sd::guidance::GuiderOutput PredictionExecution::evaluate(const Tensor<float>& x, float sigma, const models::DiffusionPredictionStep& step) {
+ auto& resources = *resources_;
+ EvaluationState state;
+ if (!resources.options.cache || resources.options.cache->mode == SD_CACHE_DISABLED)
+ return Evaluation(*this, x, sigma, step, state).run(resources.options.need_unconditional);
+ auto active = active_prediction(root_, sigma, step, resources.options.need_unconditional);
+ for (const auto* leaf : active.models) {
+ const auto windows = resources.windows.find(leaf);
+ if (windows != resources.windows.end()) {
+ const auto append = [&](const auto& entries) {
+ for (const auto& entry : entries)
+ active.conditioning.push_back(entry.range.active(sigma));
+ };
+ append(windows->second.controls);
+ append(windows->second.ip_adapters);
+ append(windows->second.pulid);
+ }
+ }
+ auto predicted = resources.spectrum.predict(active, x.shape(), step);
+ if (!predicted.empty()) {
+ sd::guidance::GuiderOutput result;
+ result.pred = std::move(predicted);
+ return result;
+ }
+ for (auto& [parameterization, cache] : resources.caches) {
+ std::vector<const void*> conditions;
+ for (const auto* leaf : active.models)
+ if (&leaf->resolved_parameterization() == parameterization)
+ conditions.push_back(&leaf->condition);
+ cache.retain_conditions(conditions);
+ state.caches.try_emplace(parameterization, cache, step.solver_index, sigma);
+ }
+ auto result = Evaluation(*this, x, sigma, step, state).run(resources.options.need_unconditional);
+ resources.spectrum.record(result.pred, step);
+ return result;
+ }
+
+ void PredictionExecution::log_cache_summary() const {
+ for (const auto& [parameterization, cache] : resources_->caches)
+ sd_sample::log_condition_cache_summary(cache, resources_->sigmas.size() - 1);
+ resources_->spectrum.log_summary();
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/predictor.h b/src/dgl/ops/predictor.h
new file mode 100644
index 0000000..8ad4fa5
--- /dev/null
+++ b/src/dgl/ops/predictor.h
@@ -0,0 +1,130 @@
+#ifndef __SD_DGL_OPS_PREDICTOR_H__
+#define __SD_DGL_OPS_PREDICTOR_H__
+
+#include "dgl/models/fwd.h"
+
+#include <cmath>
+#include <functional>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <set>
+#include <variant>
+
+#include "conditioning/conditioner.hpp"
+#include "dgl/models/denoising_parameterization.h"
+#include "dgl/models/diffusion_model.h"
+#include "dgl/models/diffusion_prediction.h"
+#include "dgl/ops/control.h"
+#include "dgl/ops/ip_adapter.h"
+#include "dgl/ops/pulid.h"
+#include "runtime/guidance.h"
+
+namespace sd {
+ struct CacheConfig;
+}
+
+namespace sd::dgl::ops {
+ struct Predictor;
+
+ struct ModelPredictor {
+ std::shared_ptr<const models::DiffusionModel> model;
+ mutable std::shared_ptr<const models::DenoisingParameterization> parameterization;
+ SDCondition condition;
+ std::shared_ptr<models::ControlBindings> controls;
+ std::shared_ptr<IPAdapterBindings> ip_adapters;
+ std::shared_ptr<PuLIDBindings> pulid;
+ models::DiffusionPredictionOptions options;
+ std::vector<sd::Tensor<float>> references;
+ bool circular_x = false;
+ bool circular_y = false;
+
+ models::DiT& dit() const { return *model->dit; }
+ const models::DenoisingParameterization& resolved_parameterization() const;
+ };
+
+ struct GuidanceBranches {
+ std::shared_ptr<Predictor> positive, negative, image_only;
+ float scale = 1.f;
+ float image_scale = 1.f;
+ std::vector<float> scale_schedule;
+
+ float scale_at(int step) const;
+ };
+
+ struct CfgPredictor {
+ GuidanceBranches branches;
+ };
+
+ struct ApgPredictor {
+ GuidanceBranches branches;
+ sd::guidance::AdaptiveProjectedGuidanceParams params;
+ };
+
+ struct SigmaWindow {
+ float start_sigma = std::numeric_limits<float>::infinity();
+ float end_sigma = 0.f;
+
+ bool active(float sigma) const { return sigma <= start_sigma && sigma >= end_sigma; }
+ bool bounded() const { return std::isfinite(start_sigma) || end_sigma > 0.f; }
+ };
+
+ struct StepWindow {
+ float start_fraction, stop_fraction;
+
+ bool active(const models::DiffusionPredictionStep& step) const;
+ };
+
+ struct SkipLayerPredictor {
+ std::shared_ptr<Predictor> base, positive, skipped;
+ float scale;
+ SigmaWindow sigmas;
+ std::optional<StepWindow> steps;
+
+ bool active(float sigma, const models::DiffusionPredictionStep& step) const;
+ };
+
+ struct SwitchPredictor {
+ std::shared_ptr<Predictor> before, after;
+ int after_steps;
+ };
+
+ struct Predictor {
+ std::variant<ModelPredictor, CfgPredictor, ApgPredictor, SkipLayerPredictor, SwitchPredictor> node;
+ };
+
+ bool provides_model_output(const Predictor& predictor);
+ models::DiT& primary_dit(const Predictor& predictor);
+ const ModelPredictor& primary_predictor(const Predictor& predictor);
+ const models::DenoisingParameterization& sampling_parameterization(const Predictor& predictor);
+ const models::DenoisingParameterization& scaling_parameterization(const Predictor& predictor);
+ const ModelPredictor& schedule_predictor(const Predictor& predictor);
+
+ struct PredictionExecutionOptions {
+ const CacheConfig* cache = nullptr;
+ bool need_unconditional = false;
+ bool dual_schedule = false;
+ bool batched_cfg = true;
+ };
+
+ class PredictionExecution {
+ class Evaluation;
+ struct Resources;
+
+ const Predictor& root_;
+ std::unique_ptr<Resources> resources_;
+ std::map<const ApgPredictor*, sd::guidance::AdaptiveProjectedGuidance> apg_;
+
+ public:
+ PredictionExecution(const Predictor& root, const std::vector<float>& sigmas, const PredictionExecutionOptions& options);
+ ~PredictionExecution();
+
+ models::DiT& dit() const;
+ const models::DenoisingParameterization& parameterization() const;
+
+ sd::guidance::GuiderOutput evaluate(const Tensor<float>& x, float sigma, const models::DiffusionPredictionStep& step);
+ void log_cache_summary() const;
+ };
+}
+
+#endif // __SD_DGL_OPS_PREDICTOR_H__
diff --git a/src/dgl/ops/preview.cpp b/src/dgl/ops/preview.cpp
new file mode 100644
index 0000000..352c271
--- /dev/null
+++ b/src/dgl/ops/preview.cpp
@@ -0,0 +1,216 @@
+#include "dgl/ops/preview.h"
+
+#include "dgl/models/denoising_parameterization.h"
+
+#include <cstdint>
+
+#include "api/tensor.h"
+#include "core/ggml_tensor_utils.h"
+#include "dgl/models/circular_axes.h"
+#include "dgl/models/diffusion_prediction.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/fwd.h"
+#include "dgl/models/vae.h"
+#include "dgl/run_events.h"
+#include "model/vae/vae.hpp"
+#include "model_definition.h"
+#include "runtime/denoiser.hpp"
+#include "runtime/latent-preview.h"
+
+namespace sd::dgl::ops {
+ static void preview_image(int step, const Tensor<float>& latents, SDVersion version, preview_t preview_mode, models::VAE* decoder, sd_preview_cb_t step_callback, void* step_callback_data, bool is_noisy) {
+ bool is_video = preview_latent_tensor_is_video(latents);
+ uint32_t dim = is_video ? static_cast<uint32_t>(latents.shape()[3]) : static_cast<uint32_t>(latents.shape()[2]);
+ const bool sefi = sd_version_is_sefi_image(version);
+ const int channel_start = sefi ? SefiFlowDenoiser::kSemChannels : 0;
+ const int channels = sefi ? SefiFlowDenoiser::kTotalChannels - channel_start
+ : decoder && preview_mode != PREVIEW_PROJ ? decoder->get_latent_channel()
+ : find_model_definition(version)->geometry.latent_channels;
+ auto _latents = channels != dim || channel_start
+ ? sd::ops::slice(latents, is_video ? 3 : 2, channel_start, channel_start + channels)
+ : latents;
+ if (preview_mode == PREVIEW_PROJ) {
+ int patch_sz = 1;
+ const float (*latent_rgb_proj)[3] = nullptr;
+ float* latent_rgb_bias = nullptr;
+ const float* latent_alpha_proj = nullptr;
+ float latent_alpha_bias = 1.f;
+
+ if (channels == 128) {
+ if (sd_version_uses_flux2_vae(version)) {
+ latent_rgb_proj = flux2_latent_rgb_proj;
+ latent_rgb_bias = flux2_latent_rgb_bias;
+ patch_sz = 2;
+ } else if (version == VERSION_LTXAV) {
+ latent_rgb_proj = ltxav_latent_rgb_proj;
+ latent_rgb_bias = ltxav_latent_rgb_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels == 64) {
+ if (version == VERSION_QWEN_IMAGE_2_1) {
+ latent_rgb_proj = qwen21_latent_rgb_proj;
+ latent_rgb_bias = qwen21_latent_rgb_bias;
+ latent_alpha_proj = qwen21_latent_alpha_proj;
+ latent_alpha_bias = qwen21_latent_alpha_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels == 48) {
+ if (sd_version_is_wan(version)) {
+ latent_rgb_proj = wan_22_latent_rgb_proj;
+ latent_rgb_bias = wan_22_latent_rgb_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels == 24) {
+ if (sd_version_is_minimax_h3(version)) {
+ latent_rgb_proj = minimax_latent_rgb_proj;
+ latent_rgb_bias = minimax_latent_rgb_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels == 16) {
+ if (sd_version_is_sd3(version)) {
+ latent_rgb_proj = sd3_latent_rgb_proj;
+ latent_rgb_bias = sd3_latent_rgb_bias;
+ } else if (sd_version_uses_flux_vae(version)) {
+ latent_rgb_proj = flux_latent_rgb_proj;
+ latent_rgb_bias = flux_latent_rgb_bias;
+ } else if (sd_version_uses_wan_vae(version)) {
+ latent_rgb_proj = wan_21_latent_rgb_proj;
+ latent_rgb_bias = wan_21_latent_rgb_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels == 4) {
+ if (sd_version_is_sdxl(version)) {
+ latent_rgb_proj = sdxl_latent_rgb_proj;
+ latent_rgb_bias = sdxl_latent_rgb_bias;
+ } else if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
+ latent_rgb_proj = sd_latent_rgb_proj;
+ latent_rgb_bias = sd_latent_rgb_bias;
+ } else {
+ LOG_WARN("No latent to RGB projection known for this model");
+ return;
+ }
+ } else if (channels != 3) {
+ LOG_WARN("No latent to RGB projection known for this model (dim = %d)", dim);
+ return;
+ }
+
+ uint32_t frames = is_video ? static_cast<uint32_t>(_latents.shape()[2]) : 1;
+ uint32_t img_width = static_cast<uint32_t>(_latents.shape()[0]) * patch_sz;
+ uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
+
+ uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3;
+ const size_t stride = size_t(img_width) * img_height * img_channels;
+ std::vector<uint8_t> data(frames * stride);
+ preview_latent_video(data.data(), _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
+ std::vector<sd::Image> images(frames);
+ for (uint32_t i = 0; i < frames; i++) {
+ images[i] = {img_width, img_height, img_channels, data.data() + i * stride};
+ }
+ sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data);
+ return;
+ }
+
+ if (decoder && (preview_mode == PREVIEW_VAE || preview_mode == PREVIEW_TAE)) {
+ auto vae_latents = decoder->dit_to_vae_latents(_latents);
+ auto decoded = decoder->first_stage_model->decode(decoder->n_threads, vae_latents, decoder->vae_tiling_params, is_video,
+ decoder->circular_x, decoder->circular_y, true);
+ if (decoded.empty()) {
+ LOG_ERROR("preview decode failed at step %d", step);
+ return;
+ }
+
+ is_video = preview_latent_tensor_is_video(decoded);
+ uint32_t frames = is_video ? static_cast<uint32_t>(decoded.shape()[2]) : 1;
+ std::vector<sd::Image> images(frames);
+ std::vector<std::unique_ptr<uint8_t, decltype(&std::free)>> pixels;
+ pixels.reserve(frames);
+ for (uint32_t i = 0; i < frames; ++i) {
+ images[i] = tensor_to_sd_image(decoded, static_cast<int>(i));
+ pixels.emplace_back(images[i].data, std::free);
+ }
+
+ sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data);
+ return;
+ }
+
+ if (preview_mode != PREVIEW_NONE) {
+ LOG_WARN("Unsupported preview mode: %d", static_cast<int>(preview_mode));
+ }
+ }
+
+ class PreviewSession final : public SamplingObserverSession {
+ SamplingRun run_;
+ std::shared_ptr<models::VAE> decoder_;
+ preview_t mode_;
+ bool circular_x_;
+ bool circular_y_;
+
+ bool selected(const SamplingStep& step) const {
+ if (!step.completes_step)
+ return false;
+ const auto interval = run_.events.callbacks().interval;
+ if (interval > 0)
+ return step.index % interval == 0;
+ if (interval < 0)
+ return step.index == -static_cast<int64_t>(interval);
+ return step.index == run_.total_steps &&
+ (run_.events.uses_legacy_preview() || run_.events.preview_info().sample_pass == 1);
+ }
+
+ void show(const SamplingStep& step, const Tensor<float>& latent, bool noisy) {
+ auto& events = run_.events;
+ events.check_cancelled();
+ const auto mode = mode_;
+ if (mode == PREVIEW_PROJ) {
+ sd::ParallelScope scope(run_.dit.tensor_executor.get());
+ preview_image(step.index, latent, run_.dit.version(), mode, nullptr,
+ events.preview_callback(), &events, noisy);
+ } else {
+ models::Model::ExecutionScope model_scope(*decoder_);
+ model_scope.require_ready();
+ models::CircularAxesScope axes(*decoder_, circular_x_, circular_y_);
+ preview_image(step.index, latent, run_.dit.version(), mode, decoder_.get(),
+ events.preview_callback(), &events, noisy);
+ }
+ events.check_cancelled();
+ }
+
+ public:
+ PreviewSession(SamplingRun run, std::shared_ptr<models::VAE> decoder, preview_t mode, bool circular_x, bool circular_y)
+ : run_(run), decoder_(std::move(decoder)), mode_(mode), circular_x_(circular_x), circular_y_(circular_y) {}
+
+ void before_prediction(const SamplingStep& step, const Tensor<float>& x) override {
+ if (!selected(step) || !run_.events.callbacks().noisy)
+ return;
+ auto noisy = x * run_.parameterization.scalings(step.sigma)[2];
+ if (step.prediction.mask_input)
+ step.prediction.blend_mask(noisy);
+ show(step, noisy, true);
+ }
+
+ void after_prediction(const SamplingStep& step, const Tensor<float>&, const sd::guidance::GuiderOutput& prediction) override {
+ if (selected(step) && run_.events.callbacks().denoised)
+ show(step, prediction.pred, false);
+ }
+ };
+
+ PreviewObserver::PreviewObserver(std::shared_ptr<models::VAE> decoder, bool circular_x, bool circular_y)
+ : decoder_(std::move(decoder)), circular_x_(circular_x), circular_y_(circular_y) {}
+
+ std::unique_ptr<SamplingObserverSession> PreviewObserver::start(const SamplingRun& run) const {
+ const auto& callbacks = run.events.callbacks();
+ if (callbacks.mode == PREVIEW_NONE || !run.events.preview_callback() || (!callbacks.noisy && !callbacks.denoised))
+ return {};
+ return std::make_unique<PreviewSession>(run, decoder_, decoder_ ? callbacks.mode : PREVIEW_PROJ, circular_x_, circular_y_);
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/preview.h b/src/dgl/ops/preview.h
new file mode 100644
index 0000000..6858c7f
--- /dev/null
+++ b/src/dgl/ops/preview.h
@@ -0,0 +1,21 @@
+#ifndef __SD_DGL_OPS_PREVIEW_H__
+#define __SD_DGL_OPS_PREVIEW_H__
+
+#include "dgl/models/fwd.h"
+
+#include "dgl/ops/sampling_observer.h"
+
+namespace sd::dgl::ops {
+
+ class PreviewObserver final : public SamplingObserver {
+ std::shared_ptr<models::VAE> decoder_;
+ bool circular_x_;
+ bool circular_y_;
+
+ public:
+ explicit PreviewObserver(std::shared_ptr<models::VAE> decoder = {}, bool circular_x = false, bool circular_y = false);
+ std::unique_ptr<SamplingObserverSession> start(const SamplingRun& run) const override;
+ };
+}
+
+#endif // __SD_DGL_OPS_PREVIEW_H__
diff --git a/src/dgl/ops/prompt.cpp b/src/dgl/ops/prompt.cpp
new file mode 100644
index 0000000..abf1a57
--- /dev/null
+++ b/src/dgl/ops/prompt.cpp
@@ -0,0 +1,156 @@
+#include "dgl/ops/prompt.h"
+
+#include <algorithm>
+#include <filesystem>
+#include <set>
+#include "conditioning/conditioner.hpp"
+#include "core/file_access.h"
+#include "core/image_tensor.h"
+#include "dgl/models/text_encoder.h"
+#include "dgl/ops/image.h"
+#include "dgl/ops/video.h"
+
+namespace sd::dgl::ops {
+ MiniMaxH3PresentationItem prepare_prompt_media(PromptMedia input) {
+ const auto* video = std::get_if<std::shared_ptr<const sd_tensor_t>>(&input.media);
+ if (input.stride < 1 || input.stride > 65536 || (input.stride != 1 && !video))
+ throw std::runtime_error("media stride requires video frames and a positive frame interval");
+ MiniMaxH3PresentationItem result;
+ if (video) {
+ if (!*video || input.fps_num <= 0 || input.fps_den <= 0)
+ throw std::runtime_error("media video requires frames and a positive frame rate");
+ const auto& tensor = **video;
+ const auto& shape = tensor.shape();
+ size_t count = 0;
+ if (tensor.type == SD_TENSOR_U8) {
+ validate_video_frames(tensor);
+ count = size_t(shape[3]);
+ } else if (tensor.type == SD_TENSOR_F32 && shape.size() == 5 &&
+ shape[0] > 0 && shape[0] <= 16384 && shape[1] > 0 && shape[1] <= 16384 &&
+ shape[2] > 0 && shape[3] == 3 && shape[4] == 1) {
+ count = size_t(shape[2]);
+ } else {
+ throw std::runtime_error("media video requires U8 [3,width,height,frames] or F32 [width,height,frames,3,1]");
+ }
+ result.kind = MiniMaxH3PresentationKind::VIDEO;
+ for (size_t i = 0; i < count; i += size_t(input.stride)) {
+ if (tensor.type == SD_TENSOR_U8) {
+ result.frames.push_back(sd_image_to_tensor(video_frame_view(tensor, i)));
+ } else {
+ Tensor<float> frame({shape[0], shape[1], 3, 1});
+ const auto plane = size_t(shape[0] * shape[1]);
+ for (size_t channel = 0; channel < 3; ++channel)
+ std::copy_n(tensor.as<float>().data() + (channel * count + i) * plane, plane, frame.data() + channel * plane);
+ result.frames.push_back(std::move(frame));
+ }
+ result.timestamps.push_back(float(i) * float(input.fps_den) / float(input.fps_num));
+ }
+ } else if (std::holds_alternative<AudioReadResult>(input.media)) {
+ const auto& audio = std::get<AudioReadResult>(input.media);
+ if (!audio.samples)
+ throw std::runtime_error("media audio requires samples");
+ audio_view(*audio.samples, audio.sample_rate);
+ result.kind = MiniMaxH3PresentationKind::AUDIO;
+ } else {
+ result.kind = MiniMaxH3PresentationKind::IMAGE;
+ auto image = std::get<Tensor<float>>(std::move(input.media));
+ if (image.dim() != 4 || image.empty() || image.shape()[3] != 1 || image.shape()[2] == 2)
+ throw std::runtime_error("media image must be a single frame with 1, 3 or 4 channels");
+ if (image.shape()[2] == 1)
+ image = sd::ensure_image_tensor_channels(image, 3);
+ result.frames.push_back(std::move(image));
+ }
+ return result;
+ }
+
+ SDCondition encode_prompt(models::TextEncoder& model, PromptEncodingOptions options) {
+ if (options.raw && options.system)
+ throw std::runtime_error("raw prompt encoding cannot be combined with system");
+ if (options.raw && (options.media || options.image_bindings))
+ throw std::runtime_error("raw prompt encoding accepts text only; omit media and image_bindings");
+ std::vector<Tensor<float>> images;
+ std::vector<MiniMaxH3PresentationItem> presentations;
+ if (options.media && !options.media->empty()) {
+ const bool images_only = std::all_of(options.media->begin(), options.media->end(), [](const PromptMedia& input) {
+ return std::holds_alternative<Tensor<float>>(input.media);
+ });
+ if (images_only && options.media->size() > model.max_reference_images())
+ throw std::runtime_error("this text encoder accepts at most " + std::to_string(model.max_reference_images()) + " media images");
+ if (!images_only && !model.supports_multimodal_prompt())
+ throw std::runtime_error("this text encoder does not support video or audio prompt media");
+ for (size_t index = 0; index < options.media->size(); ++index) {
+ try {
+ auto prepared = prepare_prompt_media(std::move((*options.media)[index]));
+ if (images_only)
+ images.push_back(std::move(prepared.frames.front()));
+ else
+ presentations.push_back(std::move(prepared));
+ } catch (const std::exception& error) {
+ throw std::runtime_error("media[" + std::to_string(index) + "]: " + error.what());
+ }
+ }
+ }
+ models::Model::ExecutionScope scope(model);
+ scope.require_ready();
+ ConditionerParams params;
+ params.text = options.text;
+ params.width = options.width;
+ params.height = options.height;
+ params.clip_skip = options.clip_skip;
+ params.raw = options.raw;
+ params.system_prompt = options.system;
+ if (!images.empty())
+ params.ref_images = &images;
+ if (!presentations.empty())
+ params.minimax_h3_references = &presentations;
+ if (params.raw) {
+ auto condition = model.get_learned_condition(params);
+ if (condition.empty())
+ throw std::runtime_error("prompt encoding failed");
+ return condition;
+ }
+ auto prompt = sd::parse_prompt_input(params.text, true);
+ std::set<std::string> embedding_paths;
+ for (auto& part : prompt) {
+ if (part.type != sd::PromptPart::Type::EMBEDDING)
+ continue;
+ if (!model.supports_textual_inversion())
+ throw std::runtime_error("inline textual inversion requires an SD 1.x, SD 2.x or SDXL CLIP text encoder");
+ sd::require_file_access(part.content, SD_DGL_FILE_READ_MODEL);
+ part.content = std::filesystem::weakly_canonical(std::filesystem::u8path(part.content)).generic_u8string();
+ ModelLoader::FileStamp stamp;
+ if (!ModelLoader::read_file_stamp(part.content, stamp))
+ throw std::runtime_error("cannot inspect textual inversion '" + part.content + "'");
+ if (!params.textual_inversion)
+ params.textual_inversion = std::make_shared<sd::TextualInversion>();
+ if (embedding_paths.insert(part.content).second)
+ params.textual_inversion->sources.push_back(std::move(stamp));
+ }
+ if (sd::prompt_has_images(prompt)) {
+ if (!model.supports_prompt_images())
+ throw std::runtime_error("inline prompt images require a compatible text encoder and vision weights");
+ if (!sd::resolve_prompt_images(prompt, [&](const std::string& source) {
+ auto image = options.image_bindings ? options.image_bindings(source) : Tensor<float>{};
+ if (image.empty())
+ image = sd_image_to_tensor(image_view(*read_image_tensor(source, 3)));
+ if (image.dim() != 4)
+ throw std::runtime_error("inline prompt images must be single images");
+ if (image.shape()[2] == 3)
+ return image;
+ if (image.shape()[2] == 2)
+ image = sd::ops::slice(image, 2, 0, 1);
+ return sd::ensure_image_tensor_channels(image, 3);
+ }))
+ throw std::runtime_error("cannot resolve inline prompt images");
+ params.prompt_input = &prompt;
+ }
+ if (params.textual_inversion)
+ params.prompt_input = &prompt;
+ if (!params.prompt_input)
+ params.text = sd::prompt_input_text(prompt);
+ auto condition = model.get_learned_condition(params);
+ if (condition.empty())
+ throw std::runtime_error("prompt encoding failed");
+ return condition;
+ }
+}
diff --git a/src/dgl/ops/prompt.h b/src/dgl/ops/prompt.h
new file mode 100644
index 0000000..d4942ac
--- /dev/null
+++ b/src/dgl/ops/prompt.h
@@ -0,0 +1,38 @@
+#ifndef __SD_DGL_OPS_PROMPT_H__
+#define __SD_DGL_OPS_PROMPT_H__
+
+#include <optional>
+#include <variant>
+
+#include "conditioning/prompt_input.h"
+#include "dgl/models/fwd.h"
+#include "dgl/ops/audio.h"
+
+struct MiniMaxH3PresentationItem;
+struct SDCondition;
+
+namespace sd::dgl::ops {
+ struct PromptMedia {
+ std::variant<Tensor<float>, std::shared_ptr<const sd_tensor_t>, AudioReadResult> media;
+ int64_t stride = 1;
+ int64_t fps_num = 24;
+ int64_t fps_den = 1;
+ };
+
+ struct PromptEncodingOptions {
+ std::string text;
+ int width = 512;
+ int height = 512;
+ int clip_skip = -1;
+ bool raw = false;
+ std::optional<std::string> system;
+ std::optional<std::vector<PromptMedia>> media;
+ // An empty tensor means the image has no host binding and should be read from its path.
+ sd::PromptImageLoader image_bindings;
+ };
+
+ MiniMaxH3PresentationItem prepare_prompt_media(PromptMedia input);
+ SDCondition encode_prompt(models::TextEncoder& model, PromptEncodingOptions options);
+}
+
+#endif // __SD_DGL_OPS_PROMPT_H__
diff --git a/src/dgl/ops/pulid.cpp b/src/dgl/ops/pulid.cpp
new file mode 100644
index 0000000..a60fa98
--- /dev/null
+++ b/src/dgl/ops/pulid.cpp
@@ -0,0 +1,50 @@
+#include "dgl/ops/pulid.h"
+
+#include "dgl/models/fwd.h"
+#include "dgl/models/pulid.h"
+#include "model/adapter/pulid.hpp"
+
+namespace sd::dgl::ops {
+ PuLIDAttention::PuLIDAttention(const std::vector<const PuLIDBinding*>& bindings, PuLIDCache& cache) {
+ adapters_.reserve(bindings.size());
+ for (const auto* binding : bindings) {
+ auto& projections = cache[binding->model.get()];
+ auto entry = projections.find(binding->embedding.get());
+ if (entry == projections.end()) {
+ auto kv = binding->model->compute_kv(*binding->embedding);
+ if (kv.empty())
+ throw std::runtime_error("PuLID key/value projection failed");
+ entry = projections.emplace(binding->embedding.get(), std::move(kv)).first;
+ }
+ const auto existing = std::find_if(adapters_.begin(), adapters_.end(), [&](const auto& adapter) {
+ return adapter.kv == &entry->second;
+ });
+ if (existing == adapters_.end())
+ adapters_.push_back({binding->model.get(), &entry->second, binding->strength});
+ else
+ existing->strength += binding->strength;
+ }
+ }
+
+ bool PuLIDAttention::applies(bool single_stream, int block) const {
+ return !adapters_.empty() && block % (single_stream ? PuLIDRunner::single_interval : PuLIDRunner::double_interval) == 0;
+ }
+
+ ggml_tensor* PuLIDAttention::apply(GGMLRunnerContext& ctx, bool single_stream, int block, ggml_tensor* image) const {
+ const size_t layer = single_stream ? PuLIDRunner::double_layers + block / PuLIDRunner::single_interval
+ : block / PuLIDRunner::double_interval;
+ auto* output = image;
+ for (const auto& adapter : adapters_) {
+ auto* delta = adapter.model->attend(ctx, layer, adapter.kv->at(layer), image);
+ output = ggml_add(ctx.ggml_ctx, output, ggml_scale(ctx.ggml_ctx, delta, adapter.strength));
+ }
+ return output;
+ }
+
+ size_t PuLIDAttention::extra_graph_size() const {
+ size_t size = 0;
+ for (const auto& adapter : adapters_)
+ size += adapter.model->extra_graph_size();
+ return size;
+ }
+}
diff --git a/src/dgl/ops/pulid.h b/src/dgl/ops/pulid.h
new file mode 100644
index 0000000..ba6869e
--- /dev/null
+++ b/src/dgl/ops/pulid.h
@@ -0,0 +1,44 @@
+#ifndef __SD_DGL_OPS_PULID_H__
+#define __SD_DGL_OPS_PULID_H__
+
+#include "dgl/models/fwd.h"
+
+#include <map>
+#include <memory>
+
+#include "core/tensor.hpp"
+#include "model/diffusion/model.hpp"
+
+namespace sd::dgl::ops {
+
+ struct PuLIDBinding {
+ std::shared_ptr<models::PuLID> model;
+ std::shared_ptr<const Tensor<float>> embedding;
+ float strength;
+ float start_sigma;
+ float end_sigma;
+ };
+
+ struct PuLIDBindings {
+ std::vector<PuLIDBinding> entries;
+ };
+
+ using PuLIDCache = std::map<const models::PuLID*, std::map<const Tensor<float>*, std::vector<Tensor<float>>>>;
+
+ class PuLIDAttention final : public FluxImageBlockProcessor {
+ struct Projected {
+ const models::PuLID* model;
+ const std::vector<Tensor<float>>* kv;
+ float strength;
+ };
+ std::vector<Projected> adapters_;
+
+ public:
+ PuLIDAttention(const std::vector<const PuLIDBinding*>& bindings, PuLIDCache& cache);
+ bool applies(bool single_stream, int block) const override;
+ ggml_tensor* apply(GGMLRunnerContext& ctx, bool single_stream, int block, ggml_tensor* image) const override;
+ size_t extra_graph_size() const override;
+ };
+}
+
+#endif // __SD_DGL_OPS_PULID_H__
diff --git a/src/dgl/ops/relay_vsr.cpp b/src/dgl/ops/relay_vsr.cpp
new file mode 100644
index 0000000..4f30e72
--- /dev/null
+++ b/src/dgl/ops/relay_vsr.cpp
@@ -0,0 +1,141 @@
+#include "dgl/ops/relay_vsr.h"
+
+#include "dgl/models/fwd.h"
+#include "dgl/ops/tensor.h"
+#include "dgl/run_events.h"
+#include "json.hpp"
+#include "model/diffusion/relay_vsr.hpp"
+#include "model/upscaler/flash_vsr.hpp"
+
+namespace sd {
+ namespace dgl::ops {
+
+ static int64_t aligned_vsr_size(int64_t size, int scale) {
+ const int alignment = 32 / scale;
+ return (size + alignment - 1) / alignment * alignment;
+ }
+
+ static sd::Tensor<float> pad_vsr_frame(sd::Tensor<float> result, int64_t width, int64_t height) {
+ const auto w = result.shape()[0], h = result.shape()[1];
+ if (width > w) {
+ auto edge = sd::ops::slice(result, 0, w - 1, w);
+ auto padding = sd::zeros<float>({width - w, h, 3, 1}) + edge;
+ result = sd::ops::concat(result, padding, 0);
+ }
+ if (height > h) {
+ auto edge = sd::ops::slice(result, 1, h - 1, h);
+ auto padding = sd::zeros<float>({width, height - h, 3, 1}) + edge;
+ result = sd::ops::concat(result, padding, 1);
+ }
+ return result;
+ }
+
+ static sd::Tensor<float> prepare_vsr_frame(const sd::Image& frame, int64_t width, int64_t height) {
+ auto result = sd_image_to_tensor(frame, -1, -1, false);
+ result /= 127.5f;
+ result -= 1.f;
+ return pad_vsr_frame(std::move(result), width, height);
+ }
+
+ static sd::Tensor<float> finish_vsr_frame(const sd::Tensor<float>& input, const sd::Tensor<float>& residual, int scale, int64_t width, int64_t height, float detail) {
+ auto output = sd::ops::interpolate(input, {input.shape()[0] * scale, input.shape()[1] * scale, 3, 1}, sd::ops::InterpolateMode::Bicubic);
+ if (detail == 1.f)
+ output += residual;
+ else
+ output += residual * detail;
+ output += 1.f;
+ output *= 127.5f;
+ output = sd::ops::slice(sd::ops::slice(output, 0, 0, width), 1, 0, height);
+ if (!std::all_of(output.values().begin(), output.values().end(), [](float value) { return std::isfinite(value); }))
+ throw std::runtime_error("RelayVSR produced non-finite pixels");
+ output = sd::ops::clamp(output, 0.f, 255.f);
+ output.permute_({2, 0, 1, 3});
+ return output;
+ }
+
+ static std::vector<int64_t> vsr_latent_shape(int scale, int64_t width, int64_t height, int64_t frames) {
+ const int stride = 16 / scale;
+ return {aligned_vsr_size(width, scale) / stride, aligned_vsr_size(height, scale) / stride, frames, 48, 1};
+ }
+
+ std::vector<int64_t> relay_vsr_latent_shape(const DiffusionModelRunner& model, int64_t width, int64_t height, int64_t frames) {
+ const auto* wan = dynamic_cast<const RelayVSR::WanVSRRunner*>(&model);
+ if (!wan)
+ throw std::runtime_error("expected a RelayVSR DiT");
+ return vsr_latent_shape(wan->config.upscale_factor, width, height, frames);
+ }
+
+ Tensor<float> relay_vsr_positive_embedding(const DiffusionModelRunner& model, ModelManager::ResourceHandle& weights) {
+ const auto* wan = dynamic_cast<const RelayVSR::WanVSRRunner*>(&model);
+ if (!wan)
+ throw std::runtime_error("expected a RelayVSR DiT");
+ std::vector<float> values;
+ if (!weights.load_float_tensor("wan.prompt_embeds", values))
+ throw std::runtime_error("cannot read RelayVSR positive embedding");
+ auto tensor = Tensor<float>::from_vector(std::move(values));
+ tensor.reshape_({wan->config.text_dim, wan->config.text_len});
+ return tensor;
+ }
+
+ Tensor<uint8_t> relay_vsr_decode(::VAE& model, int threads, const Tensor<float>& latent, const Tensor<uint8_t>& frames, const Tensor<int64_t>& indices, float detail) {
+ auto* flash = dynamic_cast<RelayVSR::FlashVSRRunner*>(&model);
+ if (!flash)
+ throw std::runtime_error("expected a RelayVSR Flash VAE");
+ const int scale = flash->config.upscale_factor;
+ if (frames.dim() != 4 || frames.shape()[0] != 3 || frames.empty())
+ throw std::runtime_error("RelayVSR expects nonempty U8 RGB frames");
+ const auto w = frames.shape()[1], h = frames.shape()[2], count = frames.shape()[3];
+ const auto& anchors = indices.values();
+ if (indices.dim() != 1 || anchors.empty() || anchors.front() != 0 || anchors.back() != count - 1 ||
+ std::adjacent_find(anchors.begin(), anchors.end(), std::greater_equal<int64_t>()) != anchors.end())
+ throw std::runtime_error("RelayVSR keyframe indices must increase from the first to the last frame");
+ if (latent.shape() != vsr_latent_shape(scale, w, h, indices.numel()))
+ throw std::runtime_error("RelayVSR keyframe latents do not match the frames, keyframe indices or decoder scale");
+ const auto ow = w * scale, oh = h * scale;
+ if (ow > 16384 || oh > 16384)
+ throw std::runtime_error("RelayVSR output dimensions exceed 16384 pixels");
+ element_count({3, ow, oh, count}, SD_TENSOR_U8);
+
+ flash->reset();
+ struct Reset {
+ RelayVSR::FlashVSRRunner& model;
+ ~Reset() { model.reset(); }
+ } reset{*flash};
+ Tensor<uint8_t> output({3, ow, oh, count});
+ const auto pw = aligned_vsr_size(w, scale), ph = aligned_vsr_size(h, scale);
+ auto cache_anchor = [&](int64_t i, int slot) {
+ auto frame = sd::ops::slice(latent, 2, i, i + 1);
+ frame.reshape_({latent.shape()[0], latent.shape()[1], 48, 1});
+ if (flash->compute(threads, frame, slot).empty())
+ throw std::runtime_error("RelayVSR Flash could not encode a keyframe");
+ };
+ int start_slot = 0;
+ cache_anchor(0, start_slot);
+ for (size_t i = 0; i < std::max<size_t>(1, anchors.size() - 1); ++i) {
+ const bool has_next = i + 1 < anchors.size();
+ const auto first = anchors[i], last = has_next ? anchors[i + 1] : first;
+ const int end_slot = has_next ? 1 - start_slot : start_slot;
+ if (has_next)
+ cache_anchor(int64_t(i + 1), end_slot);
+ flash->history = false;
+ for (auto j = first; j <= last; ++j) {
+ check_run_cancelled();
+ sd::Image frame{uint32_t(w), uint32_t(h), 3, const_cast<uint8_t*>(frames.data() + j * 3 * w * h)};
+ auto input = prepare_vsr_frame(frame, pw, ph);
+ auto residual = flash->compute(threads, input, -1, start_slot, end_slot);
+ if (residual.empty())
+ throw std::runtime_error("RelayVSR Flash decoding failed");
+ if (i != 0 && j == first)
+ continue;
+ auto pixels = finish_vsr_frame(input, residual, scale, ow, oh, detail);
+ std::transform(pixels.values().begin(), pixels.values().end(), output.data() + j * 3 * ow * oh,
+ [](float pixel) { return uint8_t(pixel); });
+ report_run_frames("video_upscale", int(j + 1), int(count));
+ }
+ start_slot = end_slot;
+ }
+ return output;
+ }
+
+ } // namespace dgl::ops
+} // namespace sd
diff --git a/src/dgl/ops/relay_vsr.h b/src/dgl/ops/relay_vsr.h
new file mode 100644
index 0000000..fe27d73
--- /dev/null
+++ b/src/dgl/ops/relay_vsr.h
@@ -0,0 +1,17 @@
+#ifndef __SD_DGL_OPS_RELAY_VSR_H__
+#define __SD_DGL_OPS_RELAY_VSR_H__
+
+#include "dgl/models/fwd.h"
+
+#include "core/tensor.hpp"
+#include "model_build_context.h"
+#include "model_component.h"
+#include "model_manager.h"
+
+namespace sd::dgl::ops {
+ std::vector<int64_t> relay_vsr_latent_shape(const DiffusionModelRunner& model, int64_t width, int64_t height, int64_t frames);
+ Tensor<float> relay_vsr_positive_embedding(const DiffusionModelRunner& model, ModelManager::ResourceHandle& weights);
+ Tensor<uint8_t> relay_vsr_decode(::VAE& model, int threads, const Tensor<float>& latent, const Tensor<uint8_t>& frames, const Tensor<int64_t>& indices, float detail);
+} // namespace sd::dgl::ops
+
+#endif // __SD_DGL_OPS_RELAY_VSR_H__
diff --git a/src/dgl/ops/sampling.cpp b/src/dgl/ops/sampling.cpp
new file mode 100644
index 0000000..f040539
--- /dev/null
+++ b/src/dgl/ops/sampling.cpp
@@ -0,0 +1,103 @@
+#include "dgl/ops/sampling.h"
+
+#include <cmath>
+
+#include "core/util.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/fwd.h"
+#include "dgl/ops/predictor.h"
+#include "dgl/ops/sampling_observer.h"
+#include "dgl/run_events.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl::ops {
+ const std::vector<float>& sampling_schedule(const sd::Tensor<float>& tensor, bool allow_repeated) {
+ const auto& sigmas = tensor.values();
+ if (!allow_repeated && sigmas.size() == 1 && std::isfinite(sigmas.front()) && sigmas.front() >= 0.f)
+ return sigmas;
+ if (sigmas.size() < 2 || sigmas.front() <= 0.f)
+ throw std::runtime_error("sampling requires at least two sigmas and a positive first sigma");
+ for (size_t i = 0; i < sigmas.size(); ++i) {
+ if (!std::isfinite(sigmas[i]) || sigmas[i] < 0.f ||
+ (i && (sigmas[i] > sigmas[i - 1] || (!allow_repeated && sigmas[i] == sigmas[i - 1]))))
+ throw std::runtime_error(allow_repeated ? "sigmas must be finite, nonnegative and nonincreasing"
+ : "sigmas must be finite, nonnegative and strictly decreasing");
+ }
+ return sigmas;
+ }
+
+ static void validate_mask(const Tensor<float>& input, const SamplingOptions& options) {
+ if (!options.denoise_mask)
+ return;
+ if (!options.base)
+ throw std::runtime_error("denoise_mask requires a clean base tensor");
+ const auto& mask = *options.denoise_mask;
+ if (options.base->shape() != input.shape() || tensor_broadcast_shape(mask.shape(), input.shape()) != input.shape())
+ throw std::runtime_error("mask or base shape differs from the sampling tensor");
+ if (!std::all_of(mask.values().begin(), mask.values().end(), [](float value) { return value >= 0.f && value <= 1.f; }))
+ throw std::runtime_error("mask values must be between zero and one");
+ }
+
+ Tensor<float> sample(const Tensor<float>& input, const std::vector<float>& sigmas, const Predictor& predictor, const std::shared_ptr<RNG>& rng, const SamplingOptions& options, RunEvents& events) {
+ if (sigmas.size() == 1)
+ return input;
+ const auto method = str_to_sample_method(options.method.c_str());
+ if (method == SAMPLE_METHOD_COUNT)
+ throw std::runtime_error("unknown sampler '" + options.method + "'");
+ const bool need_unconditional = method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD;
+ PredictionExecution execution(predictor, sigmas, {options.cache ? &*options.cache : nullptr, need_unconditional, false, options.batched_cfg});
+ const auto& parameterization = execution.parameterization();
+ SamplingSolver solver;
+ solver.execute = [&](const Tensor<float>& start, const SamplePrediction& evaluate) {
+ return sample_k_diffusion(method, evaluate, start, sigmas, rng, options.eta, parameterization.is_flow(), options.extra_args.c_str());
+ };
+ return run_sample(input, sigmas, execution, solver, options, events);
+ }
+
+ Tensor<float> run_sample(const Tensor<float>& input, const std::vector<float>& sigmas, PredictionExecution& prediction_run, const SamplingSolver& solver, const SamplingOptions& options, RunEvents& events) {
+ validate_mask(input, options);
+ const int64_t sampling_start = ggml_time_ms();
+ LOG_INFO("sampling started: method=%s, steps=%zu", options.method.c_str(), sigmas.size() - 1);
+ auto& model = prediction_run.dit();
+ const auto& parameterization = prediction_run.parameterization();
+ sd::ParallelScope tensor_scope(model.tensor_executor.get());
+ const Tensor<float> empty;
+ const auto& mask = options.denoise_mask ? *options.denoise_mask : empty;
+ const auto& base = options.denoise_mask ? *options.base : empty;
+ const bool terminal_zero = sigmas.back() == 0.f;
+ SamplingObservation observation({model, events, static_cast<int>(sigmas.size() - 1), terminal_zero, parameterization}, options.observer);
+ const auto evaluate = [&](const Tensor<float>& x, float sigma, int index) {
+ events.check_cancelled();
+ models::DiffusionPredictionStep step{base.empty() ? x : base, mask};
+ step.index = std::abs(index);
+ step.schedule_size = sigmas.size();
+ step.solver_index = index;
+ step.completes_step = index > 0 || (terminal_zero && index < 0 && step.index == static_cast<int>(sigmas.size() - 1));
+ step.mask_input = options.mask_input;
+ const auto target = static_cast<size_t>(step.index);
+ if (target < sigmas.size())
+ step.target_sigma = sigmas[target];
+ if (index + 1 >= 0 && static_cast<size_t>(index + 1) < sigmas.size())
+ step.lookahead_sigma = sigmas[static_cast<size_t>(index + 1)];
+ if (solver.prepare_step)
+ solver.prepare_step(step);
+
+ const auto observed_step = observation.step(sigma, step);
+ observation.before_prediction(observed_step, x);
+ auto prediction = prediction_run.evaluate(x, sigma, step);
+ step.blend_mask(prediction.pred);
+ observation.after_prediction(observed_step, x, prediction);
+ return prediction;
+ };
+ auto output = solver.execute(input, evaluate);
+ if (output.empty())
+ throw std::runtime_error("sampling failed");
+ prediction_run.log_cache_summary();
+ if (options.finish.value_or(false) && !terminal_zero)
+ output = parameterization.finish(sigmas.back(), output);
+ observation.end(output);
+ LOG_INFO("sampling completed, taking %.2fs", (ggml_time_ms() - sampling_start) / 1000.0);
+ return output;
+ }
+
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/sampling.h b/src/dgl/ops/sampling.h
new file mode 100644
index 0000000..9cf837d
--- /dev/null
+++ b/src/dgl/ops/sampling.h
@@ -0,0 +1,52 @@
+#ifndef __SD_DGL_OPS_SAMPLING_H__
+#define __SD_DGL_OPS_SAMPLING_H__
+
+#include "dgl/models/fwd.h"
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "config.h"
+#include "core/tensor.hpp"
+#include "runtime/guidance.h"
+
+class RNG;
+
+namespace sd::dgl {
+ class RunEvents;
+
+ namespace ops {
+ struct Predictor;
+ class PredictionExecution;
+ class SamplingObserver;
+
+ struct SamplingOptions {
+ std::string method;
+ float eta = 1.f;
+ const sd::Tensor<float>* base = nullptr;
+ const sd::Tensor<float>* denoise_mask = nullptr;
+ bool mask_input = false;
+ std::string extra_args;
+ std::optional<bool> finish;
+ std::optional<CacheConfig> cache;
+ std::shared_ptr<SamplingObserver> observer;
+ bool batched_cfg = true;
+ };
+
+ using SamplePrediction = std::function<sd::guidance::GuiderOutput(const Tensor<float>&, float, int)>;
+
+ struct SamplingSolver {
+ std::function<Tensor<float>(const Tensor<float>&, const SamplePrediction&)> execute;
+ std::function<void(models::DiffusionPredictionStep&)> prepare_step;
+ };
+
+ const std::vector<float>& sampling_schedule(const sd::Tensor<float>& tensor, bool allow_repeated = false);
+ sd::Tensor<float> sample(const sd::Tensor<float>& input, const std::vector<float>& sigmas, const Predictor& predictor, const std::shared_ptr<RNG>& rng, const SamplingOptions& options, RunEvents& events);
+ Tensor<float> run_sample(const Tensor<float>& input, const std::vector<float>& sigmas, PredictionExecution& prediction, const SamplingSolver& solver, const SamplingOptions& options, RunEvents& events);
+ }
+}
+
+#endif // __SD_DGL_OPS_SAMPLING_H__
diff --git a/src/dgl/ops/sampling_observer.cpp b/src/dgl/ops/sampling_observer.cpp
new file mode 100644
index 0000000..e1bdf7e
--- /dev/null
+++ b/src/dgl/ops/sampling_observer.cpp
@@ -0,0 +1,47 @@
+#include "dgl/ops/sampling_observer.h"
+
+#include "dgl/models/diffusion_prediction.h"
+#include "dgl/models/fwd.h"
+#include "dgl/run_events.h"
+
+namespace sd::dgl::ops {
+ SamplingObservation::SamplingObservation(SamplingRun run, const std::shared_ptr<SamplingObserver>& observer)
+ : run_(run) {
+ run_.events.begin_sample(run_.total_steps);
+ if (observer)
+ session_ = observer->start(run_);
+ run_.events.check_cancelled();
+ }
+
+ SamplingStep SamplingObservation::step(float sigma, const models::DiffusionPredictionStep& prediction) {
+ return {prediction.index, ++evaluations_, prediction.completes_step && prediction.index > completed_, sigma, prediction};
+ }
+
+ void SamplingObservation::before_prediction(const SamplingStep& step, const Tensor<float>& x) {
+ run_.events.check_cancelled();
+ if (session_)
+ session_->before_prediction(step, x);
+ run_.events.check_cancelled();
+ }
+
+ void SamplingObservation::after_prediction(const SamplingStep& step, const Tensor<float>& x, const sd::guidance::GuiderOutput& prediction) {
+ run_.events.check_cancelled();
+ if (session_)
+ session_->after_prediction(step, x, prediction);
+ run_.events.check_cancelled();
+ if (step.completes_step) {
+ completed_ = step.index;
+ if (session_)
+ session_->step_completed(step);
+ run_.events.sample_step(completed_, run_.total_steps);
+ }
+ run_.events.check_cancelled();
+ }
+
+ void SamplingObservation::end(const Tensor<float>& output) {
+ run_.events.check_cancelled();
+ if (session_)
+ session_->end(output);
+ run_.events.check_cancelled();
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/sampling_observer.h b/src/dgl/ops/sampling_observer.h
new file mode 100644
index 0000000..6f48c5a
--- /dev/null
+++ b/src/dgl/ops/sampling_observer.h
@@ -0,0 +1,63 @@
+#ifndef __SD_DGL_OPS_SAMPLING_OBSERVER_H__
+#define __SD_DGL_OPS_SAMPLING_OBSERVER_H__
+
+#include "dgl/models/fwd.h"
+
+#include <memory>
+
+#include "core/tensor.hpp"
+#include "runtime/guidance.h"
+
+namespace sd::dgl {
+ class RunEvents;
+
+ namespace ops {
+
+ struct SamplingRun {
+ models::DiT& dit;
+ RunEvents& events;
+ int total_steps;
+ bool terminal_zero;
+ const models::DenoisingParameterization& parameterization;
+ };
+
+ struct SamplingStep {
+ int index;
+ int evaluation;
+ bool completes_step;
+ float sigma;
+ const models::DiffusionPredictionStep& prediction;
+ };
+
+ class SamplingObserverSession {
+ public:
+ virtual ~SamplingObserverSession() = default;
+ virtual void before_prediction(const SamplingStep&, const Tensor<float>&) {}
+ virtual void after_prediction(const SamplingStep&, const Tensor<float>&, const sd::guidance::GuiderOutput&) {}
+ virtual void step_completed(const SamplingStep&) {}
+ virtual void end(const Tensor<float>&) {}
+ };
+
+ class SamplingObserver {
+ public:
+ virtual ~SamplingObserver() = default;
+ virtual std::unique_ptr<SamplingObserverSession> start(const SamplingRun& run) const = 0;
+ };
+
+ class SamplingObservation {
+ SamplingRun run_;
+ std::unique_ptr<SamplingObserverSession> session_;
+ int evaluations_ = 0;
+ int completed_ = 0;
+
+ public:
+ SamplingObservation(SamplingRun run, const std::shared_ptr<SamplingObserver>& observer);
+ SamplingStep step(float sigma, const models::DiffusionPredictionStep& prediction);
+ void before_prediction(const SamplingStep& step, const Tensor<float>& x);
+ void after_prediction(const SamplingStep& step, const Tensor<float>& x, const sd::guidance::GuiderOutput& prediction);
+ void end(const Tensor<float>& output);
+ };
+ }
+} // namespace sd::dgl
+
+#endif // __SD_DGL_OPS_SAMPLING_OBSERVER_H__
diff --git a/src/dgl/ops/seedvr2.cpp b/src/dgl/ops/seedvr2.cpp
new file mode 100644
index 0000000..227c049
--- /dev/null
+++ b/src/dgl/ops/seedvr2.cpp
@@ -0,0 +1,17 @@
+#include "dgl/ops/seedvr2.h"
+
+#include "dgl/models/dit.h"
+#include "dgl/models/fwd.h"
+#include "model/diffusion/seedvr2.hpp"
+
+namespace sd::dgl::ops {
+ Tensor<float> seedvr2_positive_embedding(models::DiT& dit) {
+ if (dit.version() != VERSION_SEEDVR2)
+ throw std::runtime_error("expected a SeedVR2 DiT");
+ const auto& model = static_cast<const SeedVR2::SeedVR2Runner&>(*dit.diffusion_model);
+ std::vector<float> values;
+ if (!dit.model_weights->load_float_tensor(model.positive_embedding_weight(), values))
+ throw std::runtime_error("cannot read SeedVR2 positive embedding");
+ return Tensor<float>({model.config.text_dim, model.config.text_len}, std::move(values));
+ }
+}
diff --git a/src/dgl/ops/seedvr2.h b/src/dgl/ops/seedvr2.h
new file mode 100644
index 0000000..07aca53
--- /dev/null
+++ b/src/dgl/ops/seedvr2.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_OPS_SEEDVR2_H__
+#define __SD_DGL_OPS_SEEDVR2_H__
+
+#include "dgl/models/fwd.h"
+
+#include "core/tensor.hpp"
+
+namespace sd::dgl::ops {
+ Tensor<float> seedvr2_positive_embedding(models::DiT& dit);
+}
+
+#endif // __SD_DGL_OPS_SEEDVR2_H__
diff --git a/src/dgl/ops/sefi_sampling.cpp b/src/dgl/ops/sefi_sampling.cpp
new file mode 100644
index 0000000..2ee5b93
--- /dev/null
+++ b/src/dgl/ops/sefi_sampling.cpp
@@ -0,0 +1,35 @@
+#include "dgl/ops/sefi_sampling.h"
+
+#include "dgl/models/diffusion_prediction.h"
+#include "dgl/models/dit.h"
+#include "dgl/models/fwd.h"
+#include "dgl/ops/predictor.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::dgl::ops {
+ Tensor<float> sefi_sample(const Tensor<float>& input, const Tensor<float>& sigmas, const Tensor<float>& semantic_sigmas, const Predictor& predictor, const SamplingOptions& options, RunEvents& events) {
+ if (options.method != "euler")
+ throw std::runtime_error("sefi::sample requires the Euler method");
+ const auto& texture = sampling_schedule(sigmas, true);
+ const auto& semantic = sampling_schedule(semantic_sigmas, true);
+ if (texture.size() != semantic.size())
+ throw std::runtime_error("SeFi sigma lists differ in length");
+ if (input.dim() != 4 || input.shape()[2] != SefiFlowDenoiser::kTotalChannels)
+ throw std::runtime_error("sefi::sample requires a rank-four latent with 144 channels on axis 2");
+ PredictionExecution execution(predictor, texture, {options.cache ? &*options.cache : nullptr, false, true, options.batched_cfg});
+ if (execution.parameterization().prediction() != SEFI_FLOW_PRED)
+ throw std::runtime_error("sefi::sample requires a dual-time SeFi predictor");
+
+ SefiFlowDenoiser schedule;
+ schedule.tex_sigmas = texture;
+ schedule.sem_sigmas = semantic;
+ SamplingSolver solver;
+ solver.prepare_step = [&](models::DiffusionPredictionStep& step) {
+ step.semantic_sigma = semantic.at(static_cast<size_t>(step.index - 1));
+ };
+ solver.execute = [&](const Tensor<float>& start, const SamplePrediction& evaluate) {
+ return sample_sefi_euler(&schedule, evaluate, start);
+ };
+ return run_sample(input, texture, execution, solver, options, events);
+ }
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/sefi_sampling.h b/src/dgl/ops/sefi_sampling.h
new file mode 100644
index 0000000..ee62ebb
--- /dev/null
+++ b/src/dgl/ops/sefi_sampling.h
@@ -0,0 +1,12 @@
+#ifndef __SD_DGL_OPS_SEFI_SAMPLING_H__
+#define __SD_DGL_OPS_SEFI_SAMPLING_H__
+
+#include "dgl/models/fwd.h"
+
+#include "dgl/ops/sampling.h"
+
+namespace sd::dgl::ops {
+ Tensor<float> sefi_sample(const Tensor<float>& input, const Tensor<float>& sigmas, const Tensor<float>& semantic_sigmas, const Predictor& predictor, const SamplingOptions& options, RunEvents& events);
+}
+
+#endif // __SD_DGL_OPS_SEFI_SAMPLING_H__
diff --git a/src/dgl/ops/spectrum_cache.cpp b/src/dgl/ops/spectrum_cache.cpp
new file mode 100644
index 0000000..219a51a
--- /dev/null
+++ b/src/dgl/ops/spectrum_cache.cpp
@@ -0,0 +1,47 @@
+#include "dgl/ops/spectrum_cache.h"
+
+#include "config.h"
+#include "runtime/sample-cache.h"
+
+namespace sd::dgl::ops {
+ SpectrumCache::SpectrumCache(const CacheConfig* config, size_t steps) {
+ enabled_ = config && config->mode == SD_CACHE_SPECTRUM;
+ if (enabled_) {
+ for (auto& phase : phases_)
+ phase.init(sd_sample::spectrum_config(*config), steps);
+ LOG_INFO("Spectrum enabled - warmup: %d logical steps, stop: %.0f%%; separate predictor/corrector histories",
+ config->spectrum_warmup_steps, config->spectrum_stop_percent * 100.f);
+ }
+ }
+
+ Tensor<float> SpectrumCache::predict(const PredictorCacheKey& key, const std::vector<int64_t>& shape, const models::DiffusionPredictionStep& step) {
+ if (!enabled_)
+ return {};
+ ++evaluations_;
+ if (!(key_ == key) || shape_ != shape) {
+ for (auto& phase : phases_)
+ phase.reset_history();
+ key_ = key;
+ shape_ = shape;
+ }
+ // A Heun predictor and corrector observe different inputs at the same logical step.
+ auto& phase = phases_[step.solver_index < 0 ? 0 : 1];
+ const int index = step.index - 1;
+ if (!phase.should_predict(index))
+ return {};
+ Tensor<float> output(shape);
+ phase.predict(&output, index);
+ return output;
+ }
+
+ void SpectrumCache::record(const Tensor<float>& result, const models::DiffusionPredictionStep& step) {
+ if (enabled_)
+ phases_[step.solver_index < 0 ? 0 : 1].update(result, step.index - 1);
+ }
+
+ void SpectrumCache::log_summary() const {
+ if (enabled_)
+ LOG_INFO("Spectrum reused %d/%zu predictor evaluations",
+ phases_[0].total_steps_skipped + phases_[1].total_steps_skipped, evaluations_);
+ }
+}
diff --git a/src/dgl/ops/spectrum_cache.h b/src/dgl/ops/spectrum_cache.h
new file mode 100644
index 0000000..b350eb2
--- /dev/null
+++ b/src/dgl/ops/spectrum_cache.h
@@ -0,0 +1,41 @@
+#ifndef __SD_DGL_OPS_SPECTRUM_CACHE_H__
+#define __SD_DGL_OPS_SPECTRUM_CACHE_H__
+
+#include <array>
+#include "dgl/models/diffusion_prediction.h"
+#include "runtime/spectrum.hpp"
+
+namespace sd {
+ struct CacheConfig;
+}
+namespace sd::dgl::ops {
+ struct Predictor;
+ struct ModelPredictor;
+
+ struct PredictorCacheKey {
+ std::vector<const Predictor*> switches;
+ std::vector<std::pair<const Predictor*, float>> guidance;
+ std::vector<std::pair<const Predictor*, float>> slg;
+ std::vector<const ModelPredictor*> models;
+ std::vector<bool> conditioning;
+
+ bool operator==(const PredictorCacheKey& other) const {
+ return switches == other.switches && guidance == other.guidance && slg == other.slg && models == other.models && conditioning == other.conditioning;
+ }
+ };
+
+ class SpectrumCache {
+ bool enabled_ = false;
+ std::array<SpectrumState, 2> phases_;
+ PredictorCacheKey key_;
+ std::vector<int64_t> shape_;
+ size_t evaluations_ = 0;
+
+ public:
+ SpectrumCache(const CacheConfig* config, size_t steps);
+ Tensor<float> predict(const PredictorCacheKey& key, const std::vector<int64_t>& shape, const models::DiffusionPredictionStep& step);
+ void record(const Tensor<float>& result, const models::DiffusionPredictionStep& step);
+ void log_summary() const;
+ };
+}
+#endif // __SD_DGL_OPS_SPECTRUM_CACHE_H__
diff --git a/src/dgl/ops/tensor.cpp b/src/dgl/ops/tensor.cpp
new file mode 100644
index 0000000..ff18e8f
--- /dev/null
+++ b/src/dgl/ops/tensor.cpp
@@ -0,0 +1,740 @@
+#include "dgl/ops/tensor.h"
+
+#include <map>
+
+#include "core/rng.hpp"
+
+namespace sd::dgl::ops {
+ size_t element_size(sd_tensor_type_t dtype) {
+ return with_dtype(dtype, [](auto value) { return sizeof(value); });
+ }
+
+ size_t element_count(const std::vector<int64_t>& shape, sd_tensor_type_t dtype) {
+ if (shape.size() > 8 || std::any_of(shape.begin(), shape.end(), [](int64_t dim) { return dim < 0; }))
+ throw std::runtime_error("invalid tensor shape");
+ if (std::find(shape.begin(), shape.end(), 0) != shape.end())
+ return 0;
+ const auto limit = tensor_byte_limit / element_size(dtype);
+ size_t count = 1;
+ for (auto dim : shape) {
+ if (uint64_t(dim) > limit / count)
+ throw std::runtime_error("tensor exceeds 1 GiB");
+ count *= size_t(dim);
+ }
+ return count;
+ }
+
+ size_t tensor_axis(const sd_tensor_t& tensor, int64_t axis) {
+ if (axis < 0 || size_t(axis) >= tensor.shape().size())
+ throw std::runtime_error("axis falls outside the tensor rank");
+ return size_t(axis);
+ }
+
+ static void copy_tensor_box(const sd_tensor_t& input, sd_tensor_t& output, const std::vector<int64_t>& shape, const std::vector<int64_t>& source_offset, const std::vector<int64_t>& target_offset, const std::vector<int64_t>& out_stride) {
+ if (std::find(shape.begin(), shape.end(), 0) != shape.end())
+ return;
+ const auto in_stride = sd::tensor_compute_strides(input.shape());
+ const auto bytes = element_size(input.type);
+ const auto row_bytes = size_t(shape[0]) * bytes;
+ int64_t source_start = 0, target_start = 0, rows = 1;
+ for (size_t axis = 0; axis < shape.size(); ++axis) {
+ source_start += source_offset[axis] * in_stride[axis];
+ target_start += target_offset[axis] * out_stride[axis];
+ if (axis)
+ rows *= shape[axis];
+ }
+ const auto* from = static_cast<const uint8_t*>(input.data()) + source_start * bytes;
+ auto* to = static_cast<uint8_t*>(output.mutable_data()) + target_start * bytes;
+ parallel_for(0, rows, std::max<int64_t>(1, 16384 / row_bytes), [&](int64_t begin, int64_t end) {
+ for (int64_t row = begin; row < end; ++row) {
+ int64_t index = row, source = 0, target = 0;
+ for (size_t axis = 1; axis < shape.size(); ++axis) {
+ const auto coord = index % shape[axis];
+ source += coord * in_stride[axis];
+ target += coord * out_stride[axis];
+ index /= shape[axis];
+ }
+ std::memcpy(to + target * bytes, from + source * bytes, row_bytes);
+ }
+ });
+ }
+
+ void copy_tensor_box(const sd_tensor_t& input, sd_tensor_t& output, const std::vector<int64_t>& shape, const std::vector<int64_t>& source_offset, const std::vector<int64_t>& target_offset) {
+ if (std::find(shape.begin(), shape.end(), 0) != shape.end())
+ return;
+ copy_tensor_box(input, output, shape, source_offset, target_offset, sd::tensor_compute_strides(output.shape()));
+ }
+
+ TensorIndexResult index_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<TensorIndex>& indices) {
+ const auto& shape = input->shape();
+ if (indices.empty() || indices.size() > shape.size())
+ throw std::runtime_error("tensor indexing requires one or more axes within the tensor rank");
+ std::vector<int64_t> region = shape, offset(shape.size(), 0), output_shape;
+ for (size_t axis = 0; axis < shape.size(); ++axis) {
+ if (axis >= indices.size()) {
+ output_shape.push_back(shape[axis]);
+ continue;
+ }
+ if (const auto* index = std::get_if<int64_t>(&indices[axis])) {
+ const auto position = *index < 0 ? shape[axis] + *index : *index;
+ if (position < 0 || position >= shape[axis])
+ throw std::runtime_error("tensor index out of range on axis " + std::to_string(axis));
+ offset[axis] = position;
+ region[axis] = 1;
+ } else {
+ const auto& slice = std::get<TensorSlice>(indices[axis]);
+ const auto normalize = [&](int64_t value) {
+ return value < 0 ? std::max<int64_t>(0, shape[axis] + value) : std::min(value, shape[axis]);
+ };
+ const auto start = slice.start ? normalize(*slice.start) : 0;
+ const auto end = slice.end ? normalize(*slice.end) : shape[axis];
+ offset[axis] = start;
+ region[axis] = std::max<int64_t>(0, end - start);
+ output_shape.push_back(region[axis]);
+ }
+ }
+ if (output_shape.empty()) {
+ const auto strides = sd::tensor_compute_strides(shape);
+ int64_t position = 0;
+ for (size_t axis = 0; axis < shape.size(); ++axis)
+ position += offset[axis] * strides[axis];
+ return with_dtype(input->type, [&](auto tag) -> TensorIndexResult {
+ using T = decltype(tag);
+ const auto value = static_cast<const T*>(input->data())[position];
+ if constexpr (std::is_integral_v<T>)
+ return int64_t(value);
+ else
+ return double(value);
+ });
+ }
+ if (output_shape == shape && region == shape)
+ return input;
+ auto output = make_tensor(input->type, std::move(output_shape));
+ if (!output->bytes)
+ return output;
+ // Dropping singleton integer axes preserves the region's contiguous byte order.
+ copy_tensor_box(*input, *output, region, offset, std::vector<int64_t>(shape.size(), 0), sd::tensor_compute_strides(region));
+ return output;
+ }
+
+ std::shared_ptr<sd_tensor_t> crop_tensor(const sd_tensor_t& input, const std::vector<int64_t>& axes, const std::vector<int64_t>& start, const std::vector<int64_t>& size) {
+ if (axes.empty() || axes.size() != start.size() || axes.size() != size.size())
+ throw std::runtime_error("crop requires matching axes, start and size lists");
+ auto shape = input.shape();
+ std::vector<int64_t> offset(shape.size(), 0);
+ std::vector<bool> used(shape.size(), false);
+ for (size_t i = 0; i < axes.size(); ++i) {
+ const auto axis = tensor_axis(input, axes[i]);
+ if (used[axis] || start[i] < 0 || size[i] < 0 || start[i] > shape[axis] || size[i] > shape[axis] - start[i])
+ throw std::runtime_error("crop axes must be distinct and the crop must fit inside the tensor");
+ used[axis] = true;
+ shape[axis] = size[i];
+ offset[axis] = start[i];
+ }
+ auto result = make_tensor(input.type, shape);
+ copy_tensor_box(input, *result, shape, offset, std::vector<int64_t>(shape.size(), 0));
+ return result;
+ }
+
+ uint8_t rounded_byte(double value) {
+ if (!(value > 0.0))
+ return 0;
+ if (value >= 255.0)
+ return 255;
+ return uint8_t(std::lround(value));
+ }
+
+ template <class T, class F>
+ static sd::Tensor<T> apply_spatial(const sd::Tensor<T>& tensor, size_t x, size_t y, F filter) {
+ if (x == 0 && y == 1)
+ return filter(tensor);
+ std::vector<size_t> order{x, y}, inverse(size_t(tensor.dim()));
+ for (size_t axis = 0; axis < inverse.size(); ++axis)
+ if (axis != x && axis != y)
+ order.push_back(axis);
+ for (size_t i = 0; i < order.size(); ++i)
+ inverse[order[i]] = i;
+ return filter(tensor.permute(order)).permute(inverse);
+ }
+
+ static std::pair<size_t, size_t> spatial_axes(const sd_tensor_t& input, const std::vector<int64_t>& axes) {
+ if (axes.size() != 2 || axes[0] == axes[1] || !input.bytes)
+ throw std::runtime_error("expected a nonempty tensor and two distinct axes");
+ return {tensor_axis(input, axes[0]), tensor_axis(input, axes[1])};
+ }
+
+ std::shared_ptr<sd_tensor_t> pool_tensor(const sd_tensor_t& source, const std::vector<int64_t>& axes, const std::vector<int64_t>& kernel, const std::vector<int64_t>& stride, const std::vector<int64_t>& before, const std::vector<int64_t>& after, std::optional<TensorNumber> pad_value, bool maximum) {
+ const auto* input = &source;
+ const auto spatial = spatial_axes(source, axes);
+ const auto x = spatial.first, y = spatial.second;
+ if (kernel.size() != 2 || stride.size() != 2 || before.size() != 2 || after.size() != 2)
+ throw std::runtime_error("pooling kernel, stride and padding need two entries");
+ auto output_size = [&](size_t i, int64_t length) {
+ if (kernel[i] <= 0 || stride[i] <= 0 || before[i] < 0 || after[i] < 0 ||
+ before[i] > INT64_MAX - length || after[i] > INT64_MAX - length - before[i])
+ throw std::runtime_error("invalid pooling kernel, stride or padding");
+ const auto padded = length + before[i] + after[i];
+ if (kernel[i] > padded)
+ throw std::runtime_error("pooling kernel exceeds padded size");
+ return (padded - kernel[i]) / stride[i] + 1;
+ };
+ const auto ow = output_size(0, input->shape()[x]), oh = output_size(1, input->shape()[y]);
+ auto shape = input->shape();
+ shape[x] = ow;
+ shape[y] = oh;
+ element_count(shape, input->type);
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ const auto pad = bool(pad_value) ? std::optional<T>{std::visit([](auto value) { return cast_number<T>(value); }, *pad_value)} : std::nullopt;
+ auto filtered = apply_spatial(input->as<T>(), x, y, [&](const sd::Tensor<T>& tensor) {
+ auto dims = tensor.shape();
+ const auto iw = dims[0], ih = dims[1];
+ dims[0] = ow;
+ dims[1] = oh;
+ sd::Tensor<T> output(std::move(dims));
+ const auto combine = [&](T a, T b) { return maximum ? std::max(a, b) : std::min(a, b); };
+ const auto neutral = maximum ? std::numeric_limits<T>::lowest() : std::numeric_limits<T>::max();
+ parallel_for(0, output.numel() / ow, std::max<int64_t>(1, 16384 / ow), [&](int64_t begin, int64_t end) {
+ for (int64_t row = begin; row < end; ++row) {
+ const auto* plane = tensor.data() + row / oh * iw * ih;
+ const auto y0 = (row % oh) * stride[1] - before[1], y1 = y0 + kernel[1];
+ for (int64_t col = 0; col < ow; ++col) {
+ const auto x0 = col * stride[0] - before[0], x1 = x0 + kernel[0];
+ const auto l = std::max<int64_t>(0, x0), r = std::min(iw, x1);
+ const auto t = std::max<int64_t>(0, y0), b = std::min(ih, y1);
+ const bool padded = pad && (x0 < 0 || x1 > iw || y0 < 0 || y1 > ih);
+ T value = padded ? *pad : neutral;
+ for (int64_t sy = t; sy < b; ++sy)
+ for (int64_t sx = l; sx < r; ++sx)
+ value = combine(value, plane[sy * iw + sx]);
+ output.data()[row * ow + col] = padded || (l < r && t < b) ? value : T(0);
+ }
+ }
+ });
+ return output;
+ });
+ return make_tensor(std::move(filtered));
+ });
+ }
+
+ template <class T, class Compute>
+ static sd::Tensor<T> gaussian_blur(const sd::Tensor<T>& input, int64_t radius, Compute sigma) {
+ std::vector<Compute> weights(size_t(2 * radius + 1));
+ Compute total = 0;
+ for (int64_t i = -radius; i <= radius; ++i) {
+ const auto distance = Compute(i);
+ const auto weight = i == 0 ? Compute(1) : std::exp(-(distance * distance) / (Compute(2) * sigma * sigma));
+ weights[size_t(i + radius)] = weight;
+ total += weight;
+ }
+ for (auto& weight : weights)
+ weight /= total;
+ const auto width = input.shape()[0], height = input.shape()[1];
+ // Byte kernels round their horizontal pass to float before the vertical pass.
+ using Intermediate = std::conditional_t<std::is_same_v<T, uint8_t>, float, Compute>;
+ std::vector<Intermediate> horizontal(size_t(input.numel()));
+ sd::Tensor<T> output(input.shape());
+ auto filter = [&](const auto* source, auto* target, bool along_x) {
+ using Output = std::remove_pointer_t<decltype(target)>;
+ parallel_for(0, input.numel() / width, std::max<int64_t>(1, 16384 / width), [&](int64_t begin, int64_t end) {
+ for (int64_t row = begin; row < end; ++row) {
+ const auto y = row % height;
+ const auto plane = row / height * width * height;
+ for (int64_t x = 0; x < width; ++x) {
+ Compute sum = 0;
+ for (int64_t k = -radius; k <= radius; ++k) {
+ const auto sx = along_x ? std::clamp(x + k, int64_t(0), width - 1) : x;
+ const auto sy = along_x ? y : std::clamp(y + k, int64_t(0), height - 1);
+ sum += Compute(source[plane + sy * width + sx]) * weights[size_t(k + radius)];
+ }
+ if constexpr (std::is_same_v<Output, uint8_t> && std::is_same_v<Compute, float>)
+ target[row * width + x] = uint8_t(std::clamp(sum, Compute(0), Compute(255)) + 0.5f);
+ else if constexpr (std::is_same_v<Output, uint8_t>)
+ target[row * width + x] = rounded_byte(sum);
+ else
+ target[row * width + x] = static_cast<Output>(sum);
+ }
+ }
+ });
+ };
+ filter(input.data(), horizontal.data(), true);
+ filter(horizontal.data(), output.data(), false);
+ return output;
+ }
+
+ std::shared_ptr<sd_tensor_t> blur_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes, int64_t radius, double sigma, const std::string& precision) {
+ const auto spatial = spatial_axes(*input, axes);
+ const auto x = spatial.first, y = spatial.second;
+ if (input->type != SD_TENSOR_U8 && input->type != SD_TENSOR_F32 && input->type != SD_TENSOR_F64)
+ throw std::runtime_error("gaussian blur requires u8, f32 or f64");
+ if (!radius)
+ return input;
+ if (sigma <= 0)
+ throw std::runtime_error("nonzero Gaussian radius requires positive sigma");
+ auto blur = [&](auto tag) {
+ using T = decltype(tag);
+ const bool use_double = precision == "f64" || (precision == "auto" && input->type == SD_TENSOR_F64);
+ auto result = apply_spatial(input->as<T>(), x, y, [&](const sd::Tensor<T>& tensor) {
+ return use_double ? gaussian_blur(tensor, radius, sigma) : gaussian_blur(tensor, radius, cast_number<float>(sigma));
+ });
+ return make_tensor(std::move(result));
+ };
+ if (input->type == SD_TENSOR_U8)
+ return blur(uint8_t{});
+ return with_float_dtype(input->type, blur);
+ }
+
+ static sd::Tensor<uint8_t> interpolate_bytes(const sd::Tensor<uint8_t>& input, std::vector<int64_t> shape, sd::ops::InterpolateMode mode, bool antialias) {
+ using Mode = sd::ops::InterpolateMode;
+ const bool nearest_exact = mode == Mode::NearestExact;
+ if (mode == Mode::Nearest || (nearest_exact && antialias))
+ return sd::ops::interpolate(input, std::move(shape), mode, false, antialias);
+ if ((!nearest_exact && mode != Mode::Bilinear) || antialias) {
+ const sd::Tensor<double> values(input.shape(), std::vector<double>(input.values().begin(), input.values().end()));
+ auto resized = sd::ops::interpolate(values, std::move(shape), mode, false, antialias);
+ sd::Tensor<uint8_t> output(resized.shape());
+ std::transform(resized.values().begin(), resized.values().end(), output.values().begin(), rounded_byte);
+ return output;
+ }
+
+ // Keep half-pixel sampling and rounding for byte images.
+ const auto iw = input.shape()[0], ih = input.shape()[1];
+ const auto ow = shape[0], oh = shape[1];
+ sd::Tensor<uint8_t> output(std::move(shape));
+ parallel_for(0, output.numel() / ow, std::max<int64_t>(1, 16384 / ow), [&](int64_t begin, int64_t end) {
+ for (int64_t row = begin; row < end; ++row) {
+ const auto* plane = input.data() + row / oh * iw * ih;
+ if (nearest_exact) {
+ const auto y = (2 * (row % oh) + 1) * ih / (2 * oh);
+ for (int64_t x = 0; x < ow; ++x)
+ output.data()[row * ow + x] = plane[y * iw + (2 * x + 1) * iw / (2 * ow)];
+ continue;
+ }
+ const auto sy = std::max(0.0, (row % oh + 0.5) * ih / oh - 0.5);
+ const auto y0 = std::min(ih - 1, int64_t(sy));
+ const auto y1 = std::min(ih - 1, y0 + 1);
+ const double fy = sy - y0;
+ for (int64_t x = 0; x < ow; ++x) {
+ const auto sx = std::max(0.0, (x + 0.5) * iw / ow - 0.5);
+ const auto x0 = std::min(iw - 1, int64_t(sx));
+ const auto x1 = std::min(iw - 1, x0 + 1);
+ const double fx = sx - x0;
+ const double value = (1 - fy) * ((1 - fx) * plane[y0 * iw + x0] + fx * plane[y0 * iw + x1]) +
+ fy * ((1 - fx) * plane[y1 * iw + x0] + fx * plane[y1 * iw + x1]);
+ output.data()[row * ow + x] = rounded_byte(value);
+ }
+ }
+ });
+ return output;
+ }
+
+ std::shared_ptr<sd_tensor_t> interpolate_tensor(const sd_tensor_t& input, const std::vector<int64_t>& size, const std::vector<int64_t>& axes, const std::string& name, bool antialias) {
+ if (size.size() != 2 || size[0] < 1 || size[1] < 1 || axes.size() != 2 || axes[0] == axes[1] || !input.bytes)
+ throw std::runtime_error("interpolation requires a nonempty tensor, two positive sizes and two distinct axes");
+ const auto x = tensor_axis(input, axes[0]), y = tensor_axis(input, axes[1]);
+ auto shape = input.shape();
+ shape[x] = size[0];
+ shape[y] = size[1];
+ element_count(shape, input.type);
+ using Mode = sd::ops::InterpolateMode;
+ static const std::map<std::string, Mode> modes{
+ {"nearest", Mode::Nearest},
+ {"nearest_exact", Mode::NearestExact},
+ {"nearest_max", Mode::NearestMax},
+ {"bilinear", Mode::Bilinear},
+ {"bicubic", Mode::Bicubic},
+ {"lanczos", Mode::Lanczos},
+ };
+ const auto mode = modes.at(name);
+ auto resize = [&](auto tag) {
+ using T = decltype(tag);
+ auto resample = [&](const sd::Tensor<T>& source, std::vector<int64_t> target) {
+ if constexpr (std::is_same_v<T, uint8_t>)
+ return interpolate_bytes(source, std::move(target), mode, antialias);
+ else
+ return sd::ops::interpolate(source, std::move(target), mode, false, antialias);
+ };
+ auto result = apply_spatial(input.as<T>(), x, y, [&](const sd::Tensor<T>& source) {
+ auto target = source.shape();
+ target[0] = size[0];
+ target[1] = size[1];
+ return resample(source, std::move(target));
+ });
+ return make_tensor(std::move(result));
+ };
+ if (input.type == SD_TENSOR_U8)
+ return resize(uint8_t{});
+ return with_float_dtype(input.type, resize);
+ }
+ std::shared_ptr<sd_tensor_t> filled_tensor(std::vector<int64_t> shape, sd_tensor_type_t dtype, const TensorNumber& value) {
+ const auto count = element_count(shape, dtype);
+ return with_dtype(dtype, [&](auto tag) {
+ using T = decltype(tag);
+ std::vector<T> data(count, scalar_number<T>(value));
+ return tensor_result(std::move(shape), std::move(data));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> unsqueeze_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis) {
+ auto shape = input->shape();
+ const auto rank = int64_t(shape.size());
+ if (axis < 0)
+ axis += rank + 1;
+ if (axis < 0 || axis > rank)
+ throw std::runtime_error("unsqueeze axis must be in [-rank-1, rank]");
+ if (rank == 8)
+ throw std::runtime_error("unsqueeze would exceed the maximum tensor rank of 8");
+ shape.insert(shape.begin() + axis, 1);
+ return make_tensor(input->type, std::move(shape), input->data(), input->bytes);
+ }
+
+ std::shared_ptr<sd_tensor_t> pad_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& pad, const TensorNumber& value) {
+ if (pad.size() % 2 || pad.size() / 2 > input->shape().size())
+ throw std::runtime_error("pad requires an even number of values and at most one pair per tensor axis");
+ auto shape = input->shape();
+ std::vector<int64_t> offset(shape.size(), 0);
+ bool changed = false;
+ for (size_t axis = 0; axis < pad.size() / 2; ++axis) {
+ const auto before = pad[2 * axis], after = pad[2 * axis + 1];
+ if (before < 0 || after < 0)
+ throw std::runtime_error("padding lengths must be nonnegative");
+ if (before > INT64_MAX - shape[axis] || after > INT64_MAX - shape[axis] - before)
+ throw std::runtime_error("padded tensor shape exceeds int64 range");
+ shape[axis] += before + after;
+ offset[axis] = before;
+ changed = changed || before != 0 || after != 0;
+ }
+ if (!changed) {
+ with_dtype(input->type, [&](auto tag) { scalar_number<decltype(tag)>(value); });
+ return input;
+ }
+ auto result = filled_tensor(std::move(shape), input->type, value);
+ copy_tensor_box(*input, *result, input->shape(), std::vector<int64_t>(offset.size(), 0), offset);
+ return result;
+ }
+
+ std::shared_ptr<sd_tensor_t> cast_tensor(const std::shared_ptr<sd_tensor_t>& input, sd_tensor_type_t dtype) {
+ if (input->type == dtype)
+ return input;
+ const auto count = element_count(input->shape(), dtype);
+ if (!count)
+ return tensor_result(dtype, input->shape(), nullptr, 0);
+ return with_dtype(input->type, [&](auto source) {
+ using From = decltype(source);
+ const auto* data = static_cast<const From*>(input->data());
+ return with_dtype(dtype, [&](auto target) {
+ using To = decltype(target);
+ std::vector<To> output(count);
+ std::transform(data, data + count, output.begin(), cast_number<To, From>);
+ return tensor_result(input->shape(), std::move(output));
+ });
+ });
+ }
+ std::shared_ptr<sd_tensor_t> translate_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes, const std::vector<int64_t>& offset, const TensorNumber& value) {
+ if (axes.empty() || axes.size() != offset.size())
+ throw std::runtime_error("translate requires matching axes and offsets");
+ auto shape = input->shape();
+ std::vector<int64_t> source(shape.size(), 0), target(shape.size(), 0);
+ std::vector<bool> used(shape.size(), false);
+ for (size_t i = 0; i < axes.size(); ++i) {
+ const auto axis = tensor_axis(*input, axes[i]);
+ if (used[axis])
+ throw std::runtime_error("translate axes must be distinct");
+ used[axis] = true;
+ const auto delta = std::clamp(offset[i], -shape[axis], shape[axis]);
+ source[axis] = std::max<int64_t>(0, -delta);
+ target[axis] = std::max<int64_t>(0, delta);
+ shape[axis] -= source[axis] + target[axis];
+ }
+ auto result = filled_tensor(input->shape(), input->type, value);
+ copy_tensor_box(*input, *result, shape, source, target);
+ return result;
+ }
+ std::shared_ptr<sd_tensor_t> arange_tensor(int64_t start, int64_t end, int64_t step) {
+ if (!step)
+ throw std::runtime_error("arange step cannot be zero");
+ const bool ascending = step > 0;
+ const auto distance = ascending && start < end ? uint64_t(end) - uint64_t(start)
+ : !ascending && start > end ? uint64_t(start) - uint64_t(end)
+ : uint64_t(0);
+ const auto stride = ascending ? uint64_t(step) : uint64_t(0) - uint64_t(step);
+ const auto count = distance ? 1 + (distance - 1) / stride : 0;
+ if (count > tensor_byte_limit / sizeof(int64_t))
+ throw std::runtime_error("arange tensor exceeds 1 GiB");
+ std::vector<int64_t> values(size_t(count), start);
+ for (size_t i = 1; i < values.size(); ++i)
+ values[i] = values[i - 1] + step;
+ return make_tensor(SD_TENSOR_I64, std::vector<int64_t>{int64_t(values.size())}, values.data(), values.size() * sizeof(int64_t));
+ }
+ std::shared_ptr<sd_tensor_t> index_select_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg, const std::vector<int64_t>& indices) {
+ const auto axis = tensor_axis(*input, axis_arg);
+ const auto axis_size = input->shape()[axis];
+ for (auto index : indices)
+ if (index < 0 || index >= axis_size)
+ throw std::runtime_error("index_select index falls outside the tensor axis");
+ auto shape = input->shape();
+ shape[axis] = int64_t(indices.size());
+ auto output = make_tensor(input->type, std::move(shape));
+ if (output->bytes) {
+ const auto inner_bytes = size_t(sd::tensor_compute_strides(input->shape())[axis]) * element_size(input->type);
+ const auto* source = static_cast<const uint8_t*>(input->data());
+ auto* target = static_cast<uint8_t*>(output->mutable_data());
+ const auto rows = output->bytes / inner_bytes;
+ parallel_for(0, rows, std::max<int64_t>(1, 16384 / inner_bytes), [&](int64_t begin, int64_t end) {
+ for (int64_t row = begin; row < end; ++row) {
+ const auto offset = row / indices.size() * axis_size + indices[row % indices.size()];
+ std::memcpy(target + row * inner_bytes, source + offset * inner_bytes, inner_bytes);
+ }
+ });
+ }
+ return output;
+ }
+ std::shared_ptr<sd_tensor_t> replace_slice(const std::shared_ptr<sd_tensor_t>& input, const std::shared_ptr<sd_tensor_t>& value, int64_t axis_arg, int64_t start) {
+ const auto axis = tensor_axis(*input, axis_arg);
+ if (input->type != value->type || input->shape().size() != value->shape().size())
+ throw std::runtime_error("replacement must have the same dtype and rank");
+ if (start > input->shape()[axis] || value->shape()[axis] > input->shape()[axis] - start)
+ throw std::runtime_error("replacement slice falls outside the tensor");
+ auto expected = input->shape();
+ expected[axis] = value->shape()[axis];
+ if (expected != value->shape())
+ throw std::runtime_error("replacement shape differs outside the slice axis");
+ if (!value->bytes)
+ return input;
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ auto result = input->as<T>();
+ sd::ops::slice_assign(&result, axis, start, start + value->shape()[axis], value->as<T>());
+ return tensor_result(std::move(result));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> concat_tensors(const std::vector<std::shared_ptr<sd_tensor_t>>& items, int64_t axis_arg) {
+ if (items.empty())
+ throw std::runtime_error("concat requires at least one tensor");
+ const auto& first = *items.front();
+ const auto axis = tensor_axis(first, axis_arg);
+ auto shape = first.shape();
+ shape[axis] = 0;
+ for (const auto& item : items) {
+ const auto& input = *item;
+ auto expected = first.shape();
+ if (input.type != first.type || input.shape().size() != expected.size())
+ throw std::runtime_error("concat requires matching dtypes and ranks");
+ expected[axis] = input.shape()[axis];
+ if (expected != input.shape())
+ throw std::runtime_error("tensor shapes differ outside the concat axis");
+ if (input.shape()[axis] > INT64_MAX - shape[axis])
+ throw std::runtime_error("concatenated axis exceeds int64 range");
+ shape[axis] += input.shape()[axis];
+ }
+ if (!element_count(shape, first.type))
+ return tensor_result(first.type, std::move(shape), nullptr, 0);
+ return with_dtype(first.type, [&](auto tag) {
+ using T = decltype(tag);
+ sd::Tensor<T> result(shape);
+ int64_t start = 0;
+ for (const auto& item : items) {
+ const auto& input = *item;
+ const auto end = start + input.shape()[axis];
+ if (input.bytes)
+ sd::ops::slice_assign(&result, axis, start, end, input.as<T>());
+ start = end;
+ }
+ return tensor_result(std::move(result));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> grayscale_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg) {
+ const auto axis = tensor_axis(*input, axis_arg);
+ const auto channels = input->shape()[axis];
+ if (input->type != SD_TENSOR_U8 && input->type != SD_TENSOR_F32 && input->type != SD_TENSOR_F64)
+ throw std::runtime_error("grayscale requires u8, f32 or f64");
+ if (channels == 1)
+ return input;
+ if (channels != 3 && channels != 4)
+ throw std::runtime_error("grayscale requires one, three or four channels");
+ auto shape = input->shape();
+ shape[axis] = 1;
+ auto result = make_tensor(input->type, std::move(shape));
+ if (!result->bytes)
+ return result;
+ const auto stride = sd::tensor_compute_strides(input->shape())[axis];
+ auto convert = [&](auto tag) {
+ using T = decltype(tag);
+ const auto* src = static_cast<const T*>(input->data());
+ auto* dst = static_cast<T*>(result->mutable_data());
+ sd::tensor_for_each(result->bytes / sizeof(T), [&](int64_t i) {
+ const auto offset = i / stride * stride * channels + i % stride;
+ const auto red = src[offset], green = src[offset + stride], blue = src[offset + 2 * stride];
+ if constexpr (std::is_same_v<T, uint8_t>)
+ dst[i] = uint8_t((77 * red + 150 * green + 29 * blue + 128) / 256);
+ else
+ dst[i] = (T(77) * red + T(150) * green + T(29) * blue) / T(256);
+ });
+ return result;
+ };
+ if (input->type == SD_TENSOR_U8)
+ return convert(uint8_t{});
+ return with_float_dtype(input->type, convert);
+ }
+ std::shared_ptr<sd_tensor_t> threshold_tensor(const std::shared_ptr<sd_tensor_t>& input, const TensorNumber& threshold, const TensorNumber& low_value, const TensorNumber& high_value) {
+ auto result = make_tensor(input->type, input->shape());
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ const auto low = scalar_number<T>(low_value), high = scalar_number<T>(high_value);
+ const auto* src = static_cast<const T*>(input->data());
+ auto* dst = static_cast<T*>(result->mutable_data());
+ const auto cutoff = std::visit([](auto value) { return double(value); }, threshold);
+ if constexpr (std::is_integral_v<T>) {
+ if (std::holds_alternative<int64_t>(threshold)) {
+ const auto integer = std::get<int64_t>(threshold);
+ sd::tensor_for_each(result->bytes / sizeof(T), [&](int64_t i) {
+ dst[i] = int64_t(src[i]) >= integer ? high : low;
+ });
+ return result;
+ }
+ }
+ sd::tensor_for_each(result->bytes / sizeof(T), [&](int64_t i) {
+ dst[i] = double(src[i]) >= cutoff ? high : low;
+ });
+ return result;
+ });
+ }
+ std::shared_ptr<sd_tensor_t> maximum_tensor(const std::shared_ptr<sd_tensor_t>& a, const std::shared_ptr<sd_tensor_t>& b) {
+ if (a->type != b->type)
+ throw std::runtime_error("maximum requires matching tensor dtypes");
+ auto result = make_tensor(a->type, sd::tensor_broadcast_shape(a->shape(), b->shape()));
+ if (!result->bytes)
+ return result;
+ const auto a_shape = a->shape().empty() ? std::vector<int64_t>{1} : a->shape();
+ const auto b_shape = b->shape().empty() ? std::vector<int64_t>{1} : b->shape();
+ const auto out_shape = result->shape().empty() ? std::vector<int64_t>{1} : result->shape();
+ return with_dtype(a->type, [&](auto tag) {
+ using T = decltype(tag);
+ const auto* left = static_cast<const T*>(a->data());
+ const auto* right = static_cast<const T*>(b->data());
+ auto* dst = static_cast<T*>(result->mutable_data());
+ sd::tensor_for_each_broadcast_offset(out_shape, a_shape, sd::tensor_compute_strides(a_shape),
+ b_shape, sd::tensor_compute_strides(b_shape), [&](int64_t i, int64_t ai, int64_t bi) {
+ dst[i] = std::max(left[ai], right[bi]);
+ });
+ return result;
+ });
+ }
+ std::shared_ptr<sd_tensor_t> permute(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes) {
+ if (axes.size() != input->shape().size())
+ throw std::runtime_error("permute requires one axis per tensor dimension");
+ auto shape = input->shape();
+ std::vector<bool> used(axes.size());
+ std::vector<size_t> order(axes.size());
+ bool identical = true;
+ for (size_t i = 0; i < axes.size(); ++i) {
+ const auto axis = tensor_axis(*input, axes[i]);
+ if (used[axis])
+ throw std::runtime_error("permute axes must be unique");
+ used[axis] = true;
+ order[i] = axis;
+ identical = identical && axis == i;
+ shape[i] = input->shape()[axis];
+ }
+ if (identical)
+ return input;
+ if (!input->bytes)
+ return tensor_result(input->type, std::move(shape), nullptr, 0);
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ return tensor_result(input->as<T>().permute(order));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> broadcast_to(const std::shared_ptr<sd_tensor_t>& input, std::vector<int64_t> dims) {
+ element_count(dims, input->type);
+ if (sd::tensor_broadcast_shape(input->shape(), dims) != dims)
+ throw std::runtime_error("input cannot broadcast to the target shape");
+ if (dims == input->shape())
+ return input;
+ auto result = make_tensor(input->type, std::move(dims));
+ if (result->bytes) {
+ const auto source_shape = input->shape().empty() ? std::vector<int64_t>{1} : input->shape();
+ const auto strides = sd::tensor_compute_strides(source_shape);
+ const auto stride = element_size(input->type);
+ const auto* from = static_cast<const uint8_t*>(input->data());
+ auto* to = static_cast<uint8_t*>(result->mutable_data());
+ sd::tensor_for_each_broadcast_offset(result->shape(), source_shape, strides, source_shape, strides,
+ [&](int64_t index, int64_t source, int64_t) {
+ std::memcpy(to + index * stride, from + source * stride, stride);
+ });
+ }
+ return result;
+ }
+ std::shared_ptr<sd_tensor_t> index_fill(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg, const std::vector<int64_t>& indices, const TensorNumber& fill_value) {
+ const auto axis = tensor_axis(*input, axis_arg);
+ for (auto index : indices)
+ if (index < 0 || index >= input->shape()[axis])
+ throw std::runtime_error("fill index falls outside the tensor");
+ if (indices.empty() || !input->bytes)
+ return input;
+ return with_dtype(input->type, [&](auto tag) {
+ using T = decltype(tag);
+ auto result = input->as<T>();
+ const auto value = scalar_number<T>(fill_value);
+ for (auto index : indices)
+ sd::ops::fill_slice(&result, axis, index, index + 1, value);
+ return tensor_result(std::move(result));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> randn_like(const std::shared_ptr<sd_tensor_t>& like, const std::shared_ptr<RNG>& rng) {
+ return with_float_dtype(like->type, [&](auto tag) {
+ using T = decltype(tag);
+ auto values = rng->randn(uint32_t(like->bytes / sizeof(T)));
+ if constexpr (std::is_same_v<T, float>) {
+ return tensor_result(like->shape(), std::move(values));
+ } else {
+ std::vector<T> output(values.begin(), values.end());
+ return tensor_result(like->shape(), std::move(output));
+ }
+ });
+ }
+ int64_t count_nonzero(const std::shared_ptr<sd_tensor_t>& input) {
+ if (!input->bytes)
+ return 0;
+ return with_dtype(input->type, [&](auto zero) {
+ const auto* data = static_cast<const decltype(zero)*>(input->data());
+ return int64_t(std::count_if(data, data + input->bytes / sizeof(zero),
+ [zero](auto value) { return value != zero; }));
+ });
+ }
+ std::shared_ptr<sd_tensor_t> lerp_bytes(const sd_tensor_t& a, const sd_tensor_t& b, const sd_tensor_t* w, double scalar) {
+ if (a.type != SD_TENSOR_U8 || b.type != SD_TENSOR_U8)
+ throw std::runtime_error("u8 lerp requires two u8 endpoints");
+ auto shape = sd::tensor_broadcast_shape(a.shape(), b.shape());
+ if (w) {
+ if (w->type != SD_TENSOR_F32 && w->type != SD_TENSOR_F64)
+ throw std::runtime_error("u8 lerp requires a floating-point weight tensor or scalar");
+ shape = sd::tensor_broadcast_shape(shape, w->shape());
+ }
+ auto result = make_tensor(SD_TENSOR_U8, std::move(shape));
+ if (!result->bytes)
+ return result;
+ const auto a_shape = a.shape().empty() ? std::vector<int64_t>{1} : a.shape();
+ const auto b_shape = b.shape().empty() ? std::vector<int64_t>{1} : b.shape();
+ const auto out_shape = result->shape().empty() ? std::vector<int64_t>{1} : result->shape();
+ const auto w_strides = w ? sd::tensor_compute_strides(w->shape()) : std::vector<int64_t>{};
+ const auto* a_data = static_cast<const uint8_t*>(a.data());
+ const auto* b_data = static_cast<const uint8_t*>(b.data());
+ auto* out = static_cast<uint8_t*>(result->mutable_data());
+ sd::tensor_for_each_broadcast_offset(out_shape, a_shape, sd::tensor_compute_strides(a_shape),
+ b_shape, sd::tensor_compute_strides(b_shape), [&](int64_t index, int64_t ai, int64_t bi) {
+ double blend = scalar;
+ if (w) {
+ int64_t wi = 0, coord = index;
+ for (size_t axis = 0; axis < w->shape().size(); ++axis) {
+ if (w->shape()[axis] > 1)
+ wi += coord % out_shape[axis] * w_strides[axis];
+ coord /= out_shape[axis];
+ }
+ blend = w->type == SD_TENSOR_F32 ? static_cast<const float*>(w->data())[wi]
+ : static_cast<const double*>(w->data())[wi];
+ }
+ out[index] = rounded_byte(a_data[ai] * (1.0 - blend) + b_data[bi] * blend);
+ });
+ return result;
+ }
+}
diff --git a/src/dgl/ops/tensor.h b/src/dgl/ops/tensor.h
new file mode 100644
index 0000000..951a35b
--- /dev/null
+++ b/src/dgl/ops/tensor.h
@@ -0,0 +1,100 @@
+#ifndef __SD_DGL_OPS_TENSOR_H__
+#define __SD_DGL_OPS_TENSOR_H__
+
+#include <optional>
+
+class RNG;
+#include <variant>
+#include "dgl/tensor.h"
+
+namespace sd::dgl::ops {
+ using TensorNumber = std::variant<int64_t, double>;
+ struct TensorSlice {
+ std::optional<int64_t> start;
+ std::optional<int64_t> end;
+ };
+ using TensorIndex = std::variant<int64_t, TensorSlice>;
+ using TensorIndexResult = std::variant<int64_t, double, std::shared_ptr<sd_tensor_t>>;
+
+ TensorIndexResult index_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<TensorIndex>& indices);
+ template <class F>
+ auto with_float_dtype(sd_tensor_type_t dtype, F function) {
+ if (dtype == SD_TENSOR_F32)
+ return function(float{});
+ if (dtype == SD_TENSOR_F64)
+ return function(double{});
+ throw std::runtime_error("expected f32 or f64 tensors");
+ }
+ template <typename To, typename From>
+ To cast_number(From value) {
+ if constexpr (std::is_integral_v<To>) {
+ if constexpr (std::is_floating_point_v<From>) {
+ value = std::trunc(value);
+ const auto limit = std::ldexp(From(1), std::numeric_limits<To>::digits);
+ const auto lower = std::is_signed_v<To> ? -limit : From(0);
+ if (!(value >= lower && value < limit))
+ throw std::runtime_error("value is outside the target dtype range");
+ } else if (int64_t(value) < int64_t(std::numeric_limits<To>::lowest()) ||
+ int64_t(value) > int64_t(std::numeric_limits<To>::max())) {
+ throw std::runtime_error("value is outside the target dtype range");
+ }
+ } else if constexpr (sizeof(From) > sizeof(To) && std::is_floating_point_v<From>) {
+ if (std::isfinite(value) && std::abs(value) > std::numeric_limits<To>::max())
+ throw std::runtime_error("value is outside the target dtype range");
+ }
+ return static_cast<To>(value);
+ }
+
+ size_t element_size(sd_tensor_type_t type);
+ size_t element_count(const std::vector<int64_t>& shape, sd_tensor_type_t type);
+ size_t tensor_axis(const sd_tensor_t& tensor, int64_t axis);
+ uint8_t rounded_byte(double value);
+ void copy_tensor_box(const sd_tensor_t& input, sd_tensor_t& output, const std::vector<int64_t>& shape, const std::vector<int64_t>& source_offset, const std::vector<int64_t>& target_offset);
+ std::shared_ptr<sd_tensor_t> crop_tensor(const sd_tensor_t& input, const std::vector<int64_t>& axes, const std::vector<int64_t>& start, const std::vector<int64_t>& size);
+ std::shared_ptr<sd_tensor_t> interpolate_tensor(const sd_tensor_t& input, const std::vector<int64_t>& size, const std::vector<int64_t>& axes, const std::string& mode, bool antialias);
+ std::shared_ptr<sd_tensor_t> pool_tensor(const sd_tensor_t& input, const std::vector<int64_t>& axes, const std::vector<int64_t>& kernel, const std::vector<int64_t>& stride, const std::vector<int64_t>& before, const std::vector<int64_t>& after, std::optional<TensorNumber> pad_value, bool maximum);
+ std::shared_ptr<sd_tensor_t> blur_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes, int64_t radius, double sigma, const std::string& precision);
+ template <typename T>
+ T scalar_number(const TensorNumber& value) {
+ return std::visit([](auto item) { return cast_number<T>(item); }, value);
+ }
+
+ inline std::shared_ptr<sd_tensor_t> tensor_result(sd_tensor_type_t dtype, std::vector<int64_t> shape, const void* data, size_t bytes) {
+ return make_tensor(dtype, std::move(shape), data, bytes);
+ }
+ template <typename T>
+ std::shared_ptr<sd_tensor_t> tensor_result(sd::Tensor<T> tensor, bool scalar = false) {
+ return make_tensor(std::move(tensor), scalar);
+ }
+ template <typename T>
+ std::shared_ptr<sd_tensor_t> tensor_result(std::vector<int64_t> shape, std::vector<T> values) {
+ if (values.empty())
+ return tensor_result(tensor_dtype<T>(), std::move(shape), nullptr, 0);
+ const bool scalar = shape.empty();
+ if (scalar)
+ shape.push_back(1);
+ return tensor_result(sd::Tensor<T>(std::move(shape), std::move(values)), scalar);
+ }
+ std::shared_ptr<sd_tensor_t> filled_tensor(std::vector<int64_t> shape, sd_tensor_type_t dtype, const TensorNumber& value);
+
+ std::shared_ptr<sd_tensor_t> unsqueeze_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis);
+ std::shared_ptr<sd_tensor_t> pad_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& pad, const TensorNumber& value);
+ std::shared_ptr<sd_tensor_t> cast_tensor(const std::shared_ptr<sd_tensor_t>& input, sd_tensor_type_t dtype);
+ std::shared_ptr<sd_tensor_t> translate_tensor(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes, const std::vector<int64_t>& offset, const TensorNumber& value);
+ std::shared_ptr<sd_tensor_t> arange_tensor(int64_t start, int64_t end, int64_t step);
+ std::shared_ptr<sd_tensor_t> index_select_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg, const std::vector<int64_t>& indices);
+ std::shared_ptr<sd_tensor_t> replace_slice(const std::shared_ptr<sd_tensor_t>& input, const std::shared_ptr<sd_tensor_t>& value, int64_t axis_arg, int64_t start);
+ std::shared_ptr<sd_tensor_t> concat_tensors(const std::vector<std::shared_ptr<sd_tensor_t>>& items, int64_t axis_arg);
+ std::shared_ptr<sd_tensor_t> grayscale_tensor(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg);
+ std::shared_ptr<sd_tensor_t> threshold_tensor(const std::shared_ptr<sd_tensor_t>& input, const TensorNumber& threshold, const TensorNumber& low_value, const TensorNumber& high_value);
+ std::shared_ptr<sd_tensor_t> maximum_tensor(const std::shared_ptr<sd_tensor_t>& a, const std::shared_ptr<sd_tensor_t>& b);
+ std::shared_ptr<sd_tensor_t> permute(const std::shared_ptr<sd_tensor_t>& input, const std::vector<int64_t>& axes);
+ std::shared_ptr<sd_tensor_t> broadcast_to(const std::shared_ptr<sd_tensor_t>& input, std::vector<int64_t> dims);
+ std::shared_ptr<sd_tensor_t> index_fill(const std::shared_ptr<sd_tensor_t>& input, int64_t axis_arg, const std::vector<int64_t>& indices, const TensorNumber& fill_value);
+ std::shared_ptr<sd_tensor_t> randn_like(const std::shared_ptr<sd_tensor_t>& like, const std::shared_ptr<RNG>& rng);
+ int64_t count_nonzero(const std::shared_ptr<sd_tensor_t>& input);
+ std::shared_ptr<sd_tensor_t> lerp_bytes(const sd_tensor_t& a, const sd_tensor_t& b, const sd_tensor_t* weight, double scalar);
+
+}
+
+#endif
diff --git a/src/dgl/ops/vae.cpp b/src/dgl/ops/vae.cpp
new file mode 100644
index 0000000..ff1ae7a
--- /dev/null
+++ b/src/dgl/ops/vae.cpp
@@ -0,0 +1,73 @@
+#include "dgl/ops/vae.h"
+
+#include "core/image_tensor.h"
+#include "dgl/models/circular_axes.h"
+#include "dgl/models/vae.h"
+
+namespace sd::dgl::ops {
+
+ class VAETilingScope {
+ models::VAE& model_;
+ sd::TilingConfig previous_;
+
+ public:
+ VAETilingScope(models::VAE& model, const VAEOptions& options)
+ : model_(model), previous_(model.vae_tiling_params) {
+ if (options.tiling)
+ model_.vae_tiling_params = *options.tiling;
+ }
+ ~VAETilingScope() { model_.vae_tiling_params = previous_; }
+ };
+
+ Tensor<float> encode_vae(models::VAE& model, const Tensor<float>& pixels, const std::shared_ptr<RNG>& rng, bool posterior_mode, const VAEOptions& options) {
+ models::Model::ExecutionScope scope(model);
+ scope.require_ready();
+ models::CircularAxesScope axes(model, options.circular_x, options.circular_y);
+ VAETilingScope tiling(model, options);
+ if (pixels.empty() || (pixels.dim() != 4 && pixels.dim() != 5) || pixels.shape().back() != 1)
+ throw std::runtime_error("expected pixels with width, height, optional time, channels and batch");
+ const auto channels = pixels.shape()[pixels.dim() == 5 ? 3 : 2];
+ const auto expected = model.get_image_channels();
+ if (channels != expected && ((expected != 3 && expected != 4) || channels == 2))
+ throw std::runtime_error("pixel channels do not match the VAE");
+ const auto adjusted = channels == expected ? Tensor<float>{} : sd::ensure_image_tensor_channels(pixels, expected);
+ auto output = model.encode_vae_latents(channels == expected ? pixels : adjusted, posterior_mode, rng);
+ if (output.empty())
+ throw std::runtime_error("VAE encode failed");
+ return output;
+ }
+
+ Tensor<float> decode_vae(models::VAE& model, const Tensor<float>& latent, const VAEOptions& options) {
+ models::Model::ExecutionScope scope(model);
+ scope.require_ready();
+ models::CircularAxesScope axes(model, options.circular_x, options.circular_y);
+ VAETilingScope tiling(model, options);
+ model.validate_vae_latent(latent);
+ if (latent.dim() == 4 && latent.shape().back() > 1) {
+ Tensor<float> video;
+ for (int64_t i = 0; i < latent.shape().back(); ++i) {
+ auto frame = model.decode_vae_latents(sd::ops::slice(latent, 3, i, i + 1), false);
+ if (frame.empty() || frame.dim() != 4 || frame.shape().back() != 1)
+ throw std::runtime_error("VAE did not return an individual frame");
+ if (video.empty()) {
+ auto shape = frame.shape();
+ shape.insert(shape.begin() + 2, latent.shape().back());
+ size_t remaining = (256 * 1024 * 1024) / sizeof(float);
+ for (const auto dim : shape) {
+ if (dim <= 0 || static_cast<uint64_t>(dim) > remaining)
+ throw std::runtime_error("decoded video exceeds 256 MiB");
+ remaining /= static_cast<size_t>(dim);
+ }
+ video = Tensor<float>(std::move(shape));
+ }
+ sd::ops::slice_assign(&video, 2, i, i + 1, frame.unsqueeze(2));
+ }
+ return video;
+ }
+ auto output = model.decode_vae_latents(latent, latent.dim() == 5);
+ if (output.empty())
+ throw std::runtime_error("VAE did not return pixels");
+ return output;
+ }
+
+} // namespace sd::dgl::ops
diff --git a/src/dgl/ops/vae.h b/src/dgl/ops/vae.h
new file mode 100644
index 0000000..aa9254b
--- /dev/null
+++ b/src/dgl/ops/vae.h
@@ -0,0 +1,24 @@
+#ifndef __SD_DGL_OPS_VAE_H__
+#define __SD_DGL_OPS_VAE_H__
+
+#include <memory>
+#include <optional>
+
+#include "config.h"
+#include "core/tensor.hpp"
+#include "dgl/models/fwd.h"
+
+class RNG;
+
+namespace sd::dgl::ops {
+ struct VAEOptions {
+ bool circular_x = false;
+ bool circular_y = false;
+ std::optional<sd::TilingConfig> tiling;
+ };
+
+ Tensor<float> encode_vae(models::VAE& model, const Tensor<float>& pixels, const std::shared_ptr<RNG>& rng, bool posterior_mode, const VAEOptions& options);
+ Tensor<float> decode_vae(models::VAE& model, const Tensor<float>& latent, const VAEOptions& options);
+}
+
+#endif // __SD_DGL_OPS_VAE_H__
diff --git a/src/dgl/ops/video.cpp b/src/dgl/ops/video.cpp
new file mode 100644
index 0000000..2e11f1b
--- /dev/null
+++ b/src/dgl/ops/video.cpp
@@ -0,0 +1,560 @@
+#include "dgl/ops/video.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <numeric>
+
+#include "api/metadata.h"
+#include "core/file_access.h"
+#include "dgl/ops/audio.h"
+#include "dgl/ops/image.h"
+#include "dgl/ops/video_webm.h"
+#include "dgl/ops/video_webp.h"
+
+namespace sd::dgl::ops {
+
+ void validate_video_frames(const sd_tensor_t& frames) {
+ const auto& shape = frames.shape();
+ if (frames.type != SD_TENSOR_U8 || shape.size() != 4 || shape[0] != 3 ||
+ shape[1] < 1 || shape[1] > 16384 || shape[2] < 1 || shape[2] > 16384 || shape[3] < 1)
+ throw std::runtime_error("video frames must be U8 [3, width, height, frames]");
+ }
+
+ sd::Image video_frame_view(const sd_tensor_t& frames, size_t index) {
+ validate_video_frames(frames);
+ const auto& shape = frames.shape();
+ if (index >= size_t(shape[3]))
+ throw std::runtime_error("video frame index is out of range");
+ const auto* data = static_cast<const uint8_t*>(frames.data()) + index * size_t(shape[1] * shape[2] * 3);
+ return {uint32_t(shape[1]), uint32_t(shape[2]), 3, const_cast<uint8_t*>(data)};
+ }
+
+ VideoReadResult load_video(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio) {
+ std::ifstream file(path, std::ios::binary);
+ uint8_t header[12]{};
+ if (!file || !file.read(reinterpret_cast<char*>(header), sizeof(header)))
+ throw std::runtime_error("cannot read video input header");
+ if (!std::memcmp(header, "RIFF", 4)) {
+ if (!std::memcmp(header + 8, "AVI ", 4))
+ return load_avi(path, max_frames, fps_num, fps_den, include_audio);
+ if (!std::memcmp(header + 8, "WEBP", 4))
+ return load_webp(path, max_frames, fps_num, fps_den);
+ }
+ if (header[0] == 0x1a && header[1] == 0x45 && header[2] == 0xdf && header[3] == 0xa3)
+ return load_webm(path, max_frames, fps_num, fps_den, include_audio);
+ throw std::runtime_error("unsupported video input; expected AVI, WebP or independent-frame VP8 WebM");
+ }
+
+ static uint32_t fourcc(const char* text) {
+ return uint32_t(uint8_t(text[0])) | (uint32_t(uint8_t(text[1])) << 8) |
+ (uint32_t(uint8_t(text[2])) << 16) | (uint32_t(uint8_t(text[3])) << 24);
+ }
+
+ static uint32_t read_u32(const uint8_t* p) {
+ return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24);
+ }
+
+ struct AVIChunk {
+ uint32_t id;
+ uint64_t data, size, next;
+ };
+
+ class AVIReader {
+ std::ifstream file_;
+ uint64_t size_;
+ size_t chunks_ = 0;
+
+ public:
+ explicit AVIReader(const std::filesystem::path& path)
+ : file_(path, std::ios::binary | std::ios::ate) {
+ if (!file_ || file_.tellg() < 12)
+ throw std::runtime_error("cannot read AVI file");
+ size_ = uint64_t(file_.tellg());
+ }
+
+ uint64_t size() const { return size_; }
+
+ std::vector<uint8_t> read(uint64_t offset, uint64_t size) {
+ if (offset > size_ || size > size_ - offset || size > 256 * 1024 * 1024)
+ throw std::runtime_error("invalid AVI data size");
+ std::vector<uint8_t> bytes(size_t(size), 0);
+ file_.seekg(std::streamoff(offset));
+ if (size && !file_.read(reinterpret_cast<char*>(bytes.data()), std::streamsize(size)))
+ throw std::runtime_error("truncated AVI file");
+ return bytes;
+ }
+
+ AVIChunk chunk(uint64_t offset, uint64_t end) {
+ if (offset > end || end - offset < 8 || ++chunks_ > 1000000)
+ throw std::runtime_error("invalid AVI chunk structure or too many chunks");
+ auto bytes = read(offset, 8);
+ uint64_t size = read_u32(bytes.data() + 4);
+ uint64_t padded = size + (size & 1);
+ if (padded > end - offset - 8)
+ throw std::runtime_error("AVI chunk exceeds its container");
+ return {read_u32(bytes.data()), offset + 8, size, offset + 8 + padded};
+ }
+ };
+
+ struct AVIStream {
+ int index = -1;
+ int width = 0, height = 0;
+ int64_t fps_num = 0, fps_den = 0;
+ bool jpeg = false, top_down = false;
+ };
+
+ struct AVIAudioStream {
+ int index = -1;
+ int encoding = 0, bits = 0, channels = 0;
+ uint32_t rate = 0, block = 0;
+ };
+
+ static uint16_t read_u16(const uint8_t* data) {
+ return uint16_t(data[0]) | (uint16_t(data[1]) << 8);
+ }
+
+ static void read_avi_stream(AVIReader& reader, uint64_t start, uint64_t end, int index, AVIStream& stream, AVIAudioStream* audio) {
+ std::vector<uint8_t> header, format;
+ for (uint64_t pos = start; pos < end;) {
+ auto chunk = reader.chunk(pos, end);
+ if (chunk.id == fourcc("strh")) {
+ if (chunk.size < 56 || !header.empty())
+ throw std::runtime_error("invalid AVI stream header");
+ header = reader.read(chunk.data, 56);
+ } else if (chunk.id == fourcc("strf")) {
+ if (chunk.size >= 16)
+ format = reader.read(chunk.data, std::min<uint64_t>(chunk.size, 40));
+ }
+ pos = chunk.next;
+ }
+ if (!header.empty() && read_u32(header.data()) == fourcc("auds") && audio && audio->index < 0) {
+ if (format.size() < 16 || read_u32(header.data() + 28) != 0)
+ throw std::runtime_error("invalid AVI audio header or nonzero stream start");
+ auto p = format.data();
+ audio->encoding = read_u16(p);
+ audio->channels = read_u16(p + 2);
+ audio->rate = read_u32(p + 4);
+ audio->block = read_u16(p + 12);
+ audio->bits = read_u16(p + 14);
+ if (audio->channels < 1 || audio->channels > 32 || audio->rate < 1000 || audio->rate > 384000 ||
+ !((audio->encoding == 1 && (audio->bits == 8 || audio->bits == 16 || audio->bits == 24 || audio->bits == 32)) || (audio->encoding == 3 && audio->bits == 32)) || audio->block != uint32_t(audio->channels * (audio->bits / 8)))
+ throw std::runtime_error("AVI audio requires PCM or float32; use video::load(audio=false) to skip unsupported audio");
+ audio->index = index;
+ }
+ if (header.empty() || read_u32(header.data()) != fourcc("vids") || stream.index >= 0)
+ return;
+ if (format.size() < 40 || read_u32(format.data()) < 40)
+ throw std::runtime_error("missing AVI bitmap header");
+ auto p = format.data();
+ int64_t width = read_u32(p + 4);
+ uint32_t signed_height = read_u32(p + 8);
+ bool top_down = (signed_height & 0x80000000u) != 0;
+ int64_t height = top_down ? int64_t(0x100000000ull - signed_height) : signed_height;
+ uint32_t compression = read_u32(p + 16);
+ bool jpeg = compression == fourcc("MJPG") || compression == fourcc("mjpg");
+ if (width < 1 || width > 16384 || height < 1 || height > 16384 || width * height > 64 * 1024 * 1024 ||
+ p[12] != 1 || p[13] != 0 || p[14] != 24 || p[15] != 0 || (!jpeg && compression != 0))
+ throw std::runtime_error("AVI requires 24-bit RGB or MJPEG frames with supported dimensions");
+ int64_t num = read_u32(header.data() + 24), den = read_u32(header.data() + 20);
+ if (!num || !den)
+ throw std::runtime_error("AVI has no usable frame rate");
+ int64_t divisor = std::gcd(num, den);
+ num /= divisor;
+ den /= divisor;
+ if (num > 1000000 || den > 1000000 || read_u32(header.data() + 28) != 0)
+ throw std::runtime_error("unsupported AVI frame rate or nonzero stream start");
+ stream = {index, int(width), int(height), num, den, jpeg, top_down};
+ }
+
+ static void read_avi_header(AVIReader& reader, uint64_t start, uint64_t end, AVIStream& stream, AVIAudioStream* audio) {
+ int index = 0;
+ for (uint64_t pos = start; pos < end;) {
+ auto chunk = reader.chunk(pos, end);
+ if (chunk.id == fourcc("LIST")) {
+ if (chunk.size < 4)
+ throw std::runtime_error("invalid AVI list");
+ auto type = reader.read(chunk.data, 4);
+ if (read_u32(type.data()) == fourcc("strl")) {
+ if (index >= 100)
+ throw std::runtime_error("too many AVI streams");
+ read_avi_stream(reader, chunk.data + 4, chunk.data + chunk.size, index++, stream, audio);
+ }
+ }
+ pos = chunk.next;
+ }
+ }
+
+ static void read_avi_frames(AVIReader& reader, uint64_t start, uint64_t end, int stream, std::vector<AVIChunk>& frames, int depth = 0, bool audio = false) {
+ if (depth > 8)
+ throw std::runtime_error("AVI lists are nested too deeply");
+ char prefix[] = {char('0' + stream / 10), char('0' + stream % 10), audio ? 'w' : 'd', 'b'};
+ uint32_t raw = fourcc(prefix);
+ prefix[3] = audio ? 'b' : 'c';
+ uint32_t compressed = fourcc(prefix);
+ for (uint64_t pos = start; pos < end;) {
+ auto chunk = reader.chunk(pos, end);
+ if (chunk.id == raw || chunk.id == compressed) {
+ if (chunk.size == 0 && !audio)
+ throw std::runtime_error("AVI dropped frames are unsupported");
+ if (chunk.size)
+ frames.push_back(chunk);
+ } else if (chunk.id == fourcc("LIST")) {
+ if (chunk.size < 4)
+ throw std::runtime_error("invalid AVI frame list");
+ auto type = reader.read(chunk.data, 4);
+ if (read_u32(type.data()) == fourcc("rec "))
+ read_avi_frames(reader, chunk.data + 4, chunk.data + chunk.size, stream, frames, depth + 1, audio);
+ }
+ pos = chunk.next;
+ }
+ }
+
+ VideoReadResult load_avi(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio) {
+ if (!std::filesystem::is_regular_file(path))
+ throw std::runtime_error("video path is not a regular file");
+ AVIReader reader(path);
+ auto root = reader.read(0, 12);
+ if (read_u32(root.data()) != fourcc("RIFF") || read_u32(root.data() + 8) != fourcc("AVI "))
+ throw std::runtime_error("video input supports AVI (RGB or MJPEG) only");
+ uint64_t end = uint64_t(read_u32(root.data() + 4)) + 8;
+ if (end != reader.size() || end < 12)
+ throw std::runtime_error("invalid AVI size; segmented OpenDML AVI is unsupported");
+ AVIStream stream;
+ AVIAudioStream audio;
+ std::vector<AVIChunk> movies;
+ bool header_found = false;
+ for (uint64_t pos = 12; pos < end;) {
+ auto chunk = reader.chunk(pos, end);
+ if (chunk.id == fourcc("LIST")) {
+ if (chunk.size < 4)
+ throw std::runtime_error("invalid AVI list");
+ auto type = reader.read(chunk.data, 4);
+ if (read_u32(type.data()) == fourcc("hdrl")) {
+ if (header_found)
+ throw std::runtime_error("duplicate AVI header");
+ read_avi_header(reader, chunk.data + 4, chunk.data + chunk.size, stream, include_audio ? &audio : nullptr);
+ header_found = true;
+ } else if (read_u32(type.data()) == fourcc("movi")) {
+ movies.push_back(chunk);
+ }
+ }
+ pos = chunk.next;
+ }
+ if (stream.index < 0)
+ throw std::runtime_error("AVI has no video stream");
+ std::vector<AVIChunk> frames;
+ for (const auto& movie : movies)
+ read_avi_frames(reader, movie.data + 4, movie.data + movie.size, stream.index, frames);
+ if (frames.empty())
+ throw std::runtime_error("AVI contains no video frames");
+ if (!fps_num) {
+ fps_num = stream.fps_num;
+ fps_den = stream.fps_den;
+ }
+ int64_t divisor = std::gcd(fps_num, fps_den);
+ fps_num /= divisor;
+ fps_den /= divisor;
+ uint64_t step_num = uint64_t(stream.fps_num) * fps_den;
+ uint64_t step_den = uint64_t(stream.fps_den) * fps_num;
+ uint64_t count = (frames.size() * step_den + step_num - 1) / step_num;
+ if (max_frames)
+ count = std::min(count, uint64_t(max_frames));
+ size_t frame_size = size_t(stream.width) * stream.height * 3;
+ if (count > (256 * 1024 * 1024) / frame_size)
+ throw std::runtime_error("decoded video exceeds 256 MiB; set max_frames");
+ VideoReadResult video;
+ video.frames = make_tensor(SD_TENSOR_U8, std::vector<int64_t>{3, stream.width, stream.height, int64_t(count)});
+ video.fps_num = fps_num;
+ video.fps_den = fps_den;
+ size_t last = frames.size();
+ std::shared_ptr<sd_tensor_t> image;
+ for (uint64_t i = 0; i < count; ++i) {
+ size_t source = size_t(i * step_num / step_den);
+ if (source != last) {
+ const auto& chunk = frames.at(source);
+ size_t stride = (size_t(stream.width) * 3 + 3) & ~size_t(3);
+ if (!stream.jpeg && chunk.size != stride * stream.height)
+ throw std::runtime_error("invalid RGB AVI frame size");
+ auto bytes = reader.read(chunk.data, chunk.size);
+ if (stream.jpeg) {
+ image = decode_video_jpeg(bytes, stream.width, stream.height);
+ } else {
+ image = allocate_image(stream.width, stream.height, 3);
+ for (int y = 0; y < stream.height; ++y) {
+ auto src = bytes.data() + stride * (stream.top_down ? y : stream.height - 1 - y);
+ auto dst = static_cast<uint8_t*>(image->mutable_data()) + size_t(y) * stream.width * 3;
+ for (int x = 0; x < stream.width; ++x) {
+ dst[x * 3] = src[x * 3 + 2];
+ dst[x * 3 + 1] = src[x * 3 + 1];
+ dst[x * 3 + 2] = src[x * 3];
+ }
+ }
+ }
+ last = source;
+ }
+ std::memcpy(static_cast<uint8_t*>(video.frames->mutable_data()) + size_t(i) * frame_size, image->data(), frame_size);
+ }
+ if (audio.index >= 0) {
+ std::vector<AVIChunk> packets;
+ for (const auto& movie : movies)
+ read_avi_frames(reader, movie.data + 4, movie.data + movie.size, audio.index, packets, 0, true);
+ uint64_t wanted = (count * uint64_t(video.fps_den) * audio.rate + video.fps_num - 1) / video.fps_num;
+
+ std::vector<uint8_t> data;
+ for (const auto& packet : packets) {
+ if (packet.size % audio.block)
+ throw std::runtime_error("incomplete AVI audio packet");
+ uint64_t limit = wanted * audio.block;
+ if (data.size() >= limit)
+ break;
+ uint64_t size = std::min<uint64_t>(packet.size, limit - data.size());
+ uint64_t max_bytes = (256 * 1024 * 1024 / (audio.channels * sizeof(float))) * audio.block;
+ if (data.size() + size > max_bytes)
+ throw std::runtime_error("decoded video audio exceeds 256 MiB; set max_frames");
+ auto bytes = reader.read(packet.data, size);
+ data.insert(data.end(), bytes.begin(), bytes.end());
+ }
+ if (!data.empty())
+ video.audio = decode_pcm_audio(data.data(), data.size(), audio.encoding, audio.bits, audio.channels, audio.rate);
+ }
+ return video;
+ }
+
+ class AVIWriter {
+ std::ofstream file_;
+ std::vector<uint8_t> bytes_;
+ uint64_t position_ = 0;
+ bool memory_;
+
+ public:
+ explicit AVIWriter(const std::filesystem::path* path) : memory_(!path) {
+ if (path)
+ file_.open(*path, std::ios::binary | std::ios::trunc);
+ if (!memory_ && !file_)
+ throw std::runtime_error("cannot open AVI output");
+ }
+ uint64_t position() {
+ if (memory_)
+ return position_;
+ auto pos = file_.tellp();
+ if (pos < 0)
+ throw std::runtime_error("cannot get AVI output position");
+ return uint64_t(pos);
+ }
+ void write(const void* bytes, size_t size) {
+ if (size > UINT32_MAX - position())
+ throw std::runtime_error("AVI output exceeds 4 GiB");
+ if (memory_) {
+ const auto end = size_t(position_) + size;
+ if (end > bytes_.size())
+ bytes_.resize(end);
+ if (size)
+ std::memcpy(bytes_.data() + position_, bytes, size);
+ position_ = end;
+ } else if (!file_.write(static_cast<const char*>(bytes), std::streamsize(size)))
+ throw std::runtime_error("cannot write AVI output");
+ }
+ void seek(uint64_t position) {
+ if (memory_)
+ position_ = position;
+ else
+ file_.seekp(std::streamoff(position));
+ }
+ void u32(uint32_t value) {
+ uint8_t bytes[] = {uint8_t(value), uint8_t(value >> 8), uint8_t(value >> 16), uint8_t(value >> 24)};
+ write(bytes, 4);
+ }
+ void id(const char* value) { write(value, 4); }
+ uint64_t begin(const char* type) {
+ id(type);
+ uint64_t start = position();
+ u32(0);
+ return start;
+ }
+ void end(uint64_t start) {
+ uint64_t next = position();
+ uint32_t size = uint32_t(next - start - 4);
+ seek(start);
+ u32(size);
+ seek(next);
+ if (size & 1) {
+ const char padding = 0;
+ write(&padding, 1);
+ }
+ }
+ void close() {
+ if (memory_)
+ return;
+ file_.close();
+ if (!file_)
+ throw std::runtime_error("cannot close AVI output");
+ }
+ std::vector<uint8_t> take_bytes() { return std::move(bytes_); }
+ };
+
+ static std::vector<uint8_t> write_avi(const sd_tensor_t& frames, const std::filesystem::path* path, const VideoSaveOptions& options) {
+ validate_video_frames(frames);
+ if (options.fps_num < 1 || options.fps_num > UINT32_MAX || options.fps_den < 1 || options.fps_den > UINT32_MAX)
+ throw std::runtime_error("video frame rate numerator and denominator must fit positive AVI 32-bit fields");
+ const auto divisor = std::gcd(options.fps_num, options.fps_den);
+ const auto fps_num = options.fps_num / divisor;
+ const auto fps_den = options.fps_den / divisor;
+ const auto frame_count = size_t(frames.shape()[3]);
+ const bool jpeg = options.jpeg, pcm16 = options.pcm16;
+ const int quality = options.quality;
+ const auto first = video_frame_view(frames, 0);
+ uint32_t width = first.width, height = first.height, stride = (width * 3 + 3) & ~uint32_t(3), frame_size = stride * height;
+ uint64_t microseconds = (1000000ull * fps_den + fps_num / 2) / fps_num;
+ if (microseconds > UINT32_MAX)
+ throw std::runtime_error("frame duration exceeds AVI limits");
+ sd::Audio audio{};
+ if (options.audio.samples && options.audio.samples->bytes)
+ audio = audio_view(*options.audio.samples, options.audio.sample_rate);
+ if (options.trim_audio && audio.data) {
+ const auto duration = uint64_t(frame_count) * uint64_t(fps_den);
+ const auto track_end = audio.sample_count * uint64_t(fps_num);
+ // Compare before multiplying the video duration by the sample rate.
+ if (duration <= (track_end - 1) / audio.sample_rate)
+ audio.sample_count = duration * audio.sample_rate / uint64_t(fps_num);
+ if (!audio.sample_count)
+ audio.data = nullptr;
+ }
+ uint32_t block = audio.channels * (pcm16 ? 2 : 4);
+ AVIWriter writer(path);
+ auto riff = writer.begin("RIFF");
+ writer.id("AVI ");
+ auto header = writer.begin("LIST");
+ writer.id("hdrl");
+ auto main_header = writer.begin("avih");
+ for (uint32_t value : {uint32_t(microseconds), 0u, 0u, audio.data ? 0x110u : 0x10u, uint32_t(frame_count), 0u, audio.data ? 2u : 1u, frame_size, width, height, 0u, 0u, 0u, 0u})
+ writer.u32(value);
+ writer.end(main_header);
+ auto stream = writer.begin("LIST");
+ writer.id("strl");
+ auto stream_header = writer.begin("strh");
+ writer.id("vids");
+ writer.id(jpeg ? "MJPG" : "DIB ");
+ for (uint32_t value : {0u, 0u, 0u, uint32_t(fps_den), uint32_t(fps_num), 0u, uint32_t(frame_count), frame_size, UINT32_MAX, 0u, 0u, width | (height << 16)})
+ writer.u32(value);
+ writer.end(stream_header);
+ auto format = writer.begin("strf");
+ for (uint32_t value : {40u, width, height, 1u | (24u << 16), jpeg ? fourcc("MJPG") : 0u, frame_size, 0u, 0u, 0u, 0u})
+ writer.u32(value);
+ writer.end(format);
+ writer.end(stream);
+ if (audio.data) {
+ auto track = writer.begin("LIST");
+ writer.id("strl");
+ auto stream = writer.begin("strh");
+ writer.id("auds");
+ writer.u32(0);
+ for (uint32_t value : {0u, 0u, 0u, block, audio.sample_rate * block, 0u, uint32_t(audio.sample_count), uint32_t(std::min<uint64_t>(audio.sample_count * block, 65536)), UINT32_MAX, block, 0u, 0u})
+ writer.u32(value);
+ writer.end(stream);
+ auto format = writer.begin("strf");
+ writer.u32((pcm16 ? 1u : 3u) | (audio.channels << 16));
+ writer.u32(audio.sample_rate);
+ writer.u32(audio.sample_rate * block);
+ writer.u32(block | ((pcm16 ? 16u : 32u) << 16));
+ writer.end(format);
+ writer.end(track);
+ }
+ writer.end(header);
+ const auto metadata = sd::metadata::avi_info(options.metadata);
+ writer.write(metadata.data(), metadata.size());
+ auto movie = writer.begin("LIST");
+ writer.id("movi");
+ struct Index {
+ uint32_t id, offset, size;
+ };
+ std::vector<Index> index;
+ auto packet = [&](const char* id, const std::vector<uint8_t>& bytes) {
+ index.push_back({fourcc(id), uint32_t(writer.position() - movie - 4), uint32_t(bytes.size())});
+ auto start = writer.begin(id);
+ writer.write(bytes.data(), bytes.size());
+ writer.end(start);
+ };
+ std::vector<uint8_t> bytes;
+ if (!jpeg)
+ bytes.resize(frame_size, 0);
+ uint64_t sample = 0;
+ for (size_t i = 0; i < frame_count; ++i) {
+ const auto image = video_frame_view(frames, i);
+ if (jpeg)
+ bytes = encode_video_jpeg(image, quality);
+ else {
+ for (uint32_t y = 0; y < height; ++y) {
+ auto src = image.data + size_t(y) * width * 3;
+ auto dst = bytes.data() + size_t(height - 1 - y) * stride;
+ for (uint32_t x = 0; x < width; ++x) {
+ dst[x * 3] = src[x * 3 + 2];
+ dst[x * 3 + 1] = src[x * 3 + 1];
+ dst[x * 3 + 2] = src[x * 3];
+ }
+ }
+ }
+ packet(jpeg ? "00dc" : "00db", bytes);
+ if (audio.data) {
+ uint64_t end = std::min<uint64_t>(audio.sample_count, ((i + 1) * uint64_t(fps_den) * audio.sample_rate + fps_num - 1) / fps_num);
+ if (end > sample)
+ packet("01wb", encode_pcm_audio(audio, pcm16, sample, end - sample));
+ sample = end;
+ }
+ }
+ if (audio.data && sample < audio.sample_count)
+ packet("01wb", encode_pcm_audio(audio, pcm16, sample, audio.sample_count - sample));
+ writer.end(movie);
+ auto idx = writer.begin("idx1");
+ for (const auto& item : index) {
+ writer.u32(item.id);
+ writer.u32(0x10);
+ writer.u32(item.offset);
+ writer.u32(item.size);
+ }
+ writer.end(idx);
+ writer.end(riff);
+ writer.close();
+ return writer.take_bytes();
+ }
+
+ std::vector<uint8_t> encode_avi(const sd_tensor_t& frames, const VideoSaveOptions& options) {
+ return write_avi(frames, nullptr, options);
+ }
+
+ void save_video(const sd_tensor_t& frames, const std::filesystem::path& path, const VideoSaveOptions& options) {
+ auto extension = path.extension().u8string();
+ std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ if (extension != ".avi")
+ throw std::runtime_error("video output supports .avi (RGB or MJPEG) only");
+ require_file_access(path.u8string(), SD_DGL_FILE_WRITE);
+ write_avi(frames, &path, options);
+ }
+
+ VideoReadResult load_frame_directory(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den) {
+ auto images = read_image_directory(path, max_frames, 3);
+ const auto& shape = images.front()->shape();
+ VideoReadResult result;
+ result.fps_num = fps_num ? fps_num : 24;
+ result.fps_den = fps_den;
+ const auto divisor = std::gcd(result.fps_num, result.fps_den);
+ result.fps_num /= divisor;
+ result.fps_den /= divisor;
+ const auto stride = images.front()->bytes;
+ for (const auto& image : images)
+ if (image->shape() != shape)
+ throw std::runtime_error("video frame dimensions must match");
+ if (images.size() > (256 * 1024 * 1024) / stride)
+ throw std::runtime_error("video frames exceed 256 MiB");
+ result.frames = make_tensor(SD_TENSOR_U8, std::vector<int64_t>{3, shape[1], shape[2], int64_t(images.size())});
+ for (size_t i = 0; i < images.size(); ++i) {
+ std::memcpy(static_cast<uint8_t*>(result.frames->mutable_data()) + i * stride, images[i]->data(), stride);
+ }
+ validate_video_frames(*result.frames);
+ return result;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/ops/video.h b/src/dgl/ops/video.h
new file mode 100644
index 0000000..18ff43d
--- /dev/null
+++ b/src/dgl/ops/video.h
@@ -0,0 +1,36 @@
+#ifndef __SD_DGL_OPS_VIDEO_H__
+#define __SD_DGL_OPS_VIDEO_H__
+
+#include <filesystem>
+#include "dgl/ops/audio.h"
+#include "dgl/tensor.h"
+
+namespace sd::dgl::ops {
+ struct VideoReadResult {
+ std::shared_ptr<sd_tensor_t> frames;
+ int64_t fps_num = 24;
+ int64_t fps_den = 1;
+ AudioReadResult audio;
+ };
+
+ struct VideoSaveOptions {
+ int64_t fps_num = 24;
+ int64_t fps_den = 1;
+ AudioReadResult audio;
+ bool trim_audio = false;
+ bool jpeg = true;
+ int quality = 90;
+ bool pcm16 = false;
+ std::string metadata;
+ };
+
+ void validate_video_frames(const sd_tensor_t& frames);
+ sd::Image video_frame_view(const sd_tensor_t& frames, size_t index);
+ VideoReadResult load_frame_directory(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den);
+ VideoReadResult load_video(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio);
+ VideoReadResult load_avi(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio);
+ void save_video(const sd_tensor_t& frames, const std::filesystem::path& path, const VideoSaveOptions& options);
+ std::vector<uint8_t> encode_avi(const sd_tensor_t& frames, const VideoSaveOptions& options);
+}
+
+#endif // __SD_DGL_OPS_VIDEO_H__
diff --git a/src/dgl/ops/video_encoding.cpp b/src/dgl/ops/video_encoding.cpp
new file mode 100644
index 0000000..9bfe60a
--- /dev/null
+++ b/src/dgl/ops/video_encoding.cpp
@@ -0,0 +1,111 @@
+#include "dgl/ops/media_encoding.h"
+
+#include <cstring>
+#include <memory>
+#include <stdexcept>
+
+#include "dgl/ops/video_timing.h"
+
+#ifdef SD_USE_WEBM
+#include <mkvmuxer/mkvmuxer.h>
+#include <webp/encode.h>
+#endif
+
+namespace sd::dgl::ops {
+#ifdef SD_USE_WEBM
+ namespace media_encoding_detail {
+ class MemoryWebMWriter : public mkvmuxer::IMkvWriter {
+ std::vector<uint8_t> bytes_;
+ size_t position_ = 0;
+
+ public:
+ mkvmuxer::int32 Write(const void* data, mkvmuxer::uint32 size) override {
+ if ((!data && size) || size > size_t(INT64_MAX) - position_)
+ return -1;
+ const auto end = position_ + size;
+ if (end > bytes_.size())
+ bytes_.resize(end);
+ if (size)
+ std::memcpy(bytes_.data() + position_, data, size);
+ position_ = end;
+ return 0;
+ }
+ mkvmuxer::int64 Position() const override { return int64_t(position_); }
+ mkvmuxer::int32 Position(mkvmuxer::int64 value) override {
+ if (value < 0 || uint64_t(value) > bytes_.size())
+ return -1;
+ position_ = size_t(value);
+ return 0;
+ }
+ bool Seekable() const override { return true; }
+ void ElementStartNotify(mkvmuxer::uint64, mkvmuxer::int64) override {}
+ std::vector<uint8_t> take_bytes() { return std::move(bytes_); }
+ };
+
+ static std::vector<uint8_t> vp8_frame(const sd::Image& image, int quality) {
+ uint8_t* data = nullptr;
+ const auto size = WebPEncodeRGB(image.data, image.width, image.height, image.width * 3, float(quality), &data);
+ std::unique_ptr<uint8_t, decltype(&WebPFree)> encoded(data, WebPFree);
+ if (!size)
+ throw std::runtime_error("cannot encode WebM VP8 frame");
+ for (size_t offset = 12; offset <= size && size - offset >= 8;) {
+ const auto* chunk = data + offset;
+ const uint32_t length = uint32_t(chunk[4]) | (uint32_t(chunk[5]) << 8) | (uint32_t(chunk[6]) << 16) | (uint32_t(chunk[7]) << 24);
+ if (length > size - offset - 8)
+ break;
+ if (!std::memcmp(chunk, "VP8 ", 4))
+ return {chunk + 8, chunk + 8 + length};
+ offset += 8 + size_t(length) + (length & 1);
+ }
+ throw std::runtime_error("WebP encoder did not produce a VP8 frame");
+ }
+
+ }
+#endif
+
+ std::vector<uint8_t> encode_webm(const sd_tensor_t& frames, const VideoSaveOptions& options) {
+#ifndef SD_USE_WEBM
+ throw std::runtime_error("video::encode WebM requires a build with SD_WEBM=ON");
+#else
+ using namespace media_encoding_detail;
+ validate_video_frames(frames);
+ if (options.fps_num < 1 || options.fps_num > 1000000 || options.fps_den < 1 || options.fps_den > 1000000)
+ throw std::runtime_error("WebM frame rate numerator and denominator must be in [1,1000000]");
+ if (options.audio.samples && options.audio.samples->bytes)
+ throw std::runtime_error("video::encode WebM does not support audio; use format=\"avi\" for PCM16 audio");
+ MemoryWebMWriter writer;
+ mkvmuxer::Segment segment;
+ if (!segment.Init(&writer))
+ throw std::runtime_error("cannot initialize WebM output");
+ segment.set_mode(mkvmuxer::Segment::kFile);
+ segment.OutputCues(true);
+ segment.GetSegmentInfo()->set_timecode_scale(1000);
+ segment.GetSegmentInfo()->set_writing_app("stable-diffusion.cpp");
+ segment.GetSegmentInfo()->set_muxing_app("stable-diffusion.cpp");
+ const auto first = video_frame_view(frames, 0);
+ const auto track = segment.AddVideoTrack(first.width, first.height, 0);
+ auto* video = static_cast<mkvmuxer::VideoTrack*>(segment.GetTrackByNumber(track));
+ if (!track || !video || !segment.CuesTrack(track))
+ throw std::runtime_error("cannot create WebM VP8 track");
+ video->set_codec_id("V_VP8");
+ video->set_frame_rate(double(options.fps_num) / options.fps_den);
+ const VideoSampling clock{{options.fps_num, options.fps_den}, size_t(frames.shape()[3])};
+ video->set_default_duration(uint64_t(clock.time_ns(1)));
+ if (!options.metadata.empty()) {
+ auto* tag = segment.AddTag();
+ if (!tag || !tag->add_simple_tag("COMMENT", options.metadata.c_str()))
+ throw std::runtime_error("cannot write WebM generation metadata");
+ }
+ for (size_t frame = 0; frame < clock.count; ++frame) {
+ auto packet = vp8_frame(video_frame_view(frames, frame), options.quality);
+ if (!segment.AddFrame(packet.data(), packet.size(), track, uint64_t(clock.time_ns(frame)), true))
+ throw std::runtime_error("cannot mux WebM video frame");
+ }
+ const auto duration = uint64_t(clock.time_ns(clock.count));
+ segment.set_duration(double(duration) / segment.GetSegmentInfo()->timecode_scale());
+ if (!segment.Finalize())
+ throw std::runtime_error("cannot finalize WebM output");
+ return writer.take_bytes();
+#endif
+ }
+}
diff --git a/src/dgl/ops/video_timing.cpp b/src/dgl/ops/video_timing.cpp
new file mode 100644
index 0000000..d6a990f
--- /dev/null
+++ b/src/dgl/ops/video_timing.cpp
@@ -0,0 +1,104 @@
+#include "dgl/ops/video_timing.h"
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <numeric>
+#include <stdexcept>
+
+namespace sd::dgl::ops {
+ VideoFrameRate video_frame_rate(double rate) {
+ if (!std::isfinite(rate) || rate <= 0 || rate > 1000000)
+ return {};
+ const auto nominal = int64_t(std::llround(rate * 1001 / 1000));
+ const auto divisor = std::gcd(nominal * 1000, int64_t(1001));
+ // Preserve common fractional clock rates despite the WebM float field.
+ if (nominal > 0 && nominal * 1000 / divisor <= 1000000 &&
+ static_cast<float>(double(nominal) * 1000 / 1001) == static_cast<float>(rate))
+ return {nominal * 1000 / divisor, 1001 / divisor};
+ int64_t n0 = 0, n1 = 1, d0 = 1, d1 = 0;
+ long double value = rate;
+ for (int i = 0; i < 64; ++i) {
+ const auto whole = std::floor(value);
+ const auto n = whole * n1 + n0, d = whole * d1 + d0;
+ if (n > 1000000 || d > 1000000)
+ break;
+ n0 = n1;
+ d0 = d1;
+ n1 = int64_t(n);
+ d1 = int64_t(d);
+ if (n1 > 0 && static_cast<float>(double(n1) / d1) == static_cast<float>(rate))
+ return {n1, d1};
+ const auto fraction = value - whole;
+ if (fraction <= std::numeric_limits<long double>::epsilon())
+ break;
+ value = 1 / fraction;
+ }
+ return {};
+ }
+
+ int64_t VideoSampling::time_ns(size_t frame) const {
+ const uint64_t period = uint64_t(rate.den) * 1000000000;
+ const auto whole = period / rate.num, remainder = period % rate.num;
+ const auto limit = uint64_t(std::numeric_limits<int64_t>::max());
+ if (whole && frame > limit / whole)
+ return int64_t(limit);
+ const auto integral = uint64_t(frame) * whole;
+ const auto fractional = (uint64_t(frame) / rate.num) * remainder + (uint64_t(frame) % rate.num) * remainder / rate.num;
+ return fractional > limit - integral ? int64_t(limit) : int64_t(integral + fractional);
+ }
+
+ size_t VideoSampling::source_index(const std::vector<int64_t>& timestamps, size_t frame) const {
+ return size_t(std::upper_bound(timestamps.begin(), timestamps.end(), time_ns(frame)) - timestamps.begin() - 1);
+ }
+
+ bool video_has_fixed_rate(const std::vector<int64_t>& timestamps, int64_t end_ns, VideoFrameRate rate, int64_t tolerance_ns) {
+ if (rate.num < 1 || rate.num > 1000000 || rate.den < 1 || rate.den > 1000000 || timestamps.empty() ||
+ tolerance_ns < 0 || end_ns < 0 || timestamps.front() < 0 || !std::is_sorted(timestamps.begin(), timestamps.end()))
+ return false;
+ VideoSampling sampling{rate};
+ for (size_t i = 0; i < timestamps.size(); ++i)
+ if (std::abs(timestamps[i] - sampling.time_ns(i)) > tolerance_ns)
+ return false;
+ return std::abs(end_ns - sampling.time_ns(timestamps.size())) <= tolerance_ns;
+ }
+
+ VideoSampling plan_video_sampling(const std::vector<int64_t>& timestamps, int64_t end_ns, VideoFrameRate declared_rate, int64_t tolerance_ns, int64_t fps_num, int64_t fps_den, size_t max_frames, size_t frame_bytes) {
+ if (timestamps.empty() || timestamps.front() != 0 || !std::is_sorted(timestamps.begin(), timestamps.end()) ||
+ end_ns < timestamps.back() || fps_num < 0 || fps_num > 1000000 || fps_den < 1 || fps_den > 1000000)
+ throw std::runtime_error("invalid video timestamps or frame rate");
+ const size_t capacity = frame_bytes ? (256 * 1024 * 1024) / frame_bytes : 0;
+ if (!capacity)
+ throw std::runtime_error("decoded video exceeds 256 MiB");
+ if (!fps_num) {
+ if (!video_has_fixed_rate(timestamps, end_ns, declared_rate, tolerance_ns)) {
+ const auto period = timestamps.size() > 1 ? timestamps[1] : end_ns;
+ declared_rate = period > 0 ? video_frame_rate(1000000000.0 / period) : VideoFrameRate{};
+ }
+ if (!video_has_fixed_rate(timestamps, end_ns, declared_rate, tolerance_ns))
+ throw std::runtime_error("video frame rate is variable or unknown; specify fps_num and fps_den");
+ fps_num = declared_rate.num;
+ fps_den = declared_rate.den;
+ }
+ const auto divisor = std::gcd(fps_num, fps_den);
+ VideoSampling sampling{{fps_num / divisor, fps_den / divisor}};
+ if (!end_ns && timestamps.size() == 1) {
+ sampling.count = 1;
+ return sampling;
+ }
+ if (end_ns <= 0)
+ throw std::runtime_error("video duration must be positive");
+ size_t first = 0, last = std::min(max_frames ? max_frames : capacity + 1, capacity + 1);
+ while (first < last) {
+ const auto middle = first + (last - first) / 2;
+ if (sampling.time_ns(middle) < end_ns)
+ first = middle + 1;
+ else
+ last = middle;
+ }
+ if (first > capacity)
+ throw std::runtime_error("decoded video exceeds 256 MiB; set max_frames");
+ sampling.count = first;
+ return sampling;
+ }
+}
diff --git a/src/dgl/ops/video_timing.h b/src/dgl/ops/video_timing.h
new file mode 100644
index 0000000..c7ec620
--- /dev/null
+++ b/src/dgl/ops/video_timing.h
@@ -0,0 +1,26 @@
+#ifndef __SD_DGL_OPS_VIDEO_TIMING_H__
+#define __SD_DGL_OPS_VIDEO_TIMING_H__
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace sd::dgl::ops {
+ struct VideoFrameRate {
+ int64_t num = 0;
+ int64_t den = 1;
+ };
+
+ struct VideoSampling {
+ VideoFrameRate rate;
+ size_t count = 0;
+ int64_t time_ns(size_t frame) const;
+ size_t source_index(const std::vector<int64_t>& timestamps, size_t frame) const;
+ };
+
+ VideoFrameRate video_frame_rate(double rate);
+ bool video_has_fixed_rate(const std::vector<int64_t>& timestamps, int64_t end_ns, VideoFrameRate rate, int64_t tolerance_ns);
+ VideoSampling plan_video_sampling(const std::vector<int64_t>& timestamps, int64_t end_ns, VideoFrameRate declared_rate, int64_t tolerance_ns, int64_t fps_num, int64_t fps_den, size_t max_frames, size_t frame_bytes);
+}
+
+#endif // __SD_DGL_OPS_VIDEO_TIMING_H__
diff --git a/src/dgl/ops/video_webm.cpp b/src/dgl/ops/video_webm.cpp
new file mode 100644
index 0000000..cf31446
--- /dev/null
+++ b/src/dgl/ops/video_webm.cpp
@@ -0,0 +1,205 @@
+#include "dgl/ops/video_webm.h"
+
+#include <cmath>
+#include <cstring>
+#include <fstream>
+#include <limits>
+
+#include "dgl/ops/tensor.h"
+#include "dgl/ops/video_timing.h"
+
+#ifdef SD_USE_WEBM
+#include <mkvparser/mkvparser.h>
+#include <webp/decode.h>
+
+namespace sd::dgl::ops {
+ class WebMReader : public mkvparser::IMkvReader {
+ std::ifstream file_;
+ long long size_;
+
+ public:
+ explicit WebMReader(const std::filesystem::path& path)
+ : file_(path, std::ios::binary | std::ios::ate) {
+ if (!file_ || file_.tellg() <= 0)
+ throw std::runtime_error("cannot read WebM input");
+ size_ = static_cast<long long>(file_.tellg());
+ }
+ int Read(long long position, long length, unsigned char* buffer) override {
+ if (position < 0 || length < 0 || position > size_ || length > size_ - position)
+ return -1;
+ file_.clear();
+ file_.seekg(position);
+ return !length || file_.read(reinterpret_cast<char*>(buffer), length) ? 0 : -1;
+ }
+ int Length(long long* total, long long* available) override {
+ *total = *available = size_;
+ return 0;
+ }
+ std::vector<uint8_t> packet(long long position, size_t length) {
+ if (length > 256 * 1024 * 1024 || length > size_t(std::numeric_limits<long>::max()))
+ throw std::runtime_error("WebM packet exceeds 256 MiB");
+ std::vector<uint8_t> bytes(length);
+ if (Read(position, long(length), bytes.data()))
+ throw std::runtime_error("truncated WebM packet");
+ return bytes;
+ }
+ };
+
+ struct WebMPacket {
+ long long position;
+ long size;
+ };
+
+ VideoReadResult load_webm(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio) {
+ WebMReader reader(path);
+ mkvparser::EBMLHeader header;
+ long long position = 0;
+ if (header.Parse(&reader, position) || !header.m_docType ||
+ (std::strcmp(header.m_docType, "webm") && std::strcmp(header.m_docType, "matroska")))
+ throw std::runtime_error("invalid WebM header");
+ mkvparser::Segment* raw = nullptr;
+ const auto status = mkvparser::Segment::CreateInstance(&reader, position, raw);
+ std::unique_ptr<mkvparser::Segment> segment(raw);
+ if (status || !segment || segment->Load() < 0 || !segment->GetTracks() || !segment->GetInfo())
+ throw std::runtime_error("cannot parse WebM segment");
+ const auto scale = segment->GetInfo()->GetTimeCodeScale();
+ if (scale < 1)
+ throw std::runtime_error("invalid WebM time scale");
+ const mkvparser::VideoTrack* video = nullptr;
+ const mkvparser::AudioTrack* audio = nullptr;
+ const auto* tracks = segment->GetTracks();
+ for (unsigned long i = 0; i < tracks->GetTracksCount(); ++i) {
+ const auto* track = tracks->GetTrackByIndex(i);
+ if (track && track->GetType() == mkvparser::Track::kVideo && !video)
+ video = static_cast<const mkvparser::VideoTrack*>(track);
+ if (track && track->GetType() == mkvparser::Track::kAudio && !audio && include_audio)
+ audio = static_cast<const mkvparser::AudioTrack*>(track);
+ }
+ if (!video || !video->GetCodecId() || std::strcmp(video->GetCodecId(), "V_VP8"))
+ throw std::runtime_error("WebM input supports independent VP8 keyframes only");
+ const auto width = video->GetWidth(), height = video->GetHeight();
+ if (width < 1 || width > 16384 || height < 1 || height > 16384)
+ throw std::runtime_error("invalid WebM video dimensions");
+ const size_t frame_bytes = size_t(width) * size_t(height) * 3;
+ int channels = 0, rate = 0;
+ if (audio) {
+ const auto sample_rate = audio->GetSamplingRate();
+ if (!audio->GetCodecId() || std::strcmp(audio->GetCodecId(), "A_PCM/INT/LIT") || audio->GetBitDepth() != 16 ||
+ audio->GetChannels() < 1 || audio->GetChannels() > 32 || !std::isfinite(sample_rate) ||
+ sample_rate < 1000 || sample_rate > 384000 || sample_rate != std::floor(sample_rate) || audio->GetCodecDelay())
+ throw std::runtime_error("WebM audio requires PCM16 without codec delay; use video::load(audio=false) to skip unsupported audio");
+ channels = int(audio->GetChannels());
+ rate = int(sample_rate);
+ }
+ std::vector<WebMPacket> frames, packets;
+ std::vector<int64_t> timestamps;
+ int64_t last_duration = 0;
+ for (auto* cluster = segment->GetFirst(); cluster && !cluster->EOS(); cluster = segment->GetNext(cluster)) {
+ const mkvparser::BlockEntry* entry = nullptr;
+ if (cluster->GetFirst(entry) < 0)
+ throw std::runtime_error("invalid WebM cluster");
+ while (entry && !entry->EOS()) {
+ const auto* block = entry->GetBlock();
+ const auto track = block->GetTrackNumber();
+ if (track == video->GetNumber() || (audio && track == audio->GetNumber())) {
+ const bool visual = track == video->GetNumber();
+ if (block->GetFrameCount() != 1 || (visual && (!block->IsKey() || block->IsInvisible())))
+ throw std::runtime_error(visual ? "WebM input requires one independent VP8 keyframe per block" : "laced WebM audio is unsupported; use video::load(audio=false)");
+ const auto& frame = block->GetFrame(0);
+ const auto timestamp = block->GetTime(cluster);
+ if (frame.pos < 0 || frame.len <= 0 || timestamp < 0)
+ throw std::runtime_error("invalid WebM packet");
+ if (visual) {
+ frames.push_back({frame.pos, frame.len});
+ timestamps.push_back(timestamp);
+ last_duration = 0;
+ if (entry->GetKind() == mkvparser::BlockEntry::kBlockGroup) {
+ const auto duration = static_cast<const mkvparser::BlockGroup*>(entry)->GetDurationTimeCode();
+ if (duration > std::numeric_limits<int64_t>::max() / scale)
+ throw std::runtime_error("WebM frame duration exceeds int64 range");
+ if (duration > 0)
+ last_duration = duration * scale;
+ }
+ } else {
+ packets.push_back({frame.pos, frame.len});
+ }
+ }
+ if (cluster->GetNext(entry, entry) < 0)
+ throw std::runtime_error("invalid WebM block sequence");
+ }
+ }
+ if (frames.empty())
+ throw std::runtime_error("WebM contains no video frames");
+ auto declared = video_frame_rate(video->GetFrameRate());
+ if (!declared.num && video->GetDefaultDuration())
+ declared = video_frame_rate(1000000000.0 / video->GetDefaultDuration());
+ int64_t end = segment->GetDuration();
+ if (last_duration > 0) {
+ if (timestamps.back() > std::numeric_limits<int64_t>::max() - last_duration)
+ throw std::runtime_error("WebM duration exceeds int64 range");
+ end = timestamps.back() + last_duration;
+ } else if (declared.num) {
+ const VideoSampling timing{declared};
+ end = timing.time_ns(frames.size());
+ if (!video_has_fixed_rate(timestamps, end, declared, scale)) {
+ if (timestamps.back() > std::numeric_limits<int64_t>::max() - timing.time_ns(1))
+ throw std::runtime_error("WebM duration exceeds int64 range");
+ end = timestamps.back() + timing.time_ns(1);
+ }
+ }
+ const auto sampling = plan_video_sampling(timestamps, end, declared, scale, fps_num, fps_den, max_frames, frame_bytes);
+ VideoReadResult result;
+ result.frames = make_tensor(SD_TENSOR_U8, std::vector<int64_t>{3, width, height, int64_t(sampling.count)});
+ result.fps_num = sampling.rate.num;
+ result.fps_den = sampling.rate.den;
+ auto* output = static_cast<uint8_t*>(result.frames->mutable_data());
+ size_t previous = frames.size();
+ for (size_t i = 0; i < sampling.count; ++i) {
+ const auto source = sampling.source_index(timestamps, i);
+ auto* destination = output + i * frame_bytes;
+ if (source == previous) {
+ std::memcpy(destination, destination - frame_bytes, frame_bytes);
+ continue;
+ }
+ auto bytes = reader.packet(frames[source].position, size_t(frames[source].size));
+ int actual_width = 0, actual_height = 0;
+ if (bytes.size() < 10 || (bytes[0] & 1) || bytes[3] != 0x9d || bytes[4] != 0x01 || bytes[5] != 0x2a ||
+ !WebPGetInfo(bytes.data(), bytes.size(), &actual_width, &actual_height) || actual_width != width || actual_height != height ||
+ !WebPDecodeRGBInto(bytes.data(), bytes.size(), destination, frame_bytes, int(width * 3)))
+ throw std::runtime_error("invalid or dependent VP8 frame in WebM input");
+ previous = source;
+ }
+ if (audio && !packets.empty()) {
+ const size_t block_bytes = size_t(channels) * 2;
+ uint64_t total = 0;
+ for (const auto& packet : packets) {
+ if (size_t(packet.size) % block_bytes)
+ throw std::runtime_error("invalid WebM PCM16 packet size; use video::load(audio=false)");
+ total += uint64_t(packet.size) / block_bytes;
+ }
+ const auto requested = std::ceil(static_cast<long double>(sampling.count) * sampling.rate.den * rate / sampling.rate.num);
+ const auto count = std::min(static_cast<long double>(total), requested);
+ if (count > (256 * 1024 * 1024) / (size_t(channels) * sizeof(float)))
+ throw std::runtime_error("decoded video audio exceeds 256 MiB; set max_frames");
+ const size_t wanted = size_t(count) * block_bytes;
+ std::vector<uint8_t> data;
+ data.reserve(wanted);
+ for (const auto& packet : packets) {
+ if (data.size() == wanted)
+ break;
+ const auto bytes = reader.packet(packet.position, std::min(size_t(packet.size), wanted - data.size()));
+ data.insert(data.end(), bytes.begin(), bytes.end());
+ }
+ if (!data.empty())
+ result.audio = decode_pcm_audio(data.data(), data.size(), 1, 16, channels, rate);
+ }
+ return result;
+ }
+}
+#else
+namespace sd::dgl::ops {
+ VideoReadResult load_webm(const std::filesystem::path&, size_t, int64_t, int64_t, bool) {
+ throw std::runtime_error("WebM video input requires a build with SD_WEBM=ON");
+ }
+}
+#endif
diff --git a/src/dgl/ops/video_webm.h b/src/dgl/ops/video_webm.h
new file mode 100644
index 0000000..ed2863a
--- /dev/null
+++ b/src/dgl/ops/video_webm.h
@@ -0,0 +1,10 @@
+#ifndef __SD_DGL_OPS_VIDEO_WEBM_H__
+#define __SD_DGL_OPS_VIDEO_WEBM_H__
+
+#include "dgl/ops/video.h"
+
+namespace sd::dgl::ops {
+ VideoReadResult load_webm(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den, bool include_audio);
+}
+
+#endif // __SD_DGL_OPS_VIDEO_WEBM_H__
diff --git a/src/dgl/ops/video_webp.cpp b/src/dgl/ops/video_webp.cpp
new file mode 100644
index 0000000..aebbd77
--- /dev/null
+++ b/src/dgl/ops/video_webp.cpp
@@ -0,0 +1,97 @@
+#include "dgl/ops/video_webp.h"
+
+#include <cstring>
+#include <fstream>
+#include <limits>
+
+#include "dgl/ops/tensor.h"
+#include "dgl/ops/video_timing.h"
+
+#ifdef SD_USE_WEBP
+#include <webp/demux.h>
+
+namespace sd::dgl::ops {
+ VideoReadResult load_webp(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den) {
+ std::ifstream file(path, std::ios::binary | std::ios::ate);
+ if (!file || file.tellg() <= 0)
+ throw std::runtime_error("cannot read WebP input");
+ const auto length = uint64_t(file.tellg());
+ if (length > std::numeric_limits<size_t>::max() || length > uint64_t(std::numeric_limits<std::streamsize>::max()))
+ throw std::runtime_error("WebP input is too large");
+ std::vector<uint8_t> bytes(size_t(length), 0);
+ file.seekg(0);
+ if (!file.read(reinterpret_cast<char*>(bytes.data()), std::streamsize(bytes.size())))
+ throw std::runtime_error("truncated WebP input");
+ WebPBitstreamFeatures features{};
+ if (WebPGetFeatures(bytes.data(), bytes.size(), &features) != VP8_STATUS_OK ||
+ features.width < 1 || features.width > 16384 || features.height < 1 || features.height > 16384)
+ throw std::runtime_error("invalid WebP dimensions or bitstream");
+ if (size_t(features.width) * features.height * 3 > 256 * 1024 * 1024)
+ throw std::runtime_error("decoded video exceeds 256 MiB");
+ WebPData data{bytes.data(), bytes.size()};
+ WebPAnimDecoderOptions options;
+ if (!WebPAnimDecoderOptionsInit(&options))
+ throw std::runtime_error("cannot initialize WebP animation decoder");
+ options.color_mode = MODE_RGBA;
+ std::unique_ptr<WebPAnimDecoder, decltype(&WebPAnimDecoderDelete)> decoder(WebPAnimDecoderNew(&data, &options), WebPAnimDecoderDelete);
+ WebPAnimInfo info{};
+ if (!decoder || !WebPAnimDecoderGetInfo(decoder.get(), &info) || !info.frame_count ||
+ info.frame_count > uint32_t(std::numeric_limits<int>::max()) ||
+ info.canvas_width < 1 || info.canvas_width > 16384 || info.canvas_height < 1 || info.canvas_height > 16384)
+ throw std::runtime_error("invalid WebP animation or canvas");
+ const auto* demux = WebPAnimDecoderGetDemuxer(decoder.get());
+ std::vector<int64_t> timestamps;
+ timestamps.reserve(info.frame_count);
+ int64_t end_ms = 0;
+ for (uint32_t i = 0; i < info.frame_count; ++i) {
+ WebPIterator frame{};
+ if (!WebPDemuxGetFrame(demux, int(i + 1), &frame))
+ throw std::runtime_error("cannot read WebP frame timing");
+ const int duration = frame.duration;
+ const bool complete = frame.complete;
+ WebPDemuxReleaseIterator(&frame);
+ if (!complete || duration < 0 || end_ms > std::numeric_limits<int>::max() - duration)
+ throw std::runtime_error("incomplete WebP frame or animation duration exceeds decoder limits");
+ timestamps.push_back(end_ms * 1000000);
+ end_ms += duration;
+ }
+ const size_t frame_bytes = size_t(info.canvas_width) * info.canvas_height * 3;
+ const auto sampling = plan_video_sampling(timestamps, end_ms * 1000000, {}, 0, fps_num, fps_den, max_frames, frame_bytes);
+ VideoReadResult result;
+ result.frames = make_tensor(SD_TENSOR_U8, std::vector<int64_t>{3, info.canvas_width, info.canvas_height, int64_t(sampling.count)});
+ result.fps_num = sampling.rate.num;
+ result.fps_den = sampling.rate.den;
+ auto* output = static_cast<uint8_t*>(result.frames->mutable_data());
+ uint8_t* rgba = nullptr;
+ size_t decoded = 0, previous = info.frame_count;
+ for (size_t i = 0; i < sampling.count; ++i) {
+ const auto source = sampling.source_index(timestamps, i);
+ auto* destination = output + i * frame_bytes;
+ if (source == previous) {
+ std::memcpy(destination, destination - frame_bytes, frame_bytes);
+ continue;
+ }
+ while (decoded <= source) {
+ int timestamp = 0;
+ if (!WebPAnimDecoderGetNext(decoder.get(), &rgba, ×tamp))
+ throw std::runtime_error("WebP frame decoding failed");
+ const auto expected = decoded + 1 < timestamps.size() ? timestamps[decoded + 1] / 1000000 : end_ms;
+ if (timestamp != expected)
+ throw std::runtime_error("WebP decoder timestamps do not match frame metadata");
+ ++decoded;
+ }
+ for (size_t pixel = 0; pixel < frame_bytes / 3; ++pixel)
+ for (size_t channel = 0; channel < 3; ++channel)
+ destination[pixel * 3 + channel] = uint8_t((uint32_t(rgba[pixel * 4 + channel]) * rgba[pixel * 4 + 3] + 127) / 255);
+ previous = source;
+ }
+ return result;
+ }
+}
+#else
+namespace sd::dgl::ops {
+ VideoReadResult load_webp(const std::filesystem::path&, size_t, int64_t, int64_t) {
+ throw std::runtime_error("WebP video input requires a build with SD_WEBP=ON");
+ }
+}
+#endif
diff --git a/src/dgl/ops/video_webp.h b/src/dgl/ops/video_webp.h
new file mode 100644
index 0000000..5eb6d35
--- /dev/null
+++ b/src/dgl/ops/video_webp.h
@@ -0,0 +1,10 @@
+#ifndef __SD_DGL_OPS_VIDEO_WEBP_H__
+#define __SD_DGL_OPS_VIDEO_WEBP_H__
+
+#include "dgl/ops/video.h"
+
+namespace sd::dgl::ops {
+ VideoReadResult load_webp(const std::filesystem::path& path, size_t max_frames, int64_t fps_num, int64_t fps_den);
+}
+
+#endif // __SD_DGL_OPS_VIDEO_WEBP_H__
diff --git a/src/dgl/ops/video_windows.cpp b/src/dgl/ops/video_windows.cpp
new file mode 100644
index 0000000..f2d72d4
--- /dev/null
+++ b/src/dgl/ops/video_windows.cpp
@@ -0,0 +1,64 @@
+#include "dgl/ops/video_windows.h"
+
+#include <algorithm>
+#include <cstring>
+#include "dgl/ops/tensor.h"
+#include "dgl/ops/video.h"
+
+namespace sd::dgl::ops {
+ std::vector<std::shared_ptr<sd_tensor_t>> split_video_frames(const std::shared_ptr<sd_tensor_t>& frames, size_t size, size_t overlap) {
+ if (!frames)
+ throw std::runtime_error("video frame tensor is null");
+ validate_video_frames(*frames);
+ if (!size || overlap >= size)
+ throw std::runtime_error("video overlap must be smaller than a positive window size");
+ const auto count = size_t(frames->shape()[3]);
+ if (size >= count)
+ return {frames};
+ std::vector<std::shared_ptr<sd_tensor_t>> windows;
+ for (size_t start = 0; start < count;) {
+ const auto end = std::min(start + size, count);
+ windows.push_back(crop_tensor(*frames, {3}, {int64_t(start)}, {int64_t(end - start)}));
+ if (end == count)
+ break;
+ start = end - overlap;
+ }
+ return windows;
+ }
+
+ std::shared_ptr<sd_tensor_t> join_video_windows(const std::vector<std::shared_ptr<sd_tensor_t>>& windows, size_t overlap) {
+ if (windows.empty())
+ throw std::runtime_error("video::join_windows requires at least one window");
+ constexpr uint64_t byte_limit = uint64_t(4) * 1024 * 1024 * 1024;
+ size_t count = 0, stride = 0;
+ std::vector<int64_t> shape;
+ for (size_t i = 0; i < windows.size(); ++i) {
+ if (!windows[i])
+ throw std::runtime_error("video frame tensor is null");
+ validate_video_frames(*windows[i]);
+ const auto& current = windows[i]->shape();
+ if (!i) {
+ shape = current;
+ stride = size_t(current[0] * current[1] * current[2]);
+ } else if (!std::equal(shape.begin(), shape.begin() + 3, current.begin()) || size_t(current[3]) <= overlap)
+ throw std::runtime_error("video windows have incompatible sizes or overlap");
+ const auto retained = size_t(current[3]) - (i ? overlap : 0);
+ const auto limit = byte_limit / stride;
+ if (retained > limit || count > limit - retained)
+ throw std::runtime_error("joined video exceeds 4 GiB");
+ count += retained;
+ }
+ if (windows.size() == 1)
+ return windows.front();
+ shape[3] = int64_t(count);
+ auto output = make_tensor(SD_TENSOR_U8, std::move(shape));
+ auto* destination = static_cast<uint8_t*>(output->mutable_data());
+ for (size_t i = 0; i < windows.size(); ++i) {
+ const auto skip = (i ? overlap : 0) * stride;
+ const auto bytes = windows[i]->bytes - skip;
+ std::memcpy(destination, static_cast<const uint8_t*>(windows[i]->data()) + skip, bytes);
+ destination += bytes;
+ }
+ return output;
+ }
+}
diff --git a/src/dgl/ops/video_windows.h b/src/dgl/ops/video_windows.h
new file mode 100644
index 0000000..e05f996
--- /dev/null
+++ b/src/dgl/ops/video_windows.h
@@ -0,0 +1,11 @@
+#ifndef __SD_DGL_OPS_VIDEO_WINDOWS_H__
+#define __SD_DGL_OPS_VIDEO_WINDOWS_H__
+
+#include "dgl/tensor.h"
+
+namespace sd::dgl::ops {
+ std::vector<std::shared_ptr<sd_tensor_t>> split_video_frames(const std::shared_ptr<sd_tensor_t>& frames, size_t size, size_t overlap);
+ std::shared_ptr<sd_tensor_t> join_video_windows(const std::vector<std::shared_ptr<sd_tensor_t>>& windows, size_t overlap);
+}
+
+#endif // __SD_DGL_OPS_VIDEO_WINDOWS_H__
diff --git a/src/dgl/registry.cpp b/src/dgl/registry.cpp
new file mode 100644
index 0000000..16f74ac
--- /dev/null
+++ b/src/dgl/registry.cpp
@@ -0,0 +1,173 @@
+#include "registry.h"
+
+#include <algorithm>
+#include <cmath>
+#include <sstream>
+
+#include "dgl/bindings/registration.h"
+#include "model_definition.h"
+#include "stable-diffusion.h"
+
+namespace sd::dgl {
+
+ void Registry::alias(const std::string& name, const std::string& target) {
+ if (count(name))
+ throw std::runtime_error("function '" + name + "' is already registered");
+ const auto found = find(target);
+ if (found == end())
+ throw std::runtime_error("unknown alias target '" + target + "'");
+ aliases_.emplace(name, found->first);
+ }
+
+ Operation& Registry::operator[](const std::string& name) {
+ if (aliases_.count(name))
+ throw std::runtime_error("function '" + name + "' is registered as an alias");
+ return operations_[name];
+ }
+
+ Operation make_operation(std::vector<Argument> arguments, OutputSignature output, std::function<Value(Values&, RunState&)> execute) {
+ return {std::move(arguments), std::move(output), std::move(execute)};
+ }
+
+ Operation make_resource_operation(std::vector<Argument> arguments, OutputSignature output, std::function<PreparedResource(Values&)> prepare) {
+ Operation operation{std::move(arguments), std::move(output)};
+ operation.prepare = std::move(prepare);
+ return operation;
+ }
+
+ Argument required(const std::string& name, Kind kind) {
+ return {name, {kind}};
+ }
+
+ static std::optional<Value> literal_default(const nlohmann::json& literal) {
+ if (literal.is_null())
+ return std::nullopt;
+ return Value::literal(literal);
+ }
+
+ Argument optional(const std::string& name, Kind kind, const nlohmann::json& default_literal) {
+ return {name, {kind}, false, literal_default(default_literal)};
+ }
+ Argument integer(const std::string& name, int64_t low, int64_t high, const nlohmann::json& default_literal) {
+ return {name, {Kind::Integer}, default_literal.is_null(), literal_default(default_literal), [=](const Value& v) {
+ auto n = v.scalar.get<int64_t>();
+ if (n < low || n > high)
+ throw std::runtime_error(name + " must be in [" + std::to_string(low) + ", " + std::to_string(high) + "]");
+ }};
+ }
+ Argument number(const std::string& name, double low, double high, const nlohmann::json& default_literal) {
+ return {name, {Kind::Number}, default_literal.is_null(), literal_default(default_literal), [=](const Value& v) {
+ double n = v.scalar.get<double>();
+ if (!std::isfinite(n) || n < low || n > high)
+ throw std::runtime_error(name + " is out of range");
+ }};
+ }
+ Argument choice(const std::string& name, std::vector<std::string> choices, const nlohmann::json& default_literal) {
+ return {name, {Kind::String}, default_literal.is_null(), literal_default(default_literal), [=](const Value& v) {
+ auto s = v.scalar.get<std::string>();
+ if (std::find(choices.begin(), choices.end(), s) == choices.end())
+ throw std::runtime_error("unknown " + name + " '" + s + "'");
+ }};
+ }
+ Argument list_argument(const std::string& name, Kind element) {
+ return {name, {Kind::List, element}};
+ }
+ Argument model_path(const std::string& name, bool required) {
+ return required ? sd::dgl::required(name, Kind::String) : optional(name, Kind::String, "");
+ }
+ const Registry& operations() {
+ static const Registry registry = [] {
+ Registry result;
+ register_builtin_operations(result);
+ return result;
+ }();
+ return registry;
+ }
+
+ static void write_default(std::ostream& out, const Value& value) {
+ if (value.type().kind == Kind::List) {
+ out << '[';
+ const char* separator = "";
+ for (const auto& item : value.items) {
+ out << separator;
+ write_default(out, item);
+ separator = ", ";
+ }
+ out << ']';
+ } else if (value.type().kind == Kind::Dict) {
+ out << '{';
+ const char* separator = "";
+ for (const auto& [name, field] : value.fields) {
+ out << separator << name << " = ";
+ write_default(out, field);
+ separator = ", ";
+ }
+ out << '}';
+ } else {
+ out << (value.scalar.is_null() ? type_name(value.type()) : value.scalar.dump());
+ }
+ }
+
+ static void write_output_fields(std::ostream& out, const OutputSignature& signature, const std::string& prefix = "") {
+ for (const auto& [name, field] : signature.fields) {
+ const auto path = prefix + name;
+ out << "<br>`" << path << ": " << type_name(field.type) << "`";
+ write_output_fields(out, field, path + ".");
+ }
+ }
+
+ std::string builtin_catalog() {
+ std::ostringstream out;
+ out << "<!-- Generated by sd-cli -M dgl. Update with the dgl-update-builtins CMake target. -->\n"
+ "# Built-in DGL functions\n\n"
+ "Optional parameters are marked with `?`. Default values follow `=`. "
+ "See [DGL syntax](dgl.md#dgl-syntax) for `fn`, `map` and `fold`.\n\n"
+ "| Function | Parameters | Returns |\n"
+ "|---|---|---|\n";
+ const auto& registry = operations();
+ std::map<std::string, const Operation*> functions;
+ for (const auto& [name, operation] : registry)
+ functions.emplace(name, &operation);
+ for (const auto& [name, target] : registry.aliases())
+ functions.emplace(name, ®istry.at(target));
+ for (const auto& [name, definition] : functions) {
+ const auto& operation = *definition;
+ out << "| `" << name << "`";
+ if (operation.resource())
+ out << " (resource)";
+ if (const auto alias = registry.aliases().find(name); alias != registry.aliases().end())
+ out << " (alias of `" << alias->second << "`)";
+ out << " | ";
+ bool first = true;
+ for (const auto& argument : operation.arguments) {
+ if (!first)
+ out << ", ";
+ first = false;
+ out << '`' << argument.name << (argument.required ? "" : "?") << ": " << type_name(argument.type);
+ if (argument.default_value) {
+ out << " = ";
+ write_default(out, *argument.default_value);
+ }
+ out << '`';
+ }
+ out << " | ";
+ if (operation.output.same_type_as.empty())
+ out << '`' << type_name(operation.output.type) << '`';
+ else
+ out << "same type as `" << operation.output.same_type_as << '`';
+ write_output_fields(out, operation.output);
+ out << " |\n";
+ }
+ out << "\n## Model family IDs\n\n"
+ "Pass the ID as the component loader's `family` argument. "
+ "The default is `\"auto\"`.\n\n"
+ "| ID | Model family |\n"
+ "|---|---|\n";
+ for (int version = 0; version < VERSION_COUNT; ++version) {
+ const auto* definition = find_model_definition(static_cast<SDVersion>(version));
+ out << "| `" << definition->id << "` | " << definition->name << " |\n";
+ }
+ return out.str();
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/registry.h b/src/dgl/registry.h
new file mode 100644
index 0000000..ffac38c
--- /dev/null
+++ b/src/dgl/registry.h
@@ -0,0 +1,107 @@
+#ifndef __SD_DGL_REGISTRY_H__
+#define __SD_DGL_REGISTRY_H__
+
+#include <functional>
+#include <map>
+#include <optional>
+#include <utility>
+
+#include "value.h"
+
+namespace sd::dgl {
+
+ struct CallbackSignature {
+ std::string args_parameter;
+ std::map<std::string, Type> input_fields;
+ std::map<std::string, Type> output_fields;
+ };
+
+ struct Argument {
+ std::string name;
+ Type type;
+ bool required = true;
+ std::optional<Value> default_value;
+ std::function<void(const Value&)> validate;
+ std::optional<CallbackSignature> callback;
+ };
+
+ struct RunState;
+ struct ResourceLoadRequest;
+
+ struct PreparedResource {
+ nlohmann::json key;
+ nlohmann::json revision;
+ std::function<Value(ResourceLoadRequest&)> load;
+ };
+
+ struct OutputSignature {
+ Type type;
+ std::map<std::string, OutputSignature> fields;
+ std::string same_type_as;
+
+ OutputSignature(Kind kind = Kind::Any, Kind element = Kind::Any)
+ : type{kind, element} {}
+ OutputSignature(Type type, std::map<std::string, OutputSignature> fields = {})
+ : type(type), fields(std::move(fields)) {}
+ };
+
+ struct Operation {
+ enum class ModelUse { Automatic,
+ Metadata,
+ Capture,
+ Preview };
+ std::vector<Argument> arguments;
+ OutputSignature output;
+ std::function<Value(Values&, RunState&)> execute;
+ bool sampling = false;
+ std::function<PreparedResource(Values&)> prepare;
+ ModelUse model_use = ModelUse::Automatic;
+
+ bool resource() const { return bool(prepare); }
+ };
+
+ class Registry {
+ public:
+ using Operations = std::map<std::string, Operation>;
+ using Entry = Operations::value_type;
+
+ private:
+ Operations operations_;
+ std::map<std::string, std::string> aliases_;
+
+ const std::string& canonical_name(const std::string& name) const {
+ const auto found = aliases_.find(name);
+ return found == aliases_.end() ? name : found->second;
+ }
+
+ public:
+ void alias(const std::string& name, const std::string& target);
+ Operation& operator[](const std::string& name);
+ Operation& at(const std::string& name) { return operations_.at(canonical_name(name)); }
+ const Operation& at(const std::string& name) const { return operations_.at(canonical_name(name)); }
+ auto find(const std::string& name) { return operations_.find(canonical_name(name)); }
+ auto find(const std::string& name) const { return operations_.find(canonical_name(name)); }
+ auto count(const std::string& name) const { return operations_.count(canonical_name(name)); }
+ auto begin() { return operations_.begin(); }
+ auto begin() const { return operations_.begin(); }
+ auto end() { return operations_.end(); }
+ auto end() const { return operations_.end(); }
+ const auto& aliases() const { return aliases_; }
+ };
+
+ Operation make_operation(std::vector<Argument> arguments, OutputSignature output, std::function<Value(Values&, RunState&)> execute);
+ Operation make_resource_operation(std::vector<Argument> arguments, OutputSignature output, std::function<PreparedResource(Values&)> prepare);
+
+ Argument required(const std::string& name, Kind kind);
+ Argument optional(const std::string& name, Kind kind, const nlohmann::json& default_literal = nullptr);
+ Argument integer(const std::string& name, int64_t low, int64_t high, const nlohmann::json& default_literal = nullptr);
+ Argument number(const std::string& name, double low, double high, const nlohmann::json& default_literal = nullptr);
+ Argument choice(const std::string& name, std::vector<std::string> choices, const nlohmann::json& default_literal = nullptr);
+ Argument list_argument(const std::string& name, Kind element);
+ Argument model_path(const std::string& name, bool required = true);
+ const Registry& operations();
+ std::string builtin_catalog();
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_REGISTRY_H__
diff --git a/src/dgl/result.cpp b/src/dgl/result.cpp
new file mode 100644
index 0000000..3e0816c
--- /dev/null
+++ b/src/dgl/result.cpp
@@ -0,0 +1,235 @@
+#include "result.h"
+
+#include <algorithm>
+
+#include "api/api.h"
+#include "dgl/tensor.h"
+
+namespace {
+ using sd::dgl::Kind;
+ using sd::dgl::make_tensor;
+ using sd::dgl::Value;
+
+ sd_dgl_value_t scalar(sd_dgl_value_kind_t kind, nlohmann::json value, std::string name = {}) {
+ return {kind, std::move(name), std::move(value), {}, {}};
+ }
+
+ sd_dgl_value_t tensor(std::shared_ptr<sd_tensor_t> handle, std::string name = {}) {
+ return {SD_DGL_VALUE_TENSOR, std::move(name), {}, std::move(handle), {}};
+ }
+
+ sd_dgl_value_t export_value(Value value, std::string name = {}) {
+ switch (value.type().kind) {
+ case Kind::Void:
+ return {SD_DGL_VALUE_VOID, std::move(name), {}, {}, {}};
+ case Kind::Boolean:
+ return scalar(SD_DGL_VALUE_BOOL, std::move(value.scalar), std::move(name));
+ case Kind::Integer:
+ return scalar(SD_DGL_VALUE_INT, std::move(value.scalar), std::move(name));
+ case Kind::Number:
+ return scalar(SD_DGL_VALUE_NUMBER, std::move(value.scalar), std::move(name));
+ case Kind::String:
+ return scalar(SD_DGL_VALUE_STRING, std::move(value.scalar), std::move(name));
+ case Kind::Tensor:
+ return tensor(value.as<sd_tensor_t>(), std::move(name));
+ case Kind::Bytes:
+ return {SD_DGL_VALUE_BYTES, std::move(name), {}, {}, {}, value.as<sd::dgl::Bytes>()};
+ case Kind::List: {
+ sd_dgl_value_t result{SD_DGL_VALUE_LIST, std::move(name), {}, {}, {}};
+ result.children.reserve(value.items.size());
+ for (auto& item : value.items)
+ result.children.push_back(export_value(std::move(item)));
+ return result;
+ }
+ case Kind::Dict: {
+ sd_dgl_value_t result{SD_DGL_VALUE_DICT, std::move(name), {}, {}, {}};
+ result.children.reserve(value.fields.size());
+ for (auto& [field, item] : value.fields)
+ result.children.push_back(export_value(std::move(item), field));
+ return result;
+ }
+ default:
+ return {SD_DGL_VALUE_OPAQUE, std::move(name), {}, {}, {}};
+ }
+ }
+
+ std::shared_ptr<sd_tensor_t> scalar_tensor(const sd_dgl_value_t& value) {
+ switch (value.kind) {
+ case SD_DGL_VALUE_BOOL: {
+ uint8_t boolean = value.scalar.get<bool>();
+ return make_tensor(SD_TENSOR_U8, std::vector<int64_t>{}, &boolean, sizeof(boolean));
+ }
+ case SD_DGL_VALUE_INT: {
+ auto number = value.scalar.get<int64_t>();
+ return make_tensor(SD_TENSOR_I64, std::vector<int64_t>{}, &number, sizeof(number));
+ }
+ case SD_DGL_VALUE_NUMBER: {
+ auto number = value.scalar.get<double>();
+ return make_tensor(SD_TENSOR_F64, std::vector<int64_t>{}, &number, sizeof(number));
+ }
+ case SD_DGL_VALUE_STRING: {
+ const auto& text = value.scalar.get_ref<const std::string&>();
+ return make_tensor(SD_TENSOR_U8, std::vector<int64_t>{int64_t(text.size())}, text.data(), text.size());
+ }
+ default:
+ return {};
+ }
+ }
+
+ void flatten(const sd_dgl_value_t& value, const std::string& path, sd_dgl_result_t::Tensors& tensors) {
+ if (value.kind == SD_DGL_VALUE_DICT || value.kind == SD_DGL_VALUE_LIST) {
+ for (size_t i = 0; i < value.children.size(); ++i) {
+ const auto& child = value.children[i];
+ auto name = value.kind == SD_DGL_VALUE_LIST ? std::to_string(i) : child.name;
+ flatten(child, path.empty() ? name : path + "." + name, tensors);
+ }
+ return;
+ }
+ const auto key = path.empty() ? "result" : path;
+ switch (value.kind) {
+ case SD_DGL_VALUE_TENSOR:
+ tensors.emplace(key, value.tensor);
+ break;
+ default:
+ if (auto data = scalar_tensor(value))
+ tensors.emplace(key, std::move(data));
+ break;
+ }
+ }
+
+ const sd_dgl_result_t::Tensors* flat_tensors(const sd_dgl_result_t* result) {
+ if (!result)
+ return nullptr;
+ const sd_dgl_result_t::Tensors* tensors = nullptr;
+ sd::api::call([&] {
+ tensors = &result->flat_tensors();
+ return SD_STATUS_OK;
+ });
+ return tensors;
+ }
+
+ sd_status_t require_kind(const sd_dgl_value_t* value, sd_dgl_value_kind_t kind, const void* out) {
+ sd::api::clear_error();
+ if (!value || !out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "value and output are required");
+ if (value->kind != kind)
+ return sd::api::error(SD_STATUS_TYPE_MISMATCH, nullptr, "DGL result value has a different type");
+ return SD_STATUS_OK;
+ }
+}
+
+sd_dgl_result_t::sd_dgl_result_t(Value value)
+ : type(sd::dgl::type_name(value.type())), root(export_value(std::move(value))) {}
+
+const sd_dgl_result_t::Tensors& sd_dgl_result_t::flat_tensors() const {
+ std::call_once(flat_init_, [&] {
+ Tensors tensors;
+ flatten(root, "", tensors);
+ flat_ = std::move(tensors);
+ });
+ return flat_;
+}
+
+void sd_dgl_result_free(sd_dgl_result_t* result) {
+ delete result;
+}
+
+const char* sd_dgl_result_type(const sd_dgl_result_t* result) {
+ return result ? result->type.c_str() : nullptr;
+}
+
+size_t sd_dgl_result_count(const sd_dgl_result_t* result) {
+ auto tensors = flat_tensors(result);
+ return tensors ? tensors->size() : 0;
+}
+
+const char* sd_dgl_result_name(const sd_dgl_result_t* result, size_t index) {
+ auto tensors = flat_tensors(result);
+ if (!tensors || index >= tensors->size())
+ return nullptr;
+ return std::next(tensors->begin(), index)->first.c_str();
+}
+
+const sd_tensor_t* sd_dgl_result_tensor(const sd_dgl_result_t* result, const char* name) {
+ auto tensors = name ? flat_tensors(result) : nullptr;
+ if (!tensors)
+ return nullptr;
+ auto it = tensors->find(name);
+ return it == tensors->end() ? nullptr : it->second.get();
+}
+
+const sd_dgl_value_t* sd_dgl_result_root(const sd_dgl_result_t* result) {
+ return result ? &result->root : nullptr;
+}
+
+sd_dgl_value_kind_t sd_dgl_value_kind(const sd_dgl_value_t* value) {
+ return value ? value->kind : SD_DGL_VALUE_INVALID;
+}
+
+size_t sd_dgl_value_count(const sd_dgl_value_t* value) {
+ return value ? value->children.size() : 0;
+}
+
+const sd_dgl_value_t* sd_dgl_value_at(const sd_dgl_value_t* value, size_t index) {
+ return value && index < value->children.size() ? &value->children[index] : nullptr;
+}
+
+const char* sd_dgl_value_name(const sd_dgl_value_t* value) {
+ return value && !value->name.empty() ? value->name.c_str() : nullptr;
+}
+
+const sd_dgl_value_t* sd_dgl_value_field(const sd_dgl_value_t* value, const char* name) {
+ if (!value || !name || !*name)
+ return nullptr;
+ const auto match = std::find_if(value->children.begin(), value->children.end(),
+ [&](const auto& child) { return child.name == name; });
+ return match == value->children.end() ? nullptr : &*match;
+}
+
+sd_status_t sd_dgl_value_get_bool(const sd_dgl_value_t* value, bool* out) {
+ const auto status = require_kind(value, SD_DGL_VALUE_BOOL, out);
+ if (status == SD_STATUS_OK)
+ *out = value->scalar.get<bool>();
+ return status;
+}
+
+sd_status_t sd_dgl_value_get_int(const sd_dgl_value_t* value, int64_t* out) {
+ const auto status = require_kind(value, SD_DGL_VALUE_INT, out);
+ if (status == SD_STATUS_OK)
+ *out = value->scalar.get<int64_t>();
+ return status;
+}
+
+sd_status_t sd_dgl_value_get_number(const sd_dgl_value_t* value, double* out) {
+ const auto status = require_kind(value, SD_DGL_VALUE_NUMBER, out);
+ if (status == SD_STATUS_OK)
+ *out = value->scalar.get<double>();
+ return status;
+}
+
+sd_status_t sd_dgl_value_get_string(const sd_dgl_value_t* value, const char** out, size_t* size) {
+ const auto status = require_kind(value, SD_DGL_VALUE_STRING, size ? out : nullptr);
+ if (status == SD_STATUS_OK) {
+ const auto& text = value->scalar.get_ref<const std::string&>();
+ *out = text.c_str();
+ *size = text.size();
+ }
+ return status;
+}
+
+const sd_tensor_t* sd_dgl_value_tensor(const sd_dgl_value_t* value) {
+ return value && value->kind == SD_DGL_VALUE_TENSOR ? value->tensor.get() : nullptr;
+}
+
+sd_status_t sd_dgl_value_get_bytes(const sd_dgl_value_t* value, const void** out, size_t* size) {
+ if (out)
+ *out = nullptr;
+ if (size)
+ *size = 0;
+ const auto status = require_kind(value, SD_DGL_VALUE_BYTES, size ? out : nullptr);
+ if (status == SD_STATUS_OK) {
+ *out = value->bytes->data.data();
+ *size = value->bytes->data.size();
+ }
+ return status;
+}
diff --git a/src/dgl/result.h b/src/dgl/result.h
new file mode 100644
index 0000000..c5cba84
--- /dev/null
+++ b/src/dgl/result.h
@@ -0,0 +1,32 @@
+#ifndef __SD_DGL_RESULT_H__
+#define __SD_DGL_RESULT_H__
+
+#include <mutex>
+
+#include "stable-diffusion.h"
+#include "value.h"
+
+struct sd_dgl_value_t {
+ sd_dgl_value_kind_t kind;
+ std::string name;
+ nlohmann::json scalar;
+ std::shared_ptr<sd_tensor_t> tensor;
+ std::vector<sd_dgl_value_t> children;
+ std::shared_ptr<const sd::dgl::Bytes> bytes;
+};
+
+struct sd_dgl_result_t {
+ using Tensors = std::map<std::string, std::shared_ptr<sd_tensor_t>, std::less<>>;
+
+ std::string type;
+ sd_dgl_value_t root;
+
+ explicit sd_dgl_result_t(sd::dgl::Value value);
+ const Tensors& flat_tensors() const;
+
+private:
+ mutable std::once_flag flat_init_;
+ mutable Tensors flat_;
+};
+
+#endif // __SD_DGL_RESULT_H__
diff --git a/src/dgl/run_events.cpp b/src/dgl/run_events.cpp
new file mode 100644
index 0000000..f38cd1b
--- /dev/null
+++ b/src/dgl/run_events.cpp
@@ -0,0 +1,101 @@
+#include "run_events.h"
+
+#include "diagnostic.h"
+
+namespace sd::dgl {
+ static thread_local RunEvents* active_events = nullptr;
+
+ RunEvents::RunEvents(const RunCallbacks& callbacks, bool legacy_callbacks)
+ : callbacks_(callbacks), legacy_progress_(legacy_callbacks) {
+ progress_.set_callback(callbacks_.progress, callbacks_.progress_data);
+ if (legacy_callbacks && !callbacks_.preview) {
+ legacy_preview_ = sd_get_preview_callback();
+ legacy_preview_data_ = sd_get_preview_callback_data();
+ callbacks_.mode = sd_get_preview_mode();
+ callbacks_.interval = sd_get_preview_interval();
+ callbacks_.denoised = sd_should_preview_denoised();
+ callbacks_.noisy = sd_should_preview_noisy();
+ }
+ }
+
+ void RunEvents::check_cancelled() const {
+ if (cancelled())
+ throw Error(SD_STATUS_CANCELLED, "DGL run cancelled");
+ }
+
+ void RunEvents::report(const char* stage, const char* detail, int64_t completed, int64_t total, const char* unit) const {
+ check_cancelled();
+ progress_.report(stage, detail, completed, total, unit);
+ check_cancelled();
+ }
+
+ void RunEvents::report_frames(const char* stage, int completed, int total) const {
+ report(stage, "", completed, total, "frames");
+ if (legacy_progress_ && sd_get_progress_callback())
+ sd_get_progress_callback()(completed, total, 0.f, sd_get_progress_callback_data());
+ check_cancelled();
+ }
+
+ void RunEvents::report_loading(int completed, int total, void* data) {
+ const auto& run = *static_cast<RunEvents*>(data);
+ // The loader monitors progress while its worker threads are still running.
+ if (!run.cancelled())
+ run.progress_.report("loading", "", completed, total, "tensors");
+ }
+
+ void RunEvents::begin_sample(int total_steps) {
+ check_cancelled();
+ ++preview_info_.sample_pass;
+ preview_info_.total_steps = total_steps;
+ if (legacy_preview_)
+ sd_begin_preview_pass(total_steps);
+ last_progress_us_ = ggml_time_us();
+ if (legacy_progress_ || !callbacks_.progress)
+ pretty_progress(0, total_steps, 0.f);
+ progress_.begin_sample(total_steps, "");
+ check_cancelled();
+ }
+
+ void RunEvents::sample_step(int completed, int total_steps) {
+ if (legacy_progress_ || !callbacks_.progress) {
+ const auto now = ggml_time_us();
+ pretty_progress(completed, total_steps, (now - last_progress_us_) / 1000000.f);
+ last_progress_us_ = now;
+ }
+ progress_.report_sample(completed, total_steps);
+ check_cancelled();
+ }
+
+ void RunEvents::preview_bridge(int step, int frame_count, const sd_tensor_t* const* frames, bool noisy, void* userdata) {
+ auto& run = *static_cast<RunEvents*>(userdata);
+ if (run.cancelled())
+ return;
+ if (run.callbacks_.preview)
+ run.callbacks_.preview(&run.preview_info_, step, frame_count, frames, noisy, run.callbacks_.preview_data);
+ else if (run.legacy_preview_)
+ run.legacy_preview_(step, frame_count, frames, noisy, run.legacy_preview_data_);
+ }
+
+ RunEventsScope::RunEventsScope(RunEvents& events)
+ : previous_(active_events),
+ byte_progress_(events.uses_legacy_progress() ? nullptr : RunEvents::report_loading, &events),
+ console_progress_(events.uses_legacy_progress() || !events.callbacks().progress) {
+ active_events = &events;
+ }
+ RunEventsScope::~RunEventsScope() {
+ active_events = previous_;
+ }
+
+ void check_run_cancelled() {
+ if (active_events)
+ active_events->check_cancelled();
+ }
+ void report_run_progress(const char* stage) {
+ if (active_events)
+ active_events->report(stage);
+ }
+ void report_run_frames(const char* stage, int completed, int total) {
+ if (active_events)
+ active_events->report_frames(stage, completed, total);
+ }
+} // namespace sd::dgl
diff --git a/src/dgl/run_events.h b/src/dgl/run_events.h
new file mode 100644
index 0000000..c074a2e
--- /dev/null
+++ b/src/dgl/run_events.h
@@ -0,0 +1,69 @@
+#ifndef __SD_DGL_RUN_EVENTS_H__
+#define __SD_DGL_RUN_EVENTS_H__
+
+#include <atomic>
+
+#include "core/option_types.h"
+#include "core/util.h"
+#include "runtime/generation_progress.h"
+
+namespace sd::dgl {
+ struct RunCallbacks {
+ sd_generation_progress_cb_t progress = nullptr;
+ void* progress_data = nullptr;
+ sd_dgl_preview_cb_t preview = nullptr;
+ void* preview_data = nullptr;
+ enum preview_t mode = PREVIEW_NONE;
+ int interval = 0;
+ bool denoised = false;
+ bool noisy = false;
+ };
+
+ class RunEvents {
+ std::atomic<bool> cancelled_{false};
+ RunCallbacks callbacks_;
+ sd::GenerationProgress progress_;
+ sd_preview_info_t preview_info_{};
+ sd_preview_cb_t legacy_preview_ = nullptr;
+ void* legacy_preview_data_ = nullptr;
+ bool legacy_progress_ = false;
+ int64_t last_progress_us_ = 0;
+
+ static void preview_bridge(int step, int frame_count, const sd_tensor_t* const* frames, bool noisy, void* userdata);
+
+ public:
+ RunEvents(const RunCallbacks& callbacks, bool legacy_callbacks);
+ void cancel() noexcept { cancelled_.store(true, std::memory_order_relaxed); }
+ void set_cancelled(bool cancelled) noexcept { cancelled_.store(cancelled, std::memory_order_relaxed); }
+ bool cancelled() const noexcept { return cancelled_.load(std::memory_order_relaxed); }
+ void check_cancelled() const;
+ void report(const char* stage, const char* detail = "", int64_t completed = 0, int64_t total = 0, const char* unit = "") const;
+ void report_frames(const char* stage, int completed, int total) const;
+ static void report_loading(int completed, int total, void* data);
+ bool uses_legacy_progress() const { return legacy_progress_; }
+ void begin_sample(int total_steps);
+ void sample_step(int completed, int total_steps);
+ const RunCallbacks& callbacks() const { return callbacks_; }
+ const sd_preview_info_t& preview_info() const { return preview_info_; }
+ bool uses_legacy_preview() const { return legacy_preview_ != nullptr; }
+ sd_preview_cb_t preview_callback() const { return callbacks_.preview || legacy_preview_ ? preview_bridge : nullptr; }
+ };
+
+ class RunEventsScope {
+ RunEvents* previous_;
+ ByteProgressScope byte_progress_;
+ ByteProgressConsoleScope console_progress_;
+
+ public:
+ explicit RunEventsScope(RunEvents& events);
+ ~RunEventsScope();
+ RunEventsScope(const RunEventsScope&) = delete;
+ RunEventsScope& operator=(const RunEventsScope&) = delete;
+ };
+
+ void check_run_cancelled();
+ void report_run_progress(const char* stage);
+ void report_run_frames(const char* stage, int completed, int total);
+} // namespace sd::dgl
+
+#endif // __SD_DGL_RUN_EVENTS_H__
diff --git a/src/dgl/run_plan.cpp b/src/dgl/run_plan.cpp
new file mode 100644
index 0000000..48e3edf
--- /dev/null
+++ b/src/dgl/run_plan.cpp
@@ -0,0 +1,293 @@
+#include "run_plan.h"
+
+#include <algorithm>
+#include <set>
+
+#include "context.h"
+#include "core/backend_fit.h"
+#include "core/util.h"
+#include "executor.h"
+#include "model_value.h"
+
+namespace sd::dgl {
+ static constexpr size_t MiB = 1024ULL * 1024;
+
+ static size_t add_size(size_t a, size_t b) {
+ return b > SIZE_MAX - a ? SIZE_MAX : a + b;
+ }
+
+ RunPlan::RunPlan(const Graph& graph, RunState& state)
+ : graph_(graph), state_(state), enabled_(state.context.auto_fit()), vae_preview_(state.events.preview_callback() && (state.events.callbacks().mode == PREVIEW_VAE || state.events.callbacks().mode == PREVIEW_TAE) && (state.events.callbacks().noisy || state.events.callbacks().denoised)) {
+ if (graph.resources.empty())
+ return;
+ if (std::any_of(graph.resources.begin(), graph.resources.end(), [](const auto& call) {
+ const auto backend = call.arguments.find("backend");
+ return is_model_kind(call.operation->second.output.type.kind) && backend != call.arguments.end() &&
+ trim(backend->second.scalar.template get<std::string>()).empty();
+ })) {
+ SDBackendManager backends(state.context.backends);
+ default_backend_ = enabled_ ? sd::backend_fit::select_default_backend(state.context.models->max_vram(), [&](auto device, auto free, auto total) {
+ return state.context.models->available_vram_for_fit(backends.existing_backend(device), free, total);
+ })
+ : sd_backend_resolve_name("");
+ }
+ if (!enabled_)
+ return;
+ for (const auto& call : graph.resources) {
+ state.events.check_cancelled();
+ if (call.deferred)
+ continue;
+ try {
+ auto args = call.arguments;
+ auto value = load_resource(*call.operation, args);
+ add_model(call, value);
+ prepared_.emplace(call.expression, std::move(value));
+ } catch (const std::bad_alloc&) {
+ throw;
+ } catch (const Error& error) {
+ if (error.status == SD_STATUS_CANCELLED)
+ throw;
+ throw Error(error.status, "line " + std::to_string(call.expression->line) + ", column " +
+ std::to_string(call.expression->column) + ": " + error.what());
+ } catch (const std::exception& error) {
+ throw Error(SD_STATUS_LOAD_FAILED, "line " + std::to_string(call.expression->line) + ", column " +
+ std::to_string(call.expression->column) + ": " + error.what());
+ }
+ }
+ for (const auto& entry : models_)
+ for (auto backend : entry.memory.compute_backends) {
+ if (budgets_.count(backend) || sd_backend_is_cpu(backend))
+ continue;
+ size_t free = 0, total = 0;
+ ggml_backend_dev_memory(ggml_backend_get_device(backend), &free, &total);
+ auto available = state.context.models->available_vram_for_fit(backend, free, total);
+ if (!state.context.models->max_vram_bytes(backend))
+ available -= std::min(available, 512 * MiB);
+ budgets_[backend] = available;
+ }
+ const auto ram = state.context.models->available_ram_for_fit();
+ ram_budget_ = ram.weights;
+ host_budget_ = ram.total;
+ place_models();
+ if (!models_.empty()) {
+ LOG_VERBOSE("auto-fit plan: %zu models, %zu phases", models_.size(), graph.phases.size());
+ for (const auto& [backend, bytes] : budgets_)
+ LOG_VERBOSE(" %s: available weight/compute budget %.2f MiB", ggml_backend_name(backend), bytes / double(MiB));
+ for (const auto& entry : models_)
+ LOG_VERBOSE(" %s: %.2f MiB params -> %s", entry.name.c_str(), entry.memory.params_bytes / double(MiB),
+ entry.memory.mode == ModelManager::ResidencyMode::Disk ? "disk" : entry.memory.params_backend ? ggml_backend_name(entry.memory.params_backend)
+ : "(none)");
+ for (const auto& phase : graph.phases) {
+ std::string names;
+ std::set<size_t> seen;
+ for (const auto& use : phase.resources) {
+ const auto found = origins_.find(use.resource);
+ if (active(use) && found != origins_.end() && seen.insert(found->second).second) {
+ if (!names.empty())
+ names += ", ";
+ names += models_[found->second].name;
+ }
+ }
+ LOG_VERBOSE(" phase %s: %s", phase.name.c_str(), names.empty() ? "(deferred)" : names.c_str());
+ }
+ }
+ update_priorities();
+ }
+
+ RunPlan::~RunPlan() {
+ state_.context.models->set_weight_use_priorities({});
+ }
+
+ Value RunPlan::load_resource(const Registry::Entry& operation, Values& args) {
+ auto backend = args.find("backend");
+ if (!default_backend_.empty() && is_model_kind(operation.second.output.type.kind) && backend != args.end() &&
+ trim(backend->second.scalar.get<std::string>()).empty())
+ backend->second = Value::literal(default_backend_);
+ return state_.context.load(operation, args, true);
+ }
+
+ size_t RunPlan::add_model(const Graph::ResourceCall& call, const Value& value) {
+ if (!is_model_kind(value.type().kind) || !value.model_source)
+ return SIZE_MAX;
+ const auto& key = value.model_source->key;
+ auto found = identities_.find(key);
+ size_t index;
+ if (found == identities_.end()) {
+ auto& model = model_from_value(value);
+ auto memory = model.model_weights->memory();
+ const auto forced = call.arguments.find("params_backend");
+ const bool automatic = !memory.split && (forced == call.arguments.end() || trim(forced->second.scalar.get<std::string>()).empty());
+ const auto kind = value.type().kind;
+ const auto reserve = kind == Kind::Denoiser || kind == Kind::TextEncoder ? 2048 * MiB : 1024 * MiB;
+ index = models_.size();
+ models_.push_back({key, call.expression->name, kind, std::move(memory), reserve, automatic});
+ identities_[key] = index;
+ } else
+ index = found->second;
+ origins_[call.expression] = index;
+ return index;
+ }
+
+ bool RunPlan::active(const Graph::ResourceUse& use) const {
+ return !use.preview || vae_preview_;
+ }
+
+ void RunPlan::place_models() {
+ std::vector<std::pair<size_t, Value>> order;
+ for (size_t i = 0; i < models_.size(); ++i) {
+ auto& entry = models_[i];
+ entry.placed = true;
+ auto cached = state_.context.resources.find({entry.key, {}});
+ if (cached == state_.context.resources.end())
+ continue;
+ if (auto value = cached->second.value.lock()) {
+ entry.placed = !entry.automatic;
+ order.emplace_back(i, std::move(*value));
+ }
+ }
+ std::stable_sort(order.begin(), order.end(), [&](const auto& a, const auto& b) {
+ const auto& left = models_[a.first];
+ const auto& right = models_[b.first];
+ if ((left.kind == Kind::Denoiser) != (right.kind == Kind::Denoiser))
+ return left.kind == Kind::Denoiser;
+ return left.memory.params_bytes > right.memory.params_bytes;
+ });
+ for (const auto& [index, value] : order) {
+ place(index, model_from_value(value));
+ models_[index].placed = true;
+ }
+ }
+
+ size_t RunPlan::peak_bytes(ggml_backend_t backend, size_t candidate, bool workspace) const {
+ size_t peak = 0;
+ for (size_t phase_index = current_; phase_index < graph_.phases.size(); ++phase_index) {
+ const auto& phase = graph_.phases[phase_index];
+ std::set<size_t> active_models;
+ for (const auto& use : phase.resources) {
+ auto found = origins_.find(use.resource);
+ if (active(use) && found != origins_.end())
+ active_models.insert(found->second);
+ }
+ if (!active_models.count(candidate))
+ continue;
+ size_t bytes = 0;
+ for (auto index : active_models) {
+ const auto& entry = models_[index];
+ const auto& memory = entry.memory;
+ const bool compute = std::find(memory.compute_backends.begin(), memory.compute_backends.end(), backend) != memory.compute_backends.end();
+ // Unplaced automatic models reserve staging space before competing for parameter storage.
+ const bool placed = index == candidate || entry.placed || !entry.automatic;
+ if (placed && memory.mode != ModelManager::ResidencyMode::Disk && memory.params_backend == backend)
+ bytes = add_size(bytes, memory.params_bytes);
+ else if (compute)
+ bytes = add_size(bytes, memory.largest_tensor);
+ if (compute && workspace)
+ bytes = add_size(bytes, entry.reserve);
+ }
+ peak = std::max(peak, bytes);
+ }
+ return peak;
+ }
+
+ void RunPlan::place(size_t index, models::Model& model) {
+ auto& entry = models_[index];
+ auto& memory = entry.memory;
+ if (!entry.automatic || memory.compute_backends.empty())
+ return;
+ auto target = memory.compute_backends.front();
+ memory.params_backend = target;
+ memory.mode = ModelManager::ResidencyMode::ParamBackend;
+ bool fits = true;
+ for (auto backend : memory.compute_backends)
+ fits = fits && (sd_backend_is_cpu(backend) ? peak_bytes(backend, index, false) <= ram_budget_ && peak_bytes(backend, index) <= host_budget_ : peak_bytes(backend, index) <= budgets_[backend]);
+ bool disk = false;
+ if (!fits) {
+ target = state_.context.backends->count("CPU") ? state_.context.backends->at("CPU").get() : nullptr;
+ if (!target) {
+ SDBackendManager backends(state_.context.backends);
+ std::string error;
+ if (!backends.init("cpu", "cpu", nullptr, &error))
+ throw std::runtime_error(error);
+ target = backends.runtime_backend(SDBackendModule::DIFFUSION);
+ }
+ memory.params_backend = target;
+ disk = peak_bytes(target, index, false) > ram_budget_ || peak_bytes(target, index) > host_budget_;
+ if (disk) {
+ target = memory.compute_backends.front();
+ memory.params_backend = target;
+ memory.mode = ModelManager::ResidencyMode::Disk;
+ }
+ }
+ const auto name = disk ? std::string("disk") : std::string(ggml_backend_name(target));
+ if (!model.place_params(name, target, disk)) {
+ memory = model.model_weights->memory();
+ LOG_DEBUG("preserved active or split placement for %s", entry.name.c_str());
+ }
+ }
+
+ void RunPlan::update_priorities() {
+ std::map<std::string, size_t> priorities;
+ for (const auto& model : models_)
+ priorities[model.key] = SIZE_MAX;
+ for (size_t phase = current_; phase < graph_.phases.size(); ++phase)
+ for (const auto& use : graph_.phases[phase].resources) {
+ auto found = origins_.find(use.resource);
+ if (active(use) && found != origins_.end()) {
+ auto& next = priorities[models_[found->second].key];
+ next = std::min(next, phase);
+ }
+ }
+ state_.context.models->set_weight_use_priorities(std::move(priorities));
+ }
+
+ void RunPlan::enter(const Expr* statement, const std::vector<const Expr*>& call_path, bool after) {
+ if (!enabled_)
+ return;
+ for (size_t i = current_; i < graph_.phases.size(); ++i)
+ if (graph_.phases[i].statement == statement && graph_.phases[i].call_path == call_path && graph_.phases[i].after == after) {
+ current_ = i;
+ update_priorities();
+ break;
+ }
+ }
+
+ void RunPlan::resource_loaded(const Expr* expression, const Value& value) {
+ if (!enabled_)
+ return;
+ prepared_.erase(expression);
+ for (const auto& call : graph_.resources) {
+ if (call.expression != expression)
+ continue;
+ const bool known = origins_.count(expression) != 0;
+ const auto index = add_model(call, value);
+ if (index == SIZE_MAX)
+ break;
+ auto& model = model_from_value(value);
+ auto current = model.model_weights->memory();
+ auto& entry = models_[index];
+ if (known && current.params_bytes == entry.memory.params_bytes && current.compute_backends == entry.memory.compute_backends) {
+ const bool disk = entry.memory.mode == ModelManager::ResidencyMode::Disk;
+ if (entry.automatic && entry.memory.params_backend && !model.place_params(disk ? "disk" : ggml_backend_name(entry.memory.params_backend), entry.memory.params_backend, disk))
+ entry.memory = std::move(current);
+ break;
+ }
+ entry.memory = std::move(current);
+ for (auto backend : entry.memory.compute_backends)
+ if (!sd_backend_is_cpu(backend) && !budgets_.count(backend)) {
+ size_t free = 0, total = 0;
+ ggml_backend_dev_memory(ggml_backend_get_device(backend), &free, &total);
+ auto available = state_.context.models->available_vram_for_fit(backend, free, total);
+ if (!state_.context.models->max_vram_bytes(backend))
+ available -= std::min(available, 512 * MiB);
+ budgets_[backend] = available;
+ }
+ const auto ram = state_.context.models->available_ram_for_fit();
+ ram_budget_ = ram.weights;
+ host_budget_ = ram.total;
+ place_models();
+ update_priorities();
+ LOG_DEBUG("updated the remaining plan for %s", entry.name.c_str());
+ break;
+ }
+ }
+}
diff --git a/src/dgl/run_plan.h b/src/dgl/run_plan.h
new file mode 100644
index 0000000..42b3957
--- /dev/null
+++ b/src/dgl/run_plan.h
@@ -0,0 +1,49 @@
+#ifndef __SD_DGL_RUN_PLAN_H__
+#define __SD_DGL_RUN_PLAN_H__
+
+#include "graph.h"
+#include "model_manager.h"
+
+namespace sd::dgl {
+ struct RunState;
+
+ class RunPlan {
+ struct ModelEntry {
+ std::string key, name;
+ Kind kind;
+ ModelManager::ResourceMemory memory;
+ size_t reserve = 0;
+ bool automatic = false;
+ bool placed = false;
+ };
+ const Graph& graph_;
+ RunState& state_;
+ bool enabled_;
+ bool vae_preview_;
+ size_t current_ = 0;
+ std::map<const Expr*, Value> prepared_;
+ std::map<const Expr*, size_t> origins_;
+ std::vector<ModelEntry> models_;
+ std::map<std::string, size_t> identities_;
+ std::map<ggml_backend_t, size_t> budgets_;
+ size_t ram_budget_ = 0;
+ size_t host_budget_ = 0;
+ std::string default_backend_;
+
+ size_t add_model(const Graph::ResourceCall& call, const Value& value);
+ bool active(const Graph::ResourceUse& use) const;
+ void place_models();
+ void place(size_t index, models::Model& model);
+ size_t peak_bytes(ggml_backend_t backend, size_t candidate, bool workspace = true) const;
+ void update_priorities();
+
+ public:
+ RunPlan(const Graph& graph, RunState& state);
+ ~RunPlan();
+ Value load_resource(const Registry::Entry& operation, Values& args);
+ void enter(const Expr* statement, const std::vector<const Expr*>& call_path = {}, bool after = false);
+ void resource_loaded(const Expr* expression, const Value& value);
+ };
+}
+
+#endif
diff --git a/src/dgl/syntax/lexer.cpp b/src/dgl/syntax/lexer.cpp
new file mode 100644
index 0000000..17aacd5
--- /dev/null
+++ b/src/dgl/syntax/lexer.cpp
@@ -0,0 +1,120 @@
+#include "lexer.h"
+
+#include <cctype>
+#include <stdexcept>
+
+#include "dgl/version.h"
+
+namespace sd::dgl {
+
+ [[noreturn]] static void fail(int line, int column, const std::string& message) {
+ throw std::runtime_error("line " + std::to_string(line) + ", column " + std::to_string(column) + ": " + message);
+ }
+
+ Lexer::Lexer(const std::string& source)
+ : source_(source) {
+ if (source_.compare(0, 3, "\xef\xbb\xbf") == 0)
+ offset_ = 3;
+ }
+
+ void Lexer::comment(size_t begin, int line, int column) {
+ if (source_[begin] != '#')
+ return;
+ std::string text = source_.substr(begin + 1, offset_ - begin - 1);
+ size_t key = text.find_first_not_of(" \t\r");
+ if (key == std::string::npos || text.compare(key, 7, "version") != 0)
+ return;
+ size_t colon = text.find_first_not_of(" \t", key + 7);
+ if (colon == std::string::npos || text[colon] != ':')
+ return;
+ if (seen_token_)
+ fail(line, column, "version header must precede all code");
+ if (seen_version_)
+ fail(line, column, "duplicate version header");
+ size_t value = text.find_first_not_of(" \t\r", colon + 1);
+ Version required;
+ try {
+ required = Version::parse(value == std::string::npos ? "" : text.substr(value, text.find_last_not_of(" \t\r") - value + 1));
+ } catch (const std::invalid_argument& error) {
+ fail(line, column, error.what());
+ }
+ if (!runtime_version.supports(required)) {
+ std::string message = "script requires DGL " + required.str() + "; runtime supports DGL " + runtime_version.str() +
+ " (the major version must match and the runtime must meet the script's minimum version)";
+ if (required.major == 0 && required.minor == 0 && required.patch == 0)
+ message += "; 0.0.0 is a development script: verify it before updating its version declaration";
+ fail(line, column, message);
+ }
+ seen_version_ = true;
+ }
+
+ void Lexer::advance_char() {
+ if (source_[offset_++] == '\n') {
+ ++line_;
+ column_ = 1;
+ } else
+ ++column_;
+ }
+
+ Token Lexer::next() {
+ while (offset_ < source_.size()) {
+ if (std::isspace(static_cast<unsigned char>(source_[offset_])))
+ advance_char();
+ else if (source_[offset_] == '#' || (source_[offset_] == '/' && offset_ + 1 < source_.size() && source_[offset_ + 1] == '/')) {
+ size_t begin = offset_;
+ int line = line_, column = column_;
+ while (offset_ < source_.size() && source_[offset_] != '\n')
+ advance_char();
+ comment(begin, line, column);
+ } else
+ break;
+ }
+ Token token{"", line_, column_};
+ if (offset_ == source_.size())
+ return token;
+ seen_token_ = true;
+ size_t begin = offset_;
+ char c = source_[offset_];
+ advance_char();
+ if (c == '"') {
+ bool closed = false;
+ while (offset_ < source_.size()) {
+ char current = source_[offset_];
+ if (current == '\n' || current == '\r')
+ fail(token.line, token.column, "unescaped newline in string");
+ advance_char();
+ if (current == '"') {
+ closed = true;
+ break;
+ }
+ if (current == '\\' && offset_ < source_.size())
+ advance_char();
+ }
+ if (!closed)
+ fail(token.line, token.column, "unterminated string");
+ } else if (std::isalpha(static_cast<unsigned char>(c)) || c == '_') {
+ while (offset_ < source_.size() && (std::isalnum(static_cast<unsigned char>(source_[offset_])) || source_[offset_] == '_'))
+ advance_char();
+ } else if (c == ':' && offset_ < source_.size() && source_[offset_] == ':') {
+ advance_char();
+ } else if (std::isdigit(static_cast<unsigned char>(c))) {
+ while (offset_ < source_.size() && std::isdigit(static_cast<unsigned char>(source_[offset_])))
+ advance_char();
+ if (offset_ < source_.size() && source_[offset_] == '.') {
+ advance_char();
+ while (offset_ < source_.size() && std::isdigit(static_cast<unsigned char>(source_[offset_])))
+ advance_char();
+ }
+ if (offset_ < source_.size() && (source_[offset_] == 'e' || source_[offset_] == 'E')) {
+ advance_char();
+ if (offset_ < source_.size() && (source_[offset_] == '+' || source_[offset_] == '-'))
+ advance_char();
+ while (offset_ < source_.size() && std::isdigit(static_cast<unsigned char>(source_[offset_])))
+ advance_char();
+ }
+ }
+ token.text = source_.substr(begin, offset_ - begin);
+ return token;
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/syntax/lexer.h b/src/dgl/syntax/lexer.h
new file mode 100644
index 0000000..44e693b
--- /dev/null
+++ b/src/dgl/syntax/lexer.h
@@ -0,0 +1,33 @@
+#ifndef __SD_DGL_LEXER_H__
+#define __SD_DGL_LEXER_H__
+
+#include <cstddef>
+#include <string>
+
+namespace sd::dgl {
+
+ struct Token {
+ std::string text;
+ int line = 1;
+ int column = 1;
+ };
+
+ class Lexer {
+ const std::string& source_;
+ size_t offset_ = 0;
+ int line_ = 1;
+ int column_ = 1;
+ bool seen_token_ = false;
+ bool seen_version_ = false;
+
+ void comment(size_t begin, int line, int column);
+ void advance_char();
+
+ public:
+ explicit Lexer(const std::string& source);
+ Token next();
+ };
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_LEXER_H__
diff --git a/src/dgl/syntax/parser.cpp b/src/dgl/syntax/parser.cpp
new file mode 100644
index 0000000..414e5e9
--- /dev/null
+++ b/src/dgl/syntax/parser.cpp
@@ -0,0 +1,370 @@
+#include "syntax.h"
+
+#include <cctype>
+#include <set>
+#include <stdexcept>
+
+#include "lexer.h"
+
+namespace sd::dgl {
+
+ class Parser {
+ Lexer lexer_;
+ Token token_;
+ size_t expression_count_ = 0;
+
+ void next() { token_ = lexer_.next(); }
+
+ [[noreturn]] void fail(const std::string& message) const {
+ throw std::runtime_error("line " + std::to_string(token_.line) + ", column " + std::to_string(token_.column) + ": " + message);
+ }
+
+ bool take(const std::string& text) {
+ if (token_.text != text)
+ return false;
+ next();
+ return true;
+ }
+
+ void expect(const std::string& text) {
+ if (!take(text))
+ fail("expected '" + text + "', got '" + token_.text + "'");
+ }
+
+ std::string identifier() {
+ if (token_.text.empty() || (!std::isalpha(static_cast<unsigned char>(token_.text[0])) && token_.text[0] != '_'))
+ fail("expected identifier");
+ std::string result = token_.text;
+ if (result == "return" || result == "fn" || result == "true" || result == "false" || result == "if" || result == "else")
+ fail("reserved identifier '" + result + "'");
+ next();
+ return result;
+ }
+
+ nlohmann::json scalar_literal(const std::string& text) {
+ try {
+ auto parsed = nlohmann::json::parse(text);
+ if ((text[0] == '-' || std::isdigit(static_cast<unsigned char>(text[0]))) && text.find_first_of(".eE") == std::string::npos && !parsed.is_number_integer())
+ fail("integer exceeds int64 range");
+ if (parsed.is_number_unsigned() && parsed.get<uint64_t>() > uint64_t(INT64_MAX))
+ fail("integer exceeds int64 range");
+ return parsed;
+ } catch (const std::exception&) {
+ fail("invalid scalar literal");
+ }
+ }
+
+ static int precedence(const std::string& op) {
+ if (op == "+" || op == "-")
+ return 1;
+ if (op == "*" || op == "/")
+ return 2;
+ return -1;
+ }
+
+ static std::shared_ptr<Expr> arithmetic(const Token& op, std::shared_ptr<Expr> left, std::shared_ptr<Expr> right) {
+ auto result = std::make_shared<Expr>();
+ result->form = Expr::Form::Call;
+ result->line = op.line;
+ result->column = op.column;
+ result->name = op.text == "+" ? "math::add" : op.text == "-" ? "math::sub"
+ : op.text == "*" ? "math::mul"
+ : "math::div";
+ result->arguments = {{"", std::move(left)}, {"", std::move(right)}};
+ return result;
+ }
+
+ std::shared_ptr<Expr> index_suffix(std::shared_ptr<Expr> expr, int& depth, bool literal_only) {
+ while (token_.text == "[") {
+ if (literal_only)
+ fail("var values must be literals");
+ if (++depth > 64 || (expression_count_ += 2) > 1000000)
+ fail("DGL expression limit exceeded");
+ auto call = std::make_shared<Expr>();
+ call->form = Expr::Form::Index;
+ call->name = "index";
+ call->line = token_.line;
+ call->column = token_.column;
+ auto axes = std::make_shared<Expr>();
+ axes->form = Expr::Form::List;
+ axes->line = token_.line;
+ axes->column = token_.column;
+ next();
+ if (token_.text == "]")
+ fail("indexing requires at least one axis");
+ do {
+ if (++expression_count_ > 1000000)
+ fail("DGL expression limit exceeded");
+ auto axis = std::make_shared<Expr>();
+ axis->form = Expr::Form::Dict;
+ axis->line = token_.line;
+ axis->column = token_.column;
+ if (token_.text == "::")
+ fail("indexing steps are not supported");
+ std::shared_ptr<Expr> first;
+ if (token_.text != ":")
+ first = expression(depth + 1);
+ if (token_.text == "::")
+ fail("indexing steps are not supported");
+ if (take(":")) {
+ if (first)
+ axis->arguments.push_back({"start", std::move(first)});
+ if (token_.text != "," && token_.text != "]")
+ axis->arguments.push_back({"end", expression(depth + 1)});
+ if (token_.text == ":" || token_.text == "::")
+ fail("indexing steps are not supported");
+ } else {
+ axis->arguments.push_back({"index", std::move(first)});
+ }
+ axes->arguments.push_back({"", std::move(axis)});
+ } while (take(",") && token_.text != "]");
+ expect("]");
+ call->arguments = {{"input", std::move(expr)}, {"indices", std::move(axes)}};
+ expr = std::move(call);
+ }
+ return expr;
+ }
+
+ std::shared_ptr<Expr> block(int depth) {
+ auto expr = std::make_shared<Expr>();
+ expr->form = Expr::Form::Block;
+ expr->line = token_.line;
+ expr->column = token_.column;
+ expect("{");
+ std::set<std::string> names;
+ while (!take("}")) {
+ if (token_.text.empty())
+ fail("unterminated if block");
+ if (token_.text == "return")
+ fail("use a final expression for an if block result; return is only allowed in functions");
+ auto value = expression(depth + 1);
+ if (take("=")) {
+ expr->body.push_back(binding(*value, names, depth + 1));
+ take(";");
+ } else {
+ take(";");
+ if (token_.text.empty())
+ fail("unterminated if block");
+ if (token_.text == "}" && !(value->form == Expr::Form::Conditional && value->arguments.size() == 2))
+ expr->body.push_back({{}, std::move(value), Statement::Form::Result});
+ else {
+ if (value->form != Expr::Form::Call && value->form != Expr::Form::Index && value->form != Expr::Form::Conditional)
+ fail("only function calls and conditional expressions can be standalone statements");
+ expr->body.push_back({{}, std::move(value), Statement::Form::Call});
+ }
+ }
+ }
+ return expr;
+ }
+
+ std::shared_ptr<Expr> expression(int depth = 0, bool literal_only = false, int minimum_precedence = 0) {
+ if (depth > 64 || ++expression_count_ > 1000000)
+ fail("DGL expression limit exceeded");
+ auto expr = std::make_shared<Expr>();
+ expr->line = token_.line;
+ expr->column = token_.column;
+ if (token_.text == "$")
+ fail("write input values in the DGL source before compiling");
+ if (take("if")) {
+ if (literal_only)
+ fail("var values must be literals");
+ expr->form = Expr::Form::Conditional;
+ expr->name = "if";
+ auto condition = expression(depth + 1);
+ auto selected = block(depth);
+ expr->arguments = {{"condition", std::move(condition)}, {"then", std::move(selected)}};
+ if (take("else"))
+ expr->arguments.push_back({"else", block(depth)});
+ } else if (token_.text == "+" || token_.text == "-") {
+ const auto sign = token_;
+ next();
+ if (!token_.text.empty() && std::isdigit(static_cast<unsigned char>(token_.text[0]))) {
+ expr->literal = scalar_literal((sign.text == "-" ? "-" : "") + token_.text);
+ next();
+ } else {
+ if (literal_only)
+ fail("var values must be literals");
+ expr->literal = 0;
+ auto operand = expression(depth + 1, false, 3);
+ expr = arithmetic(sign, std::move(expr), std::move(operand));
+ if (++expression_count_ > 1000000)
+ fail("DGL expression limit exceeded");
+ }
+ } else if (take("(")) {
+ expr = expression(depth + 1, literal_only);
+ expect(")");
+ } else if (take("{")) {
+ expr->form = Expr::Form::Dict;
+ std::set<std::string> names;
+ if (!take("}")) {
+ do {
+ auto name = identifier();
+ if (!names.insert(name).second)
+ fail("duplicate dict field");
+ expect("=");
+ expr->arguments.push_back({name, expression(depth + 1, literal_only)});
+ } while (take(",") && token_.text != "}");
+ expect("}");
+ }
+ } else if (take("[")) {
+ expr->form = Expr::Form::List;
+ if (!take("]")) {
+ do {
+ expr->arguments.push_back({"", expression(depth + 1, literal_only)});
+ } while (take(",") && token_.text != "]");
+ expect("]");
+ }
+ } else if (!token_.text.empty() && (token_.text[0] == '"' || std::isdigit(static_cast<unsigned char>(token_.text[0])) || token_.text == "true" || token_.text == "false")) {
+ expr->literal = scalar_literal(token_.text);
+ next();
+ } else {
+ if (literal_only)
+ fail("var values must be literals");
+ expr->name = identifier();
+ expr->form = Expr::Form::Reference;
+ bool qualified = false;
+ while (take("::")) {
+ expr->name += "::" + identifier();
+ qualified = true;
+ }
+ if (take("(")) {
+ expr->form = Expr::Form::Call;
+ bool named = false;
+ if (!take(")")) {
+ do {
+ auto arg = expression(depth + 1);
+ std::string name;
+ if (take("=")) {
+ if (arg->form != Expr::Form::Reference || arg->name.find('.') != std::string::npos)
+ fail("expected argument name");
+ name = arg->name;
+ arg = expression(depth + 1);
+ named = true;
+ } else if (named)
+ fail("positional arguments must precede named arguments");
+ expr->arguments.push_back({name, arg});
+ } while (take(",") && token_.text != ")");
+ expect(")");
+ }
+ } else if (qualified) {
+ fail("expected '(' after qualified function name");
+ }
+ while (take(".")) {
+ if (expr->form != Expr::Form::Reference)
+ fail("assign a call result before accessing its fields");
+ expr->name += "." + identifier();
+ }
+ }
+ expr = index_suffix(std::move(expr), depth, literal_only);
+ while (precedence(token_.text) >= minimum_precedence) {
+ if (literal_only)
+ fail("var values must be literals");
+ if (++depth > 64 || ++expression_count_ > 1000000)
+ fail("DGL expression limit exceeded");
+ const auto op = token_;
+ const auto priority = precedence(op.text);
+ next();
+ auto right = expression(depth + 1, false, priority + 1);
+ expr = arithmetic(op, std::move(expr), std::move(right));
+ }
+ return expr;
+ }
+
+ Statement binding(const Expr& target, std::set<std::string>& names, int depth = 0) {
+ if (target.form != Expr::Form::Reference || target.name.find('.') != std::string::npos)
+ fail("expected a binding name before '='");
+ if (!names.insert(target.name).second)
+ fail("duplicate binding '" + target.name + "'");
+ return {target.name, expression(depth)};
+ }
+
+ Statement binding(std::set<std::string>& names, bool literal_only = false) {
+ auto name = identifier();
+ if (!names.insert(name).second)
+ fail("duplicate binding '" + name + "'");
+ expect("=");
+ return {std::move(name), expression(0, literal_only)};
+ }
+
+ std::vector<Statement> body(std::vector<Statement> statements = {}, std::set<std::string> names = {}) {
+ expect("{");
+ bool returned = false;
+ while (!take("}")) {
+ if (token_.text.empty())
+ fail("unterminated function body");
+ if (returned)
+ fail("return must be the last statement");
+ if (take("return")) {
+ returned = true;
+ statements.push_back({{}, token_.text == ";" || token_.text == "}" ? nullptr : expression(), Statement::Form::Return});
+ } else {
+ auto value = expression();
+ if (take("=")) {
+ statements.push_back(binding(*value, names));
+ } else {
+ if (value->form != Expr::Form::Call && value->form != Expr::Form::Index && value->form != Expr::Form::Conditional)
+ fail("only function calls and conditional expressions can be standalone statements");
+ statements.push_back({{}, std::move(value), Statement::Form::Call});
+ }
+ }
+ take(";");
+ }
+ return statements;
+ }
+
+ public:
+ explicit Parser(const std::string& source)
+ : lexer_(source) { next(); }
+
+ Syntax parse(bool host_inputs) {
+ Syntax program;
+ std::vector<Statement> variables;
+ std::set<std::string> variable_names;
+ if (take("var")) {
+ expect("{");
+ while (!take("}")) {
+ if (token_.text.empty())
+ fail("unterminated var block");
+ variables.push_back(binding(variable_names, true));
+ take(";");
+ }
+ }
+ while (take("fn")) {
+ std::string name = identifier();
+ Function function;
+ expect("(");
+ std::set<std::string> names;
+ if (!take(")")) {
+ if (name == "main" && !host_inputs)
+ fail("fn main must have no parameters");
+ do {
+ auto parameter = identifier();
+ if (!names.insert(parameter).second)
+ fail("duplicate parameter");
+ function.parameters.push_back(parameter);
+ } while (take(","));
+ expect(")");
+ }
+ function.body = name == "main" ? body(std::move(variables), std::move(variable_names)) : body();
+ if (!program.functions.emplace(name, std::move(function)).second)
+ fail("duplicate function '" + name + "'");
+ }
+ if (token_.text == "var")
+ fail("var block may appear once, before the functions");
+ if (!token_.text.empty())
+ fail("expected a function declaration");
+ auto entry = program.functions.extract("main");
+ if (entry.empty())
+ fail("missing entry point 'fn main'");
+ program.entry = std::move(entry.mapped());
+ return program;
+ }
+ };
+
+ Syntax parse_syntax(const std::string& source, bool host_inputs) {
+ if (source.size() > 512 * 1024 * 1024)
+ throw std::runtime_error("DGL source exceeds 512 MiB");
+ return Parser(source).parse(host_inputs);
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/syntax/syntax.h b/src/dgl/syntax/syntax.h
new file mode 100644
index 0000000..8841727
--- /dev/null
+++ b/src/dgl/syntax/syntax.h
@@ -0,0 +1,58 @@
+#ifndef __SD_DGL_SYNTAX_SYNTAX_H__
+#define __SD_DGL_SYNTAX_SYNTAX_H__
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "json.hpp"
+
+namespace sd::dgl {
+
+ struct Expr;
+
+ struct Statement {
+ enum class Form { Binding,
+ Call,
+ Return,
+ Result };
+ std::string name;
+ std::shared_ptr<Expr> value;
+ Form form = Form::Binding;
+ };
+
+ struct Expr {
+ enum class Form { Literal,
+ Reference,
+ FunctionReference,
+ Call,
+ Index,
+ Conditional,
+ List,
+ Dict,
+ Block };
+ Form form = Form::Literal;
+ nlohmann::json literal;
+ std::string name;
+ std::vector<std::pair<std::string, std::shared_ptr<Expr>>> arguments;
+ std::vector<Statement> body;
+ int line = 1;
+ int column = 1;
+ };
+
+ struct Function {
+ std::vector<std::string> parameters;
+ std::vector<Statement> body;
+ };
+
+ struct Syntax {
+ std::map<std::string, Function> functions;
+ Function entry;
+ };
+
+ Syntax parse_syntax(const std::string& source, bool host_inputs);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_SYNTAX_SYNTAX_H__
diff --git a/src/dgl/tensor.h b/src/dgl/tensor.h
new file mode 100644
index 0000000..a49275c
--- /dev/null
+++ b/src/dgl/tensor.h
@@ -0,0 +1,37 @@
+#ifndef __SD_DGL_TENSOR_H__
+#define __SD_DGL_TENSOR_H__
+
+#include <memory>
+#include <utility>
+
+#include "api/tensor.h"
+
+namespace sd::dgl {
+ inline constexpr size_t tensor_byte_limit = size_t(1024) * 1024 * 1024;
+ using sd::api::tensor_dtype;
+
+ template <class F>
+ decltype(auto) with_dtype(sd_tensor_type_t dtype, F function) {
+ switch (dtype) {
+ case SD_TENSOR_U8:
+ return function(uint8_t{});
+ case SD_TENSOR_I32:
+ return function(int32_t{});
+ case SD_TENSOR_I64:
+ return function(int64_t{});
+ case SD_TENSOR_F32:
+ return function(float{});
+ case SD_TENSOR_F64:
+ return function(double{});
+ default:
+ throw std::runtime_error("invalid tensor dtype");
+ }
+ }
+
+ template <class... Args>
+ std::shared_ptr<sd_tensor_t> make_tensor(Args&&... args) {
+ return {new sd_tensor_t(std::forward<Args>(args)...), sd_tensor_free};
+ }
+}
+
+#endif // __SD_DGL_TENSOR_H__
diff --git a/src/dgl/tensor_io.cpp b/src/dgl/tensor_io.cpp
new file mode 100644
index 0000000..afe6f7b
--- /dev/null
+++ b/src/dgl/tensor_io.cpp
@@ -0,0 +1,57 @@
+#include "tensor_io.h"
+
+#include <atomic>
+#include <chrono>
+#include <filesystem>
+
+#include "core/file_access.h"
+#include "model_io/npy_io.h"
+
+namespace sd::dgl {
+
+ std::shared_ptr<sd_tensor_t> load_npy(const std::string& path, size_t max_bytes) {
+ require_file_access(path, SD_DGL_FILE_READ);
+ npy::Reader reader(path, max_bytes);
+ auto tensor = make_tensor(reader.type(), reader.shape());
+ reader.read(tensor->mutable_data(), tensor->bytes);
+ return tensor;
+ }
+
+ class TemporaryNpyFile {
+ std::filesystem::path directory_;
+ std::filesystem::path path_;
+
+ public:
+ explicit TemporaryNpyFile(const std::filesystem::path& parent) {
+ static std::atomic<uint64_t> sequence{0};
+ const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count();
+ const auto prefix = ".dgl-tensor-" + std::to_string(timestamp) + "-";
+ do {
+ directory_ = parent / (prefix + std::to_string(sequence.fetch_add(1, std::memory_order_relaxed)));
+ path_ = directory_ / "tensor.npy";
+ } while (!std::filesystem::create_directory(directory_));
+ }
+
+ TemporaryNpyFile(const TemporaryNpyFile&) = delete;
+ TemporaryNpyFile& operator=(const TemporaryNpyFile&) = delete;
+
+ const std::filesystem::path& path() const { return path_; }
+
+ ~TemporaryNpyFile() {
+ std::error_code ignored;
+ std::filesystem::remove(path_, ignored);
+ std::filesystem::remove(directory_, ignored);
+ }
+ };
+
+ void save_npy_atomic(const sd_tensor_t& tensor, const std::string& path) {
+ if (path.empty())
+ throw std::runtime_error("tensor output path is required");
+ require_file_access(path, SD_DGL_FILE_WRITE);
+ const auto destination = std::filesystem::u8path(path);
+ const TemporaryNpyFile temporary(destination.parent_path());
+ npy::write(temporary.path(), tensor.type, tensor.shape(), tensor.data(), tensor.bytes);
+ std::filesystem::rename(temporary.path(), destination);
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/tensor_io.h b/src/dgl/tensor_io.h
new file mode 100644
index 0000000..eea6612
--- /dev/null
+++ b/src/dgl/tensor_io.h
@@ -0,0 +1,15 @@
+#ifndef __SD_DGL_TENSOR_IO_H__
+#define __SD_DGL_TENSOR_IO_H__
+
+#include <string>
+
+#include "tensor.h"
+
+namespace sd::dgl {
+
+ std::shared_ptr<sd_tensor_t> load_npy(const std::string& path, size_t max_bytes);
+ void save_npy_atomic(const sd_tensor_t& tensor, const std::string& path);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_TENSOR_IO_H__
diff --git a/src/dgl/value.cpp b/src/dgl/value.cpp
new file mode 100644
index 0000000..19e47cf
--- /dev/null
+++ b/src/dgl/value.cpp
@@ -0,0 +1,153 @@
+#include "value.h"
+
+#include <cmath>
+#include <stdexcept>
+
+namespace sd::dgl {
+
+ bool accepts(Type expected, Type actual) {
+ if (expected.kind == Kind::Void || actual.kind == Kind::Void)
+ return expected.kind == actual.kind;
+ if (expected.kind == Kind::Any || actual.kind == Kind::Any)
+ return true;
+ if (expected.kind == Kind::Number && actual.kind == Kind::Integer)
+ return true;
+ if (expected.kind == Kind::Model)
+ return is_model_kind(actual.kind);
+ if (expected.kind == Kind::IntVector)
+ return actual.kind == Kind::Tensor || (actual.kind == Kind::List && accepts({Kind::Integer}, {actual.element}));
+ return expected.kind == actual.kind &&
+ (expected.kind != Kind::List || accepts({expected.element}, {actual.element}));
+ }
+
+ bool accepts_list_element(Kind element, Kind actual) {
+ return actual != Kind::Any && actual != Kind::List && actual != Kind::Void && accepts({element}, {actual});
+ }
+
+ bool is_model_kind(Kind kind) {
+ switch (kind) {
+ case Kind::Denoiser:
+ case Kind::TextEncoder:
+ case Kind::VAE:
+ case Kind::ControlNet:
+ case Kind::VisionEncoder:
+ case Kind::AudioEncoder:
+ case Kind::AudioVAE:
+ case Kind::IPAdapter:
+ case Kind::PhotoMaker:
+ case Kind::PuLID:
+ case Kind::AnimateDiff:
+ case Kind::LatentUpscaler:
+ case Kind::Upscaler:
+ case Kind::Detector:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ Value Value::void_value() {
+ Value value;
+ value.type_.kind = Kind::Void;
+ return value;
+ }
+
+ Value Value::literal(const nlohmann::json& scalar) {
+ Value value;
+ value.scalar = scalar;
+ if (scalar.is_boolean())
+ value.type_.kind = Kind::Boolean;
+ else if (scalar.is_number_integer()) {
+ if (scalar.is_number_unsigned() && scalar.get<uint64_t>() > uint64_t(INT64_MAX))
+ throw std::runtime_error("integer exceeds int64 range");
+ value.scalar = scalar.get<int64_t>();
+ value.type_.kind = Kind::Integer;
+ } else if (scalar.is_number_float()) {
+ if (!std::isfinite(scalar.get<double>()))
+ throw std::runtime_error("number must be finite");
+ value.type_.kind = Kind::Number;
+ } else if (scalar.is_string())
+ value.type_.kind = Kind::String;
+ else
+ throw std::runtime_error("expected a scalar value");
+ return value;
+ }
+
+ Value Value::dict(Values fields) {
+ for (const auto& [name, field] : fields)
+ if (field.type().kind == Kind::Void)
+ throw std::runtime_error("dict field '" + name + "' has no value");
+ Value value;
+ value.type_.kind = Kind::Dict;
+ value.fields = std::move(fields);
+ return value;
+ }
+
+ Value Value::list(std::vector<Value> values, Kind declared_element) {
+ Value value;
+ value.type_ = {Kind::List, declared_element};
+ auto& element = value.type_.element;
+ if (element == Kind::Any && !values.empty()) {
+ element = values.front().type_.kind;
+ for (const auto& item : values)
+ if ((element == Kind::Integer && item.type_.kind == Kind::Number) || (element == Kind::Number && item.type_.kind == Kind::Integer))
+ element = Kind::Number;
+ }
+ for (const auto& item : values) {
+ if (!accepts_list_element(element, item.type_.kind))
+ throw std::runtime_error("lists must contain values of one non-list type");
+ }
+ value.items = std::move(values);
+ return value;
+ }
+
+ Value::Cached::Cached(Value value, bool retain)
+ : value_(std::move(value)), object_(value_.object_) {
+ if (!retain)
+ value_.object_.reset();
+ }
+
+ std::optional<Value> Value::Cached::lock() const {
+ if (auto object = object_.lock()) {
+ auto value = value_;
+ value.object_ = std::move(object);
+ return value;
+ }
+ return {};
+ }
+
+ std::string type_name(Type type) {
+ if (type.kind == Kind::Bytes)
+ return "bytes";
+ static const char* names[] = {"any", "bool", "int", "number", "string", "upscaler", "detector", "list", "tensor", "dict", "denoiser", "text_encoder", "vae", "predictor", "control_net", "control_bindings", "ip_adapter_bindings", "vision_encoder", "audio_encoder", "audio_vae", "ip_adapter", "photomaker", "pulid", "pulid_bindings", "animatediff", "latent_upscaler", "rng"};
+ if (type.kind == Kind::List)
+ return "list<" + type_name({type.element}) + ">";
+ if (type.kind == Kind::IntVector)
+ return "i64 tensor or list<int>";
+ if (type.kind == Kind::Model)
+ return "model";
+ if (type.kind == Kind::Lora)
+ return "lora";
+ if (type.kind == Kind::Function)
+ return "fn";
+ if (type.kind == Kind::SamplingObserver)
+ return "sampling_observer";
+ if (type.kind == Kind::Void)
+ return "void";
+ return names[static_cast<size_t>(type.kind)];
+ }
+
+ int64_t get_int(const Values& args, const std::string& name) {
+ return args.at(name).scalar.get<int64_t>();
+ }
+ double get_number(const Values& args, const std::string& name) {
+ return args.at(name).scalar.get<double>();
+ }
+ std::string get_string(const Values& args, const std::string& name) {
+ return args.at(name).scalar.get<std::string>();
+ }
+ bool get_bool(const Values& args, const std::string& name) {
+ return args.at(name).scalar.get<bool>();
+ }
+
+} // namespace sd::dgl
diff --git a/src/dgl/value.h b/src/dgl/value.h
new file mode 100644
index 0000000..3951cb5
--- /dev/null
+++ b/src/dgl/value.h
@@ -0,0 +1,196 @@
+#ifndef __SD_DGL_VALUE_H__
+#define __SD_DGL_VALUE_H__
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "bytes.h"
+#include "diagnostic.h"
+#include "json.hpp"
+#include "models/fwd.h"
+#include "ops/fwd.h"
+
+class RNG;
+struct sd_tensor_t;
+
+namespace sd::dgl {
+
+ enum class Kind { Any,
+ Boolean,
+ Integer,
+ Number,
+ String,
+ Upscaler,
+ Detector,
+ List,
+ Tensor,
+ Dict,
+ Denoiser,
+ TextEncoder,
+ VAE,
+ Predictor,
+ ControlNet,
+ ControlBindings,
+ IPAdapterBindings,
+ VisionEncoder,
+ AudioEncoder,
+ AudioVAE,
+ IPAdapter,
+ PhotoMaker,
+ PuLID,
+ PuLIDBindings,
+ AnimateDiff,
+ LatentUpscaler,
+ Rng,
+ IntVector,
+ Model,
+ Lora,
+ Function,
+ SamplingObserver,
+ Void,
+ Bytes };
+
+ struct Type {
+ Kind kind = Kind::Any;
+ Kind element = Kind::Any;
+ bool operator==(const Type& other) const { return kind == other.kind && element == other.element; }
+ };
+
+ struct ModelSource {
+ std::string key;
+ std::string revision;
+ };
+
+ std::string type_name(Type type);
+ bool accepts(Type expected, Type actual);
+ bool accepts_list_element(Kind element, Kind actual);
+
+ template <class T>
+ struct ObjectKind;
+
+ template <>
+ struct ObjectKind<RNG> : std::integral_constant<Kind, Kind::Rng> {};
+ template <>
+ struct ObjectKind<sd_tensor_t> : std::integral_constant<Kind, Kind::Tensor> {};
+ template <>
+ struct ObjectKind<models::AudioEncoder> : std::integral_constant<Kind, Kind::AudioEncoder> {};
+ template <>
+ struct ObjectKind<models::AudioVAE> : std::integral_constant<Kind, Kind::AudioVAE> {};
+ template <>
+ struct ObjectKind<models::ControlNet> : std::integral_constant<Kind, Kind::ControlNet> {};
+ template <>
+ struct ObjectKind<models::ControlBindings> : std::integral_constant<Kind, Kind::ControlBindings> {};
+ template <>
+ struct ObjectKind<ops::IPAdapterBindings> : std::integral_constant<Kind, Kind::IPAdapterBindings> {};
+ template <>
+ struct ObjectKind<models::DetailDetector> : std::integral_constant<Kind, Kind::Detector> {};
+ template <>
+ struct ObjectKind<models::DiffusionModel> : std::integral_constant<Kind, Kind::Denoiser> {};
+ template <>
+ struct ObjectKind<models::AnimateDiff> : std::integral_constant<Kind, Kind::AnimateDiff> {};
+ template <>
+ struct ObjectKind<models::LatentUpscaler> : std::integral_constant<Kind, Kind::LatentUpscaler> {};
+ template <>
+ struct ObjectKind<models::ImageUpscaler> : std::integral_constant<Kind, Kind::Upscaler> {};
+ template <>
+ struct ObjectKind<models::IPAdapter> : std::integral_constant<Kind, Kind::IPAdapter> {};
+ template <>
+ struct ObjectKind<models::LoraFile> : std::integral_constant<Kind, Kind::Lora> {};
+ template <>
+ struct ObjectKind<models::PhotoMaker> : std::integral_constant<Kind, Kind::PhotoMaker> {};
+ template <>
+ struct ObjectKind<models::PuLID> : std::integral_constant<Kind, Kind::PuLID> {};
+ template <>
+ struct ObjectKind<ops::PuLIDBindings> : std::integral_constant<Kind, Kind::PuLIDBindings> {};
+ template <>
+ struct ObjectKind<ops::Predictor> : std::integral_constant<Kind, Kind::Predictor> {};
+ template <>
+ struct ObjectKind<ops::SamplingObserver> : std::integral_constant<Kind, Kind::SamplingObserver> {};
+ template <>
+ struct ObjectKind<models::TextEncoder> : std::integral_constant<Kind, Kind::TextEncoder> {};
+ template <>
+ struct ObjectKind<models::VAE> : std::integral_constant<Kind, Kind::VAE> {};
+ template <>
+ struct ObjectKind<models::VisionEncoder> : std::integral_constant<Kind, Kind::VisionEncoder> {};
+
+ template <>
+ struct ObjectKind<Bytes> : std::integral_constant<Kind, Kind::Bytes> {};
+
+ class Value {
+ Type type_;
+ std::shared_ptr<void> object_;
+
+ public:
+ class Cached;
+
+ nlohmann::json scalar;
+ std::vector<Value> items;
+ std::map<std::string, Value> fields;
+ std::shared_ptr<const models::LoraConfig> loras;
+ std::shared_ptr<const ModelSource> model_source;
+
+ Type type() const { return type_; }
+
+ static Value literal(const nlohmann::json& scalar);
+ static Value void_value();
+ static Value list(std::vector<Value> values, Kind element = Kind::Any);
+ static Value dict(std::map<std::string, Value> fields);
+
+ template <class T>
+ static Value object(std::shared_ptr<T> object) {
+ if (!object)
+ throw std::invalid_argument("DGL object payload is null");
+ Value value;
+ value.type_.kind = ObjectKind<T>::value;
+ value.object_ = std::move(object);
+ return value;
+ }
+
+ template <class T>
+ const T* get_if() const {
+ return type_.kind == ObjectKind<T>::value ? static_cast<const T*>(object_.get()) : nullptr;
+ }
+
+ template <class T>
+ std::shared_ptr<T> as() const {
+ constexpr Kind expected = ObjectKind<T>::value;
+ if (type_.kind != expected)
+ throw Error(SD_STATUS_TYPE_MISMATCH, "expected " + type_name({expected}) + ", got " + type_name(type_));
+ if (!object_)
+ throw std::logic_error("DGL object payload is missing");
+ return std::static_pointer_cast<T>(object_);
+ }
+ };
+
+ class Value::Cached {
+ Value value_;
+ std::weak_ptr<void> object_;
+
+ public:
+ Cached(Value value, bool retain);
+ std::optional<Value> lock() const;
+ bool expired() const { return object_.expired(); }
+ bool idle() const { return value_.object_.use_count() == 1; }
+ Type type() const { return value_.type(); }
+ };
+
+ using Values = std::map<std::string, Value>;
+ using ScriptFunction = std::function<Value(Value, const Value&)>;
+
+ template <>
+ struct ObjectKind<ScriptFunction> : std::integral_constant<Kind, Kind::Function> {};
+
+ bool is_model_kind(Kind kind);
+ int64_t get_int(const Values& args, const std::string& name);
+ double get_number(const Values& args, const std::string& name);
+ std::string get_string(const Values& args, const std::string& name);
+ bool get_bool(const Values& args, const std::string& name);
+
+} // namespace sd::dgl
+
+#endif // __SD_DGL_VALUE_H__
diff --git a/src/dgl/version.cpp b/src/dgl/version.cpp
new file mode 100644
index 0000000..a5e2022
--- /dev/null
+++ b/src/dgl/version.cpp
@@ -0,0 +1,35 @@
+#include "dgl/version.h"
+
+#include <charconv>
+#include <stdexcept>
+#include <tuple>
+
+namespace sd::dgl {
+ Version Version::parse(std::string_view text) {
+ constexpr const char* format_error = "DGL version must use MAJOR.MINOR.PATCH (for example, 0.0.0); integer and shortened versions are not supported";
+ if (text.empty())
+ throw std::invalid_argument(format_error);
+ Version version;
+ uint32_t* components[] = {&version.major, &version.minor, &version.patch};
+ const char* next = text.data();
+ const char* end = next + text.size();
+ for (size_t i = 0; i < 3; ++i) {
+ const auto parsed = std::from_chars(next, end, *components[i]);
+ if (parsed.ec == std::errc::result_out_of_range)
+ throw std::invalid_argument("DGL version component exceeds the unsigned 32-bit range");
+ if (parsed.ec != std::errc{} ||
+ (i < 2 ? parsed.ptr == end || *parsed.ptr != '.' : parsed.ptr != end))
+ throw std::invalid_argument(format_error);
+ next = parsed.ptr + (i < 2 ? 1 : 0);
+ }
+ return version;
+ }
+
+ std::string Version::str() const {
+ return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch);
+ }
+
+ bool Version::supports(const Version& required) const {
+ return major == required.major && std::tie(minor, patch) >= std::tie(required.minor, required.patch);
+ }
+}
diff --git a/src/dgl/version.h b/src/dgl/version.h
new file mode 100644
index 0000000..72d71f3
--- /dev/null
+++ b/src/dgl/version.h
@@ -0,0 +1,22 @@
+#ifndef __SD_DGL_VERSION_H__
+#define __SD_DGL_VERSION_H__
+
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+namespace sd::dgl {
+ struct Version {
+ uint32_t major = 0;
+ uint32_t minor = 0;
+ uint32_t patch = 0;
+
+ static Version parse(std::string_view text);
+ std::string str() const;
+ bool supports(const Version& required) const;
+ };
+
+ inline constexpr Version runtime_version{0, 0, 0};
+}
+
+#endif // __SD_DGL_VERSION_H__
diff --git a/src/extensions/generation_extension.h b/src/extensions/generation_extension.h
deleted file mode 100644
index 018092e..0000000
--- a/src/extensions/generation_extension.h
+++ /dev/null
@@ -1,79 +0,0 @@
-#ifndef __SD_EXTENSIONS_GENERATION_EXTENSION_H__
-#define __SD_EXTENSIONS_GENERATION_EXTENSION_H__
-
-#include <functional>
-#include <map>
-#include <memory>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "conditioning/conditioner.hpp"
-#include "config.h"
-#include "core/ggml_extend_backend.h"
-#include "model/diffusion/model.hpp"
-#include "model_loader.h"
-#include "model_manager.h"
-#include "stable-diffusion.h"
-
-struct GenerationExtensionInitContext {
- const sd::ContextConfig* params;
- SDVersion version;
- const String2TensorStorage& tensor_storage_map;
- bool photomaker_source_available;
- std::shared_ptr<ModelManager> model_manager;
- int n_threads;
- std::function<bool(SDBackendModule)> ensure_backend_pair;
- std::function<ggml_backend_t(SDBackendModule)> backend_for;
- std::function<ggml_backend_t(SDBackendModule)> params_backend_for;
-};
-
-struct GenerationExtensionConditionContext {
- Conditioner* conditioner;
- ConditionerParams& condition_params;
- const sd::PhotoMakerConfig& pm_params;
- const sd::PuLIDConfig& pulid_params;
- int n_threads;
- int total_steps;
-};
-
-struct GenerationExtension {
- virtual ~GenerationExtension() = default;
-
- virtual ModelComponent component() const = 0;
- const char* name() const { return model_component_name(component()); }
- virtual bool is_enabled() const {
- return false;
- }
- virtual bool init(const GenerationExtensionInitContext&) {
- return true;
- }
- virtual void get_param_tensors(std::map<std::string, ggml_tensor*>&) {}
- virtual void collect_loras(std::vector<ModelManager::LoraSpec>&) {}
- virtual void add_ignore_tensors(std::set<std::string>&) const {}
- virtual void runner_end() {}
- virtual void reset_runtime_condition() {}
- virtual bool prepare_condition(GenerationExtensionConditionContext&) {
- return false;
- }
- virtual const SDCondition& before_condition(int step,
- const SDCondition& condition) const {
- return condition;
- }
-
- // Called in the denoise loop for each enabled extension, after the per-step
- // DiffusionParams (including its version-specific `extra`) has been built,
- // but before diffusion_model->compute(). Lets an extension feed data into
- // the diffusion forward that the conditioning-side hooks can't reach -- it
- // can set/override fields on `params` (typically the architecture-specific
- // `params.extra`, e.g. a guidance tensor, control payload, or an identity
- // embedding for an adapter that injects inside the model's blocks). The
- // extension targets whichever `extra` variant matches the active model.
- // Mutates `params` only, never the extension. Default no-op.
- virtual void before_diffusion(DiffusionParams& /*params*/, int /*step*/) const {}
-};
-
-std::shared_ptr<GenerationExtension> create_photomaker_extension();
-std::shared_ptr<GenerationExtension> create_pulid_extension();
-
-#endif
diff --git a/src/extensions/photomaker_embedding.cpp b/src/extensions/photomaker_embedding.cpp
new file mode 100644
index 0000000..11691b9
--- /dev/null
+++ b/src/extensions/photomaker_embedding.cpp
@@ -0,0 +1,64 @@
+#include "photomaker_embedding.h"
+
+#include <algorithm>
+#include <cctype>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+
+#include "ggml.h"
+#include "model_io/npy_io.h"
+
+namespace sd::photomaker {
+
+ static Tensor<float> load_legacy_embedding(const std::filesystem::path& path,
+ std::vector<int64_t> shape,
+ size_t bytes) {
+ std::ifstream file(path, std::ios::binary | std::ios::ate);
+ if (!file)
+ throw std::runtime_error("cannot open PhotoMaker ID embedding");
+ const std::streamoff file_size = file.tellg();
+ file.seekg(0);
+
+ int32_t header[3];
+ if (!file.read(reinterpret_cast<char*>(header), sizeof(header)) ||
+ header[0] < 2 || header[0] > 4 || header[1] < 0 || header[2] != GGML_TYPE_F32) {
+ throw std::runtime_error("invalid legacy PhotoMaker ID embedding header");
+ }
+ int32_t dims[4]{1, 1, 1, 1};
+ if (!file.read(reinterpret_cast<char*>(dims), header[0] * sizeof(int32_t)) ||
+ dims[0] != shape[0] || dims[1] != shape[1] || dims[2] != 1 || dims[3] != 1) {
+ throw std::runtime_error("legacy PhotoMaker ID embedding shape does not match the input images");
+ }
+
+ const auto payload = std::streamoff(sizeof(header) + header[0] * sizeof(int32_t)) + header[1];
+ if (file_size < payload || uint64_t(file_size - payload) != bytes)
+ throw std::runtime_error("invalid legacy PhotoMaker ID embedding payload size");
+ file.seekg(payload);
+ Tensor<float> result(std::move(shape));
+ if (!file.read(reinterpret_cast<char*>(result.data()), bytes))
+ throw std::runtime_error("cannot read legacy PhotoMaker ID embedding");
+ return result;
+ }
+
+ Tensor<float> load_id_embedding(const std::string& path, int image_count) {
+ constexpr size_t bytes_per_image = id_embedding_width * sizeof(float);
+ if (image_count <= 0 || size_t(image_count) > std::numeric_limits<size_t>::max() / bytes_per_image)
+ throw std::runtime_error("invalid PhotoMaker input image count");
+ const auto bytes = size_t(image_count) * bytes_per_image;
+ std::vector<int64_t> shape{id_embedding_width, image_count};
+ const auto file_path = std::filesystem::u8path(path);
+ auto extension = file_path.extension().u8string();
+ std::transform(extension.begin(), extension.end(), extension.begin(), [](unsigned char c) { return char(std::tolower(c)); });
+ if (extension != ".npy")
+ return load_legacy_embedding(file_path, std::move(shape), bytes);
+
+ npy::Reader reader(path, bytes);
+ if (reader.type() != SD_TENSOR_F32 || reader.shape() != shape)
+ throw std::runtime_error("PhotoMaker v2 expects a float32 NumPy array with shape (image_count, 512)");
+ Tensor<float> result(std::move(shape));
+ reader.read(result.data(), bytes);
+ return result;
+ }
+
+} // namespace sd::photomaker
diff --git a/src/extensions/photomaker_embedding.h b/src/extensions/photomaker_embedding.h
new file mode 100644
index 0000000..befd8d7
--- /dev/null
+++ b/src/extensions/photomaker_embedding.h
@@ -0,0 +1,14 @@
+#ifndef __SD_EXTENSIONS_PHOTOMAKER_EMBEDDING_H__
+#define __SD_EXTENSIONS_PHOTOMAKER_EMBEDDING_H__
+
+#include "core/tensor.hpp"
+
+namespace sd::photomaker {
+
+ inline constexpr int64_t id_embedding_width = 512;
+
+ Tensor<float> load_id_embedding(const std::string& path, int image_count);
+
+} // namespace sd::photomaker
+
+#endif // __SD_EXTENSIONS_PHOTOMAKER_EMBEDDING_H__
diff --git a/src/extensions/photomaker_extension.cpp b/src/extensions/photomaker_extension.cpp
deleted file mode 100644
index 59b9a2c..0000000
--- a/src/extensions/photomaker_extension.cpp
+++ /dev/null
@@ -1,322 +0,0 @@
-#include <cinttypes>
-#include "extensions/generation_extension.h"
-
-#include <algorithm>
-#include <cstring>
-#include <tuple>
-#include <utility>
-
-#include "core/tensor_ggml.hpp"
-#include "core/util.h"
-#include "model/adapter/pmid.hpp"
-
-static std::tuple<std::vector<int>, std::vector<float>, std::vector<bool>>
-tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
- const std::string& text,
- int trigger_token_count,
- int32_t image_token) {
- auto tokens_and_weights = clip_conditioner.tokenize(text);
- std::vector<int> source_tokens = std::move(tokens_and_weights.first);
- std::vector<float> source_weights = std::move(tokens_and_weights.second);
- if (source_tokens.empty()) {
- return {};
- }
-
- if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer->BOS_TOKEN_ID) {
- source_tokens.erase(source_tokens.begin());
- source_weights.erase(source_weights.begin());
- }
- if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer->EOS_TOKEN_ID) {
- source_tokens.pop_back();
- source_weights.pop_back();
- }
-
- std::vector<int> tokens;
- std::vector<float> weights;
- int32_t class_idx = -1;
- for (size_t i = 0; i < source_tokens.size(); i++) {
- int token = source_tokens[i];
- if (token == image_token) {
- if (!tokens.empty()) {
- class_idx = static_cast<int32_t>(tokens.size()) - 1;
- int class_token = tokens.back();
- float class_weight = weights.back();
- for (int j = 1; j < trigger_token_count; j++) {
- tokens.push_back(class_token);
- weights.push_back(class_weight);
- }
- }
- continue;
- }
- tokens.push_back(token);
- weights.push_back(source_weights[i]);
- }
-
- clip_conditioner.tokenizer->pad_tokens(tokens,
- &weights,
- nullptr,
- clip_conditioner.text_model->model.n_token,
- clip_conditioner.text_model->model.n_token,
- true);
- std::vector<bool> class_token_mask;
- for (int i = 0; i < tokens.size(); i++) {
- class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
- }
-
- return std::make_tuple(tokens, weights, class_token_mask);
-}
-
-static std::tuple<SDCondition, std::vector<bool>>
-get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
- int n_threads,
- const ConditionerParams& conditioner_params,
- const std::string& trigger_word,
- int trigger_token_count) {
- std::vector<int> image_tokens;
- if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
- return {};
- }
- if (image_tokens.size() != 1) {
- LOG_ERROR("PhotoMaker trigger word must encode to one token");
- return {};
- }
- auto tokens_and_weights = tokenize_photomaker_trigger(clip_conditioner,
- conditioner_params.text,
- trigger_token_count,
- image_tokens[0]);
- std::vector<int>& tokens = std::get<0>(tokens_and_weights);
- std::vector<float>& weights = std::get<1>(tokens_and_weights);
- std::vector<bool>& trigger_mask = std::get<2>(tokens_and_weights);
- if (tokens.empty()) {
- return {};
- }
- auto cond = clip_conditioner.get_learned_condition_common(n_threads,
- tokens,
- weights,
- conditioner_params.clip_skip,
- conditioner_params.width,
- conditioner_params.height,
- conditioner_params.zero_out_masked);
- return std::make_tuple(std::move(cond), trigger_mask);
-}
-
-static bool remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
- const std::string& prompt,
- const std::string& trigger_word,
- std::string& result) {
- std::vector<int> image_tokens;
- if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
- return false;
- }
- if (image_tokens.size() != 1) {
- LOG_ERROR("PhotoMaker trigger word must encode to one token");
- return false;
- }
- auto tokens_and_weights = clip_conditioner.tokenize(prompt);
- std::vector<int>& tokens = tokens_and_weights.first;
- if (tokens.empty()) {
- return false;
- }
- auto it = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
- if (it == tokens.end()) {
- LOG_ERROR("PhotoMaker trigger word was not found in tokenized prompt");
- return false;
- }
- tokens.erase(it);
- return clip_conditioner.decode(tokens, result);
-}
-
-struct PhotoMakerExtension : public GenerationExtension {
- std::shared_ptr<PhotoMakerIDEncoder> pmid_model;
- bool enabled = false;
- std::string model_path;
- std::string trigger_word = "img";
- SDCondition id_condition;
- int start_merge_step = -1;
-
- ModelComponent component() const override {
- return ModelComponent::PhotoMaker;
- }
-
- bool is_enabled() const override {
- return enabled;
- }
-
- bool init(const GenerationExtensionInitContext& ctx) override {
- model_path = SAFE_STR(ctx.params->photo_maker_path);
- if (model_path.empty() || !ctx.photomaker_source_available) {
- return true;
- }
-
- if (!ctx.ensure_backend_pair(SDBackendModule::PHOTOMAKER)) {
- return false;
- }
-
- PMVersion pm_version = std::strstr(model_path.c_str(), "v2") != nullptr ? PM_VERSION_2 : PM_VERSION_1;
- pmid_model = std::make_shared<PhotoMakerIDEncoder>(ctx.backend_for(SDBackendModule::PHOTOMAKER),
- ctx.tensor_storage_map,
- "pmid",
- ctx.version,
- pm_version,
- 20.f,
- ctx.model_manager);
- pmid_model->set_scale_overrides(ctx.params->linear_scale, ctx.params->attn_scale);
- if (pm_version == PM_VERSION_2) {
- LOG_INFO("using PhotoMaker Version 2");
- }
-
- enabled = true;
- return true;
- }
-
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
- if (!enabled || pmid_model == nullptr) {
- return;
- }
-
- pmid_model->get_param_tensors(tensors, "pmid");
- }
-
- void collect_loras(std::vector<ModelManager::LoraSpec>& loras) override {
- if (!enabled || model_path.empty()) {
- return;
- }
- ModelManager::LoraSpec lora;
- lora.path = model_path;
- lora.multiplier = 1.0f;
- lora.tensor_name_prefix_filter = "lora.model";
- lora.required = true;
- loras.push_back(std::move(lora));
- }
-
- void add_ignore_tensors(std::set<std::string>& ignore_tensors) const override {
- if (!enabled) {
- return;
- }
- ignore_tensors.insert("pmid.unet.");
- }
-
- void runner_end() override {
- if (pmid_model != nullptr) {
- pmid_model->runner_end();
- }
- }
-
- void reset_runtime_condition() override {
- id_condition = {};
- start_merge_step = -1;
- }
-
- bool prepare_condition(GenerationExtensionConditionContext& ctx) override {
- reset_runtime_condition();
- if (!enabled || pmid_model == nullptr) {
- return false;
- }
-
- bool pmv2 = pmid_model->get_version() == PM_VERSION_2;
- if (ctx.pm_params.id_images_count <= 0 || ctx.pm_params.id_images == nullptr) {
- LOG_WARN("Provided PhotoMaker model file, but NO input ID images");
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
- auto* clip_conditioner = dynamic_cast<FrozenCLIPEmbedderWithCustomWords*>(ctx.conditioner);
- if (clip_conditioner == nullptr) {
- LOG_WARN("PhotoMaker requires FrozenCLIPEmbedderWithCustomWords conditioner");
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
-
- int clip_image_size = 224;
- pmid_model->style_strength = ctx.pm_params.style_strength;
- sd::Tensor<float> id_image_tensor;
- for (int i = 0; i < ctx.pm_params.id_images_count; i++) {
- auto id_image = sd_image_to_tensor(ctx.pm_params.id_images[i]);
- auto processed_id_image = clip_preprocess(id_image, clip_image_size, clip_image_size);
- if (id_image_tensor.empty()) {
- id_image_tensor = processed_id_image;
- } else {
- id_image_tensor = sd::ops::concat(id_image_tensor, processed_id_image, 3);
- }
- }
-
- int64_t t0 = ggml_time_ms();
- int trigger_token_count = pmv2 ? 2 * ctx.pm_params.id_images_count : ctx.pm_params.id_images_count;
- auto cond_tup = get_photomaker_condition_with_trigger(*clip_conditioner,
- ctx.n_threads,
- ctx.condition_params,
- trigger_word,
- trigger_token_count);
- SDCondition prepared_id_condition = std::get<0>(cond_tup);
- auto class_tokens_mask = std::get<1>(cond_tup);
- if (prepared_id_condition.empty()) {
- LOG_ERROR("failed to encode PhotoMaker prompt");
- return false;
- }
- if (std::find(class_tokens_mask.begin(), class_tokens_mask.end(), true) == class_tokens_mask.end()) {
- LOG_WARN("PhotoMaker trigger word '%s' was not found in prompt", trigger_word.c_str());
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
-
- sd::Tensor<float> id_embeds;
- if (pmv2 && ctx.pm_params.id_embed_path != nullptr) {
- try {
- id_embeds = sd::load_tensor_from_file_as_tensor<float>(ctx.pm_params.id_embed_path);
- } catch (const std::exception&) {
- id_embeds = {};
- }
- }
- if (pmv2 && id_embeds.empty()) {
- LOG_WARN("Provided PhotoMaker images, but NO valid ID embeds file for PM v2");
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
- if (pmv2 && ctx.pm_params.id_images_count != id_embeds.shape()[1]) {
- LOG_WARN("PhotoMaker image count (%d) does NOT match ID embeds (%d). You should run face_detect.py again.",
- ctx.pm_params.id_images_count,
- static_cast<int>(id_embeds.shape()[1]));
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
-
- auto res = pmid_model->compute(ctx.n_threads,
- id_image_tensor,
- prepared_id_condition.c_crossattn,
- id_embeds,
- class_tokens_mask);
- if (res.empty()) {
- LOG_ERROR("Photomaker ID Stacking failed");
- LOG_WARN("Turn off PhotoMaker for this request");
- return false;
- }
-
- prepared_id_condition.c_crossattn = std::move(res);
- int64_t t1 = ggml_time_ms();
- std::string prompt;
- if (!remove_photomaker_trigger_from_prompt(*clip_conditioner,
- ctx.condition_params.text,
- trigger_word,
- prompt)) {
- return false;
- }
- id_condition = std::move(prepared_id_condition);
- start_merge_step = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
- ctx.condition_params.text = std::move(prompt);
- LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
- LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step);
-
- return true;
- }
-
- const SDCondition& before_condition(int step,
- const SDCondition& condition) const override {
- if (!id_condition.empty() && start_merge_step != -1 && step > start_merge_step) {
- return id_condition;
- }
- return condition;
- }
-};
-
-std::shared_ptr<GenerationExtension> create_photomaker_extension() {
- return std::make_shared<PhotoMakerExtension>();
-}
diff --git a/src/extensions/pulid_extension.cpp b/src/extensions/pulid_embedding.cpp
similarity index 61%
rename from src/extensions/pulid_extension.cpp
rename to src/extensions/pulid_embedding.cpp
index fa660c3..4485bab 100644
--- a/src/extensions/pulid_extension.cpp
+++ b/src/extensions/pulid_embedding.cpp
@@ -1,13 +1,11 @@
-#include "extensions/generation_extension.h"
+#include "extensions/pulid_embedding.h"
#include <cstring>
-#include <variant>
-
#include "core/tensor_ggml.hpp"
#include "core/util.h"
#include "gguf.h"
-static sd::Tensor<float> load_pulid_id_embedding(const char* path) {
+sd::Tensor<float> load_pulid_id_embedding(const char* path) {
sd::Tensor<float> empty;
if (path == nullptr || strlen(path) == 0) {
return empty;
@@ -73,51 +71,3 @@ static sd::Tensor<float> load_pulid_id_embedding(const char* path) {
ggml_free(ctx_data);
return out;
}
-
-struct PuLIDExtension : public GenerationExtension {
- bool enabled = false;
- sd::Tensor<float> id_embedding;
- float id_weight = 1.0f;
-
- ModelComponent component() const override {
- return ModelComponent::PuLID;
- }
-
- bool is_enabled() const override {
- return enabled;
- }
-
- bool init(const GenerationExtensionInitContext& ctx) override {
- enabled = strlen(SAFE_STR(ctx.params->pulid_weights_path)) > 0;
- return true;
- }
-
- void reset_runtime_condition() override {
- id_embedding = {};
- id_weight = 1.0f;
- }
-
- bool prepare_condition(GenerationExtensionConditionContext& ctx) override {
- reset_runtime_condition();
- if (!enabled) {
- return false;
- }
- id_embedding = load_pulid_id_embedding(ctx.pulid_params.id_embedding_path);
- id_weight = ctx.pulid_params.id_weight;
- return false; // PuLID does not modify the conditioning
- }
-
- void before_diffusion(DiffusionParams& params, int /*step*/) const override {
- if (!enabled || id_embedding.empty()) {
- return;
- }
- if (auto* flux_extra = std::get_if<FluxDiffusionExtra>(¶ms.extra)) {
- flux_extra->pulid_id = &id_embedding;
- flux_extra->pulid_id_weight = id_weight;
- }
- }
-};
-
-std::shared_ptr<GenerationExtension> create_pulid_extension() {
- return std::make_shared<PuLIDExtension>();
-}
diff --git a/src/extensions/pulid_embedding.h b/src/extensions/pulid_embedding.h
new file mode 100644
index 0000000..8ff3c73
--- /dev/null
+++ b/src/extensions/pulid_embedding.h
@@ -0,0 +1,8 @@
+#ifndef __SD_EXTENSIONS_PULID_EMBEDDING_H__
+#define __SD_EXTENSIONS_PULID_EMBEDDING_H__
+
+#include "core/tensor.hpp"
+
+sd::Tensor<float> load_pulid_id_embedding(const char* path);
+
+#endif // __SD_EXTENSIONS_PULID_EMBEDDING_H__
diff --git a/src/generation_info.cpp b/src/generation_info.cpp
deleted file mode 100644
index f56ef98..0000000
--- a/src/generation_info.cpp
+++ /dev/null
@@ -1,235 +0,0 @@
-#include "generation_info.h"
-#include "config.h"
-#include "core/option_types.h"
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-
-#include "pipeline/request.h"
-#include "runtime/guidance.h"
-
-namespace sd {
- using nlohmann::json;
-
- static const char* text(const char* value) {
- return value ? value : "";
- }
-
- static double shortest_f32_as_double(float value) {
- if (!std::isfinite(value)) {
- return static_cast<double>(value);
- }
- char buf[64];
- buf[0] = '\0';
- for (int prec = 1; prec <= 9; ++prec) {
- std::snprintf(buf, sizeof(buf), "%.*g", prec, static_cast<double>(value));
- if (std::strtof(buf, nullptr) == value) {
- break;
- }
- }
- return std::strtod(buf, nullptr);
- }
-
- static json sampling_info(const sd::SampleConfig& p, bool high_noise = false) {
- const auto& g = p.guidance;
- json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}};
- if (!high_noise && p.shifted_timestep > 0)
- info["shifted_timestep"] = p.shifted_timestep;
- if (std::isfinite(g.img_cfg) && g.img_cfg != 1.f)
- info["guidance"]["img_cfg"] = g.img_cfg;
- if (p.sample_method != SAMPLE_METHOD_COUNT)
- info["sample_method"] = sd_sample_method_name(p.sample_method);
- if (!high_noise && p.scheduler != SCHEDULER_COUNT)
- info["scheduler"] = sd_scheduler_name(p.scheduler);
- if (std::isfinite(p.eta))
- info["eta"] = p.eta;
- if (!high_noise && std::isfinite(p.flow_shift))
- info["flow_shift"] = p.flow_shift;
- if (!high_noise && p.custom_sigmas && p.custom_sigmas_count > 0)
- info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
- if (p.extra_sample_args && *p.extra_sample_args)
- info["extra_sample_args"] = p.extra_sample_args;
- if (g.slg.layers && g.slg.layer_count > 0 &&
- (g.slg.scale != 0.f || guidance::parse_skip_layer_guidance_uncond_arg(p.extra_sample_args)))
- info["guidance"]["slg"] = {{"scale", g.slg.scale}, {"layer_start", g.slg.layer_start}, {"layer_end", g.slg.layer_end}, {"layers", std::vector<int>(g.slg.layers, g.slg.layers + g.slg.layer_count)}};
- return info;
- }
-
- static json hires_info(const sd::HiresConfig& p) {
- json info = {{"enabled", p.enabled}};
- if (!p.enabled)
- return info;
- info.update({{"upscaler", sd_hires_upscaler_name(p.upscaler)}, {"scale", p.scale}, {"target_width", p.target_width}, {"target_height", p.target_height}, {"steps", p.steps}, {"denoising_strength", p.denoising_strength}});
- if (p.upscaler == SD_HIRES_UPSCALER_MODEL) {
- info["model_path"] = text(p.model_path);
- info["upscale_tile_size"] = p.upscale_tile_size;
- }
- if (p.custom_sigmas && p.custom_sigmas_count > 0)
- info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
- return info;
- }
-
- static json cache_info(const sd::CacheConfig& p) {
- static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"};
- if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM)
- return {{"mode", static_cast<int>(p.mode)}};
- json info = {{"mode", modes[p.mode]}};
- if (p.mode == SD_CACHE_DISABLED)
- return info;
- if (p.mode == SD_CACHE_EASYCACHE || p.mode == SD_CACHE_UCACHE) {
- if (std::isfinite(p.reuse_threshold))
- info["reuse_threshold"] = p.reuse_threshold;
- info.update({{"start_percent", p.start_percent}, {"end_percent", p.end_percent}});
- if (p.mode == SD_CACHE_UCACHE)
- info.update({{"error_decay_rate", p.error_decay_rate}, {"use_relative_threshold", p.use_relative_threshold}, {"reset_error_on_compute", p.reset_error_on_compute}});
- } else if (p.mode == SD_CACHE_DBCACHE || p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT) {
- // TaylorSeer also uses the shared DBCache thresholds and limits.
- info.update({{"Fn_compute_blocks", p.Fn_compute_blocks},
- {"Bn_compute_blocks", p.Bn_compute_blocks},
- {"residual_diff_threshold", p.residual_diff_threshold},
- {"max_warmup_steps", p.max_warmup_steps},
- {"max_cached_steps", p.max_cached_steps},
- {"max_continuous_cached_steps", p.max_continuous_cached_steps}});
- if (p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT)
- info.update({{"taylorseer_n_derivatives", p.taylorseer_n_derivatives}, {"taylorseer_skip_interval", p.taylorseer_skip_interval}});
- if (p.scm_mask && *p.scm_mask)
- info.update({{"scm_mask", p.scm_mask}, {"scm_policy_dynamic", p.scm_policy_dynamic}});
- } else if (p.mode == SD_CACHE_SPECTRUM) {
- info.update({{"spectrum_w", p.spectrum_w},
- {"spectrum_m", p.spectrum_m},
- {"spectrum_lam", p.spectrum_lam},
- {"spectrum_window_size", p.spectrum_window_size},
- {"spectrum_flex_window", p.spectrum_flex_window},
- {"spectrum_warmup_steps", p.spectrum_warmup_steps},
- {"spectrum_stop_percent", p.spectrum_stop_percent}});
- }
- return info;
- }
-
- template <typename Params>
- static json common_info(const Params& p, const char* kind) {
- json info = {{"schema", "sdcpp.generation/v1"}, {"kind", kind}, {"prompt", text(p.prompt)}, {"negative_prompt", text(p.negative_prompt)}, {"width", p.width}, {"height", p.height}, {"clip_skip", p.clip_skip}, {"seed", std::to_string(p.seed)}, {"sample_params", sampling_info(p.sample_params)}};
- if (p.hires.enabled)
- info["hires"] = hires_info(p.hires);
- if (p.cache.mode != SD_CACHE_DISABLED)
- info["cache"] = cache_info(p.cache);
- if (p.circular_x)
- info["circular_x"] = true;
- if (p.circular_y)
- info["circular_y"] = true;
- if (p.ref_image_args && *p.ref_image_args)
- info["ref_image_args"] = p.ref_image_args;
- if (p.image_preprocess.rules && *p.image_preprocess.rules)
- info["image_preprocess"] = {{"rules", p.image_preprocess.rules}};
- const auto& t = p.vae_tiling_params;
- if (t.enabled || t.temporal_tiling) {
- auto& tiling = info["vae_tiling_params"];
- if (t.enabled)
- tiling.update({{"enabled", true}, {"tile_size_w", t.tile_size_w}, {"tile_size_h", t.tile_size_h}, {"target_overlap", t.target_overlap}, {"rel_size_w", t.rel_size_w}, {"rel_size_h", t.rel_size_h}});
- if (t.temporal_tiling)
- tiling["temporal_tiling"] = true;
- if (t.extra_tiling_args && *t.extra_tiling_args)
- tiling["extra_tiling_args"] = t.extra_tiling_args;
- }
- if (p.lora_count > 0) {
- info["loras"] = json::array();
- for (uint32_t i = 0; i < p.lora_count; ++i) {
- // nlohmann::json promotes float to double, so dump the shortest
- // decimal that round-trips the float32 (e.g. 0.85f stays 0.85
- // instead of 0.8500000238418579) to keep replay exact.
- const double multiplier = shortest_f32_as_double(p.loras[i].multiplier);
- info["loras"].push_back({{"path", text(p.loras[i].path)}, {"multiplier", multiplier}, {"is_high_noise", p.loras[i].is_high_noise}});
- }
- }
- return info;
- }
-
- static json image_info(const sd::ImageConfig& p) {
- auto info = common_info(p, "image");
- if (p.init_image.data)
- info["strength"] = p.strength;
- if (p.control_image.data) {
- info["control_strength"] = p.control_strength;
- if (p.control_type != SD_CONTROL_TYPE_NONE)
- info["control_type"] = sd_control_type_name(p.control_type);
- }
- if (p.ip_adapter_image.data)
- info["ip_adapter_strength"] = p.ip_adapter_strength;
- if (p.pm_params.id_images && p.pm_params.id_images_count > 0) {
- info["pm_params"] = {{"style_strength", p.pm_params.style_strength}};
- if (p.pm_params.id_embed_path && *p.pm_params.id_embed_path)
- info["pm_params"]["id_embed_path"] = p.pm_params.id_embed_path;
- }
- if (p.pulid_params.id_embedding_path && *p.pulid_params.id_embedding_path)
- info["pulid_params"] = {{"id_embedding_path", p.pulid_params.id_embedding_path}, {"id_weight", p.pulid_params.id_weight}};
- return info;
- }
-
- template <typename Params>
- static Params resolved_params(const pipeline::GenerationRequest& request, Params p) {
- p.width = request.width;
- p.height = request.height;
- p.seed = request.seed;
- p.clip_skip = request.clip_skip;
- p.sample_params = request.sample_params;
- p.hires = request.hires;
- p.vae_tiling_params = request.vae_tiling_params;
- return p;
- }
-
- json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params) {
- auto p = resolved_params(request, params);
- p.qwen_image_layers = request.qwen_image_layers;
- auto info = image_info(p);
- if (!request.models.empty())
- info["models"] = request.models;
- if (request.is_layered_image)
- info["qwen_image_layers"] = p.qwen_image_layers;
- return info;
- }
-
- json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params) {
- const auto p = resolved_params(request, params);
- auto info = common_info(p, "video");
- if (!request.models.empty())
- info["models"] = request.models;
- info.update({{"video_frames", request.frames}, {"fps", request.fps}});
- if (p.init_image.data || p.end_image.data)
- info["strength"] = p.strength;
- if (request.use_vace)
- info["vace_strength"] = p.vace_strength;
- if (request.use_high_noise)
- info.update({{"moe_boundary", p.moe_boundary}, {"high_noise_sample_params", sampling_info(request.high_noise_sample_params, true)}});
- if (p.ref_videos_count > 0) {
- info["ref_videos"] = json::array();
- for (int i = 0; i < p.ref_videos_count; ++i)
- info["ref_videos"].push_back({{"fps", p.ref_videos[i].fps}});
- }
- return info;
- }
-
- json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params) {
- auto inpaint = image_info(inpaint_params);
- inpaint["strength"] = inpaint_params.strength;
- // ADetailer clears these inputs before generating each crop.
- for (const auto* key : {"control_strength", "control_type", "pm_params", "pulid_params", "hires", "image_preprocess"})
- inpaint.erase(key);
- sd::ImageConfig defaults;
- if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers)
- inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers;
- json adetailer = json::object();
- if (params.prompt && *params.prompt)
- adetailer["prompt"] = params.prompt;
- if (params.negative_prompt && *params.negative_prompt)
- adetailer["negative_prompt"] = params.negative_prompt;
- if (params.extra_ad_args && *params.extra_ad_args)
- adetailer["extra_ad_args"] = params.extra_ad_args;
- return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"adetailer", adetailer}, {"inpaint_params", inpaint}};
- }
-
- json upscale_generation_info(uint32_t upscale_factor) {
- return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}};
- }
-
-} // namespace sd
diff --git a/src/generation_info.h b/src/generation_info.h
deleted file mode 100644
index d84673f..0000000
--- a/src/generation_info.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef __SD_GENERATION_INFO_H__
-#define __SD_GENERATION_INFO_H__
-
-#include <vector>
-
-#include "config.h"
-#include "json.hpp"
-#include "stable-diffusion.h"
-
-namespace sd::pipeline {
- struct GenerationRequest;
-}
-
-namespace sd {
- nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params);
- nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params);
- nlohmann::json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params);
- nlohmann::json upscale_generation_info(uint32_t upscale_factor);
-
-}
-
-#endif // __SD_GENERATION_INFO_H__
diff --git a/src/model.h b/src/model.h
index 2184f39..85dfb68 100644
--- a/src/model.h
+++ b/src/model.h
@@ -72,6 +72,8 @@ enum SDVersion {
VERSION_CHROMA,
VERSION_P_H3_IMAGE,
VERSION_MINIMAX_H3_IMAGE,
+ VERSION_RELAY_VSR,
+ VERSION_SEEDVR2,
VERSION_COUNT,
};
diff --git a/src/model/adapter/ip_adapter.hpp b/src/model/adapter/ip_adapter.hpp
index b36977d..0858606 100644
--- a/src/model/adapter/ip_adapter.hpp
+++ b/src/model/adapter/ip_adapter.hpp
@@ -115,17 +115,39 @@ namespace IPAdapter {
}
};
+ struct AttentionKV {
+ sd::Tensor<float> key;
+ sd::Tensor<float> value;
+ };
+
+ using AttentionCache = std::map<std::string, AttentionKV>;
+
struct IPAdapterRunner : public GGMLRunner {
+ struct AttentionProjection {
+ Linear key;
+ Linear value;
+ int64_t width;
+
+ AttentionProjection(int64_t input_width, int64_t output_width)
+ : key(input_width, output_width, false), value(input_width, output_width, false), width(output_width) {}
+ };
+
ImageProjModel image_proj;
Resampler resampler;
bool is_plus = false;
int64_t num_tokens = 4;
std::string prefix;
+ std::map<std::string, AttentionProjection> attention;
+
+ int64_t token_width() const {
+ return is_plus ? resampler.output_dim : image_proj.ctx_dim;
+ }
IPAdapterRunner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map,
const std::string prefix,
- std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+ std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+ bool load_attention = false)
: GGMLRunner(backend, weight_manager), prefix(prefix) {
is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end();
if (is_plus) {
@@ -175,6 +197,32 @@ namespace IPAdapter {
image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
}
+ if (load_attention) {
+ init_attention(tensor_storage_map);
+ }
+ }
+
+ void init_attention(const String2TensorStorage& weights) {
+ const std::string layer_suffix = ".attn2.to_k_ip.weight";
+ const std::string weight_suffix = ".to_k_ip.weight";
+ for (const auto& [name, key] : weights) {
+ if (name.size() <= layer_suffix.size() ||
+ name.compare(name.size() - layer_suffix.size(), layer_suffix.size(), layer_suffix) != 0)
+ continue;
+ auto layer = name.substr(0, name.size() - weight_suffix.size());
+ const auto value = weights.find(layer + ".to_v_ip.weight");
+ if (value == weights.end() || key.n_dims != 2 || value->second.n_dims != 2 ||
+ key.logical_ne0() != token_width() || value->second.logical_ne0() != token_width() ||
+ key.ne[1] <= 0 || key.ne[1] != value->second.ne[1]) {
+ throw std::runtime_error("invalid IP-Adapter key/value weights for '" + layer + "'");
+ }
+ auto& projection = attention.try_emplace(layer, token_width(), key.ne[1]).first->second;
+ projection.key.init(params_ctx, weights, layer + ".to_k_ip");
+ projection.value.init(params_ctx, weights, layer + ".to_v_ip");
+ }
+ if (attention.empty()) {
+ throw std::runtime_error("IP-Adapter has no attention weights for the selected family");
+ }
}
std::string get_desc() override {
@@ -187,6 +235,43 @@ namespace IPAdapter {
} else {
image_proj.get_param_tensors(tensors, prefix + ".image_proj");
}
+ for (auto& [layer, projection] : attention) {
+ projection.key.get_param_tensors(tensors, layer + ".to_k_ip");
+ projection.value.get_param_tensors(tensors, layer + ".to_v_ip");
+ }
+ }
+
+ AttentionCache compute_attention(int n_threads, const sd::Tensor<float>& tokens) {
+ std::map<std::string, std::pair<ggml_tensor*, ggml_tensor*>> outputs;
+ auto get_graph = [&]() {
+ auto* graph = new_graph_custom(1024 + 32 * attention.size());
+ auto* input = make_input(tokens);
+ auto ctx = get_context();
+ for (auto& [layer, projection] : attention) {
+ auto* key = projection.key.forward(&ctx, input);
+ auto* value = projection.value.forward(&ctx, input);
+ ggml_set_output(key);
+ ggml_set_output(value);
+ ggml_build_forward_expand(graph, key);
+ ggml_build_forward_expand(graph, value);
+ outputs.insert_or_assign(layer, std::make_pair(key, value));
+ }
+ return graph;
+ };
+ AttentionCache cache;
+ auto read_outputs = [&]() {
+ for (const auto& [layer, kv] : outputs) {
+ auto key = sd::make_sd_tensor_from_ggml<float>(kv.first);
+ auto value = sd::make_sd_tensor_from_ggml<float>(kv.second);
+ if (key.empty() || value.empty())
+ return false;
+ cache.emplace(layer, AttentionKV{std::move(key), std::move(value)});
+ }
+ return true;
+ };
+ if (!GGMLRunner::compute(get_graph, n_threads, true, true, read_outputs).has_value())
+ return {};
+ return cache;
}
ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index 7d71b34..168b68e 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -40,18 +40,23 @@ struct LoraModel : public GGMLRunner {
typedef std::function<bool(const std::string&)> filter_t;
+ const std::shared_ptr<ModelManager::ResourceHandle> weights;
+
LoraModel(const std::string& id, ggml_backend_t backend, ggml_backend_t params, std::shared_ptr<ModelManager> manager, ModelLoader::FileId file, SDVersion version, ModelManager::ResidencyMode mode = ModelManager::ResidencyMode::ParamBackend, bool follow_compute = false)
- : GGMLRunner(backend, manager), lora_id(id), params_backend(params), source_file(file), source_version(version), residency_mode(mode), params_follow_compute(follow_compute) {
- load_failed = source_file == 0 || manager == nullptr || manager->loader().file_revision(source_file) == 0;
+ : LoraModel(id, backend, params, manager ? std::make_shared<ModelManager::ResourceHandle>(std::move(manager), 0) : nullptr, file, version, mode, follow_compute) {}
+
+ LoraModel(const std::string& id, ggml_backend_t backend, ggml_backend_t params, std::shared_ptr<ModelManager::ResourceHandle> resource, ModelLoader::FileId file, SDVersion version, ModelManager::ResidencyMode mode = ModelManager::ResidencyMode::ParamBackend, bool follow_compute = false)
+ : GGMLRunner(backend, resource), lora_id(id), params_backend(params), source_file(file), source_version(version), residency_mode(mode), params_follow_compute(follow_compute), weights(std::move(resource)) {
+ load_failed = source_file == 0 || weights == nullptr || weights->loader().file_revision(source_file) == 0;
if (!load_failed) {
- file_path = manager->loader().file_path(source_file);
+ file_path = weights->loader().file_path(source_file);
}
}
~LoraModel() override {
runner_end();
- if (auto manager = std::dynamic_pointer_cast<ModelManager>(residency_manager.lock())) {
- GGML_ASSERT(manager->unregister_param_tensors(registered_params));
+ if (weights) {
+ GGML_ASSERT(weights->unregister_param_tensors(registered_params));
}
}
@@ -59,15 +64,16 @@ struct LoraModel : public GGMLRunner {
return "lora";
}
+ ggml_backend_t backend() const { return runtime_backend; }
+
bool init_params(int n_threads, filter_t filter = nullptr) {
- auto model_manager = std::dynamic_pointer_cast<ModelManager>(residency_manager.lock());
- if (model_manager == nullptr)
+ if (!weights)
return false;
if (load_failed || !registered_params.empty())
return false;
- model_manager->set_n_threads(n_threads);
+ weights->set_n_threads(n_threads);
metadata_alpha = -1.f;
- const auto metadata = model_manager->loader().file_metadata(source_file);
+ const auto metadata = weights->loader().file_metadata(source_file);
const auto alpha = metadata.find("alpha");
// Converters that write "baked_scale" have already folded alpha / rank into lora_up.
if (alpha != metadata.end() && metadata.find("baked_scale") == metadata.end()) {
@@ -84,7 +90,7 @@ struct LoraModel : public GGMLRunner {
LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha);
}
if (sd_version_is_minimax_h3(source_version)) {
- for (const auto& entry : model_manager->loader().file_tensors(source_file)) {
+ for (const auto& entry : weights->loader().file_tensors(source_file)) {
const auto name = convert_tensor_name(entry.first, VERSION_COUNT);
if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) {
// Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value].
@@ -92,7 +98,7 @@ struct LoraModel : public GGMLRunner {
}
}
}
- const auto sources = model_manager->loader().file_tensors(source_file, source_version);
+ const auto sources = weights->loader().file_tensors(source_file, source_version);
std::map<std::string, ggml_tensor*> tensors;
std::map<std::string, ggml_tensor*> scalars;
std::set<std::string> scalar_names;
@@ -117,16 +123,16 @@ struct LoraModel : public GGMLRunner {
*dst = found == scalars.end() ? nullptr : found->second;
return true;
};
- if (!model_manager->loader().load_file_tensors(source_file, source_version, callback, scalar_names))
+ if (!weights->loader().load_file_tensors(source_file, source_version, callback, scalar_names))
return false;
}
- if (!model_manager->register_param_tensors(ModelComponent::LoRA, tensors, residency_mode,
- runtime_backend, params_backend, nullptr, false, params_follow_compute,
- nullptr, source_file, source_version))
+ if (!weights->register_param_tensors(ModelComponent::LoRA, tensors, residency_mode,
+ runtime_backend, params_backend, nullptr, false, params_follow_compute,
+ nullptr, source_file, source_version))
return false;
for (const auto& entry : tensors)
registered_params.push_back(entry.second);
- return model_manager->validate_registered_tensors();
+ return weights->validate_registered_tensors();
}
float scalar_value(ggml_tensor* tensor) const {
@@ -136,8 +142,8 @@ struct LoraModel : public GGMLRunner {
void release_loaded_tensors() {
runner_end();
- if (auto manager = std::dynamic_pointer_cast<ModelManager>(residency_manager.lock())) {
- GGML_ASSERT(manager->unregister_param_tensors(registered_params));
+ if (weights) {
+ GGML_ASSERT(weights->unregister_param_tensors(registered_params));
}
registered_params.clear();
free_params_ctx();
diff --git a/src/model/adapter/pmid.hpp b/src/model/adapter/pmid.hpp
index 163d256..6116cb5 100644
--- a/src/model/adapter/pmid.hpp
+++ b/src/model/adapter/pmid.hpp
@@ -320,18 +320,13 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
blocks["fuse_module"] = std::shared_ptr<GGMLBlock>(new FuseModule(2048));
}
- ggml_tensor* forward(GGMLRunnerContext* ctx,
- ggml_tensor* id_pixel_values,
- ggml_tensor* prompt_embeds,
- ggml_tensor* class_tokens_mask,
- ggml_tensor* class_tokens_mask_pos,
- ggml_tensor* left,
- ggml_tensor* right) {
+ FuseModule& fusion() { return *std::static_pointer_cast<FuseModule>(blocks.at("fuse_module")); }
+
+ ggml_tensor* project(GGMLRunnerContext* ctx, ggml_tensor* id_pixel_values) {
// x: [N, channels, h, w]
auto vision_model = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
auto visual_projection = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
auto visual_projection_2 = std::dynamic_pointer_cast<Linear>(blocks["visual_projection_2"]);
- auto fuse_module = std::dynamic_pointer_cast<FuseModule>(blocks["fuse_module"]);
ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size]
ggml_tensor* id_embeds = visual_projection->forward(ctx, shared_id_embeds); // [N, proj_dim(768)]
@@ -343,13 +338,7 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection {
id_embeds = ggml_concat(ctx->ggml_ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
id_embeds = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, id_embeds, 1, 2, 0, 3));
- ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
- prompt_embeds,
- id_embeds,
- class_tokens_mask,
- class_tokens_mask_pos,
- left, right);
- return updated_prompt_embeds;
+ return id_embeds;
}
};
@@ -368,30 +357,16 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo
num_tokens));
}
- ggml_tensor* forward(GGMLRunnerContext* ctx,
- ggml_tensor* id_pixel_values,
- ggml_tensor* prompt_embeds,
- ggml_tensor* class_tokens_mask,
- ggml_tensor* class_tokens_mask_pos,
- ggml_tensor* id_embeds,
- ggml_tensor* left,
- ggml_tensor* right) {
+ FuseModule& fusion() { return *std::static_pointer_cast<FuseModule>(blocks.at("fuse_module")); }
+
+ ggml_tensor* project(GGMLRunnerContext* ctx, ggml_tensor* id_pixel_values, ggml_tensor* id_embeds) {
// x: [N, channels, h, w]
auto vision_model = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
- auto fuse_module = std::dynamic_pointer_cast<FuseModule>(blocks["fuse_module"]);
auto qformer_perceiver = std::dynamic_pointer_cast<QFormerPerceiver>(blocks["qformer_perceiver"]);
// ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size]
ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false); // [N, hidden_size]
- id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state);
-
- ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx,
- prompt_embeds,
- id_embeds,
- class_tokens_mask,
- class_tokens_mask_pos,
- left, right);
- return updated_prompt_embeds;
+ return qformer_perceiver->forward(ctx, id_embeds, last_hidden_state);
}
};
@@ -439,7 +414,7 @@ public:
return pm_version;
}
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix = "pmid") {
if (pm_version == PM_VERSION_1)
id_encoder.get_param_tensors(tensors, prefix);
else if (pm_version == PM_VERSION_2)
@@ -448,7 +423,7 @@ public:
ggml_cgraph* build_graph(const sd::Tensor<float>& id_pixel_values_tensor,
const sd::Tensor<float>& prompt_embeds_tensor,
- std::vector<bool>& class_tokens_mask,
+ const std::vector<bool>& class_tokens_mask,
const sd::Tensor<float>& id_embeds_tensor = {}) {
ctm.clear();
ctmf16.clear();
@@ -462,9 +437,8 @@ public:
ggml_cgraph* gf = ggml_new_graph(compute_ctx);
- ggml_tensor* id_pixel_values = make_input(id_pixel_values_tensor);
- ggml_tensor* prompt_embeds = make_input(prompt_embeds_tensor);
- ggml_tensor* id_embeds = make_optional_input(id_embeds_tensor);
+ ggml_tensor* prompt_embeds = make_input(prompt_embeds_tensor);
+ ggml_tensor* id_embeds = make_optional_input(id_embeds_tensor);
int64_t hidden_size = prompt_embeds->ne[0];
int64_t seq_length = prompt_embeds->ne[1];
@@ -528,23 +502,12 @@ public:
}
}
}
- ggml_tensor* updated_prompt_embeds = nullptr;
- if (pm_version == PM_VERSION_1)
- updated_prompt_embeds = id_encoder.forward(&runner_ctx,
- id_pixel_values,
- prompt_embeds,
- class_tokens_mask_d,
- class_tokens_mask_pos,
- left, right);
- else if (pm_version == PM_VERSION_2)
- updated_prompt_embeds = id_encoder2.forward(&runner_ctx,
- id_pixel_values,
- prompt_embeds,
- class_tokens_mask_d,
- class_tokens_mask_pos,
- id_embeds,
- left, right);
-
+ if (!id_pixel_values_tensor.empty()) {
+ id_embeds = project(runner_ctx, make_input(id_pixel_values_tensor), id_embeds);
+ }
+ auto& fusion = pm_version == PM_VERSION_1 ? id_encoder.fusion() : id_encoder2.fusion();
+ auto* updated_prompt_embeds = fusion.forward(&runner_ctx, prompt_embeds, id_embeds,
+ class_tokens_mask_d, class_tokens_mask_pos, left, right);
ggml_build_forward_expand(gf, updated_prompt_embeds);
return gf;
@@ -554,13 +517,30 @@ public:
const sd::Tensor<float>& id_pixel_values,
const sd::Tensor<float>& prompt_embeds,
const sd::Tensor<float>& id_embeds,
- std::vector<bool>& class_tokens_mask) {
+ const std::vector<bool>& class_tokens_mask) {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds);
};
return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true));
}
+
+ sd::Tensor<float> compute_tokens(int n_threads, const sd::Tensor<float>& pixels, const sd::Tensor<float>& embedding) {
+ auto get_graph = [&]() {
+ auto* graph = new_graph_custom(4096);
+ auto ctx = get_context();
+ auto* tokens = project(ctx, make_input(pixels), make_optional_input(embedding));
+ tokens = ggml_reshape_2d(ctx.ggml_ctx, tokens, tokens->ne[0], ggml_nelements(tokens) / tokens->ne[0]);
+ ggml_build_forward_expand(graph, tokens);
+ return graph;
+ };
+ return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true));
+ }
+
+private:
+ ggml_tensor* project(GGMLRunnerContext& ctx, ggml_tensor* pixels, ggml_tensor* embedding) {
+ return pm_version == PM_VERSION_1 ? id_encoder.project(&ctx, pixels) : id_encoder2.project(&ctx, pixels, embedding);
+ }
};
#endif // __SD_MODEL_ADAPTER_PMID_HPP__
diff --git a/src/model/adapter/pulid.hpp b/src/model/adapter/pulid.hpp
index 3ba84fb..3dd2c04 100644
--- a/src/model/adapter/pulid.hpp
+++ b/src/model/adapter/pulid.hpp
@@ -40,17 +40,22 @@ public:
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* id_embedding,
ggml_tensor* image_tokens) {
- auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm1"]);
- auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
- auto to_q = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
- auto to_kv = std::dynamic_pointer_cast<Linear>(blocks["to_kv"]);
- auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out"]);
+ return attend(ctx, project_kv(ctx, id_embedding), image_tokens);
+ }
+
+ ggml_tensor* project_kv(GGMLRunnerContext* ctx, ggml_tensor* id_embedding) {
+ auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm1"]);
+ auto to_kv = std::dynamic_pointer_cast<Linear>(blocks["to_kv"]);
+ return to_kv->forward(ctx, norm1->forward(ctx, id_embedding));
+ }
- ggml_tensor* x_normed = norm1->forward(ctx, id_embedding);
+ ggml_tensor* attend(GGMLRunnerContext* ctx, ggml_tensor* kv, ggml_tensor* image_tokens) {
+ auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
+ auto to_q = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
+ auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out"]);
ggml_tensor* lat_normed = norm2->forward(ctx, image_tokens);
- ggml_tensor* q = to_q->forward(ctx, lat_normed); // [N, T_img, 2048]
- ggml_tensor* kv = to_kv->forward(ctx, x_normed); // [N, T_img, 3072]
+ ggml_tensor* q = to_q->forward(ctx, lat_normed); // [N, T_img, 2048]
ggml_tensor* k = ggml_view_3d(ctx->ggml_ctx, kv,
inner_dim, kv->ne[1], kv->ne[2],
@@ -74,4 +79,82 @@ public:
}
};
+class PuLIDRunner : public GGMLRunner {
+ std::vector<PuLIDPerceiverAttentionCA> layers_;
+
+ static std::string layer_name(size_t index) {
+ return "model.diffusion_model.pulid_ca." + std::to_string(index);
+ }
+
+public:
+ static constexpr int double_blocks = 19;
+ static constexpr int single_blocks = 38;
+ static constexpr int double_interval = 2;
+ static constexpr int single_interval = 4;
+ static constexpr int double_layers = (double_blocks + double_interval - 1) / double_interval;
+ static constexpr int layer_count = double_layers + (single_blocks + single_interval - 1) / single_interval;
+
+ PuLIDRunner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<RunnerWeightManager> manager)
+ : GGMLRunner(backend, std::move(manager)), layers_(layer_count) {
+ for (size_t index = 0; index < layers_.size(); ++index) {
+ const auto name = layer_name(index);
+ if (!weights.count(name + ".to_kv.weight"))
+ throw std::runtime_error("missing PuLID weights for '" + name + "'");
+ layers_[index].init(params_ctx, weights, name);
+ }
+ if (weights.count(layer_name(layers_.size()) + ".to_kv.weight"))
+ throw std::runtime_error("unsupported PuLID attention layer count");
+ }
+
+ std::string get_desc() override { return "pulid"; }
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+ size_t extra_graph_size() const {
+ return layer_count * 32 + (weight_adapter ? weight_adapter->get_extra_graph_size() : 0);
+ }
+
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
+ for (size_t index = 0; index < layers_.size(); ++index)
+ layers_[index].get_param_tensors(tensors, layer_name(index));
+ }
+
+ std::vector<sd::Tensor<float>> compute_kv(int n_threads, const sd::Tensor<float>& embedding) {
+ std::vector<ggml_tensor*> outputs(layers_.size());
+ auto get_graph = [&]() {
+ auto* graph = new_graph_custom(1024);
+ auto ctx = get_context();
+ auto* identity = make_input(embedding);
+ for (size_t i = 0; i < layers_.size(); ++i) {
+ outputs[i] = layers_[i].project_kv(&ctx, identity);
+ ggml_set_output(outputs[i]);
+ ggml_build_forward_expand(graph, outputs[i]);
+ }
+ return graph;
+ };
+ std::vector<sd::Tensor<float>> kv;
+ auto read_outputs = [&]() {
+ kv.reserve(layers_.size());
+ for (const auto* output : outputs) {
+ auto tensor = sd::make_sd_tensor_from_ggml<float>(output);
+ if (tensor.empty())
+ return false;
+ kv.push_back(std::move(tensor));
+ }
+ return true;
+ };
+ if (!GGMLRunner::compute(get_graph, n_threads, true, true, read_outputs).has_value())
+ return {};
+ return kv;
+ }
+
+ ggml_tensor* attend(GGMLRunnerContext& ctx, size_t layer, const sd::Tensor<float>& kv, ggml_tensor* image) {
+ auto branch = ctx;
+ branch.weight_adapter = weight_adapter;
+ branch.linear_scale = linear_scale;
+ branch.attn_scale = attn_scale;
+ auto* input = sd::make_ggml_tensor(ctx.ggml_ctx, kv, false);
+ ctx.set_backend_tensor_data(input, kv.data());
+ return layers_[layer].attend(&branch, input, image);
+ }
+};
+
#endif // __PULID_HPP__
diff --git a/src/model/common/block.hpp b/src/model/common/block.hpp
index bed820c..d8e193a 100644
--- a/src/model/common/block.hpp
+++ b/src/model/common/block.hpp
@@ -310,9 +310,13 @@ protected:
bool xtra_dim = false;
bool enable_ip = false;
bool has_ip = false;
+ std::string layer_name;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+ if (enable_ip && !prefix.empty()) {
+ layer_name = prefix.substr(0, prefix.size() - 1);
+ }
if (enable_ip &&
tensor_storage_map.find(prefix + "to_k_ip.weight") != tensor_storage_map.end()) {
has_ip = true;
@@ -384,6 +388,9 @@ public:
auto x_ip = ggml_ext_attention_ext(ctx, q, k_ip, v_ip, n_head, nullptr, false, ctx->flash_attn_enabled);
x = ggml_add(ctx->ggml_ctx, x, ggml_scale(ctx->ggml_ctx, x_ip, ctx->ip_scale));
}
+ if (enable_ip && ctx->attn2_processor) {
+ x = ctx->attn2_processor->apply(*ctx, layer_name, q, x, n_head);
+ }
x = to_out_0->forward(ctx, x); // [N, n_token, query_dim]
return x;
diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp
index 805b868..0342d6d 100644
--- a/src/model/common/ggml_block.hpp
+++ b/src/model/common/ggml_block.hpp
@@ -6,6 +6,7 @@
#include <map>
#include <memory>
#include <set>
+#include <stdexcept>
#include <string>
#include <tuple>
#include <unordered_map>
@@ -24,6 +25,17 @@ protected:
GGMLBlockMap blocks;
ParameterMap params;
+ static int64_t weight_scale_elements(const TensorStorage& storage, int64_t features, const std::string& name) {
+ const auto elements = storage.nelements();
+ if (elements != 1 && elements != features) {
+ throw std::runtime_error("unsupported weight scale for '" + name + "': [" +
+ std::to_string(storage.ne[0]) + ", " + std::to_string(storage.ne[1]) + ", " +
+ std::to_string(storage.ne[2]) + ", " + std::to_string(storage.ne[3]) +
+ "], expected a scalar or " + std::to_string(features) + " output scales");
+ }
+ return elements;
+ }
+
ggml_type get_type(const std::string& name, const String2TensorStorage& tensor_storage_map, ggml_type default_type) {
ggml_type wtype = default_type;
auto iter = tensor_storage_map.find(name);
@@ -178,10 +190,9 @@ protected:
const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
auto weight_scale_storage = tensor_storage_map.find(prefix + "weight_scale");
if (weight_scale_storage != tensor_storage_map.end()) {
- const int64_t scale_nelements = weight_scale_storage->second.nelements();
- GGML_ASSERT(scale_nelements == 1 || scale_nelements == out_features);
- params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
- has_weight_scale = true;
+ const int64_t scale_nelements = weight_scale_elements(weight_scale_storage->second, out_features, prefix + "weight_scale");
+ params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
+ has_weight_scale = true;
}
if (w4_convrot_kind == 2) {
params["weight_codebook"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 16);
@@ -340,9 +351,8 @@ protected:
GGML_ASSERT(wtype == GGML_TYPE_I8);
auto scale_storage = tensor_storage_map.find(prefix + "weight_scale");
GGML_ASSERT(scale_storage != tensor_storage_map.end());
- const int64_t scale_nelements = scale_storage->second.nelements();
- GGML_ASSERT(scale_nelements == 1 || scale_nelements == num_embeddings);
- params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
+ const int64_t scale_nelements = weight_scale_elements(scale_storage->second, num_embeddings, prefix + "weight_scale");
+ params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
if (weight_storage->second.int8_convrot) {
int8_convrot_group_size = weight_storage->second.int8_convrot_group_size;
int remainder = int8_convrot_group_size;
@@ -478,8 +488,9 @@ public:
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
- ggml_tensor* w = params["weight"];
- ggml_tensor* b = nullptr;
+ const float scale = ctx->conv_scale > 0.f ? ctx->conv_scale : this->scale;
+ ggml_tensor* w = params["weight"];
+ ggml_tensor* b = nullptr;
if (bias) {
b = params["bias"];
}
@@ -565,8 +576,9 @@ public:
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
- ggml_tensor* w = params["weight"];
- ggml_tensor* b = nullptr;
+ const float scale = ctx->conv_scale > 0.f ? ctx->conv_scale : this->scale;
+ ggml_tensor* w = params["weight"];
+ ggml_tensor* b = nullptr;
if (bias) {
b = params["bias"];
}
diff --git a/src/model/common/ltx.hpp b/src/model/common/ltx.hpp
new file mode 100644
index 0000000..c66a1e7
--- /dev/null
+++ b/src/model/common/ltx.hpp
@@ -0,0 +1,297 @@
+#ifndef __SD_MODEL_COMMON_LTX_HPP__
+#define __SD_MODEL_COMMON_LTX_HPP__
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "core/ggml_tensor_utils.h"
+#include "model/common/block.hpp"
+#include "model/common/rope.hpp"
+#include "model_loader.h"
+
+namespace LTXV {
+
+ constexpr int LTXAV_GRAPH_SIZE = 102400;
+
+ __STATIC_INLINE__ ggml_tensor* rms_norm(ggml_context* ctx,
+ ggml_tensor* x,
+ float eps = 1e-6f) {
+ return ggml_rms_norm(ctx, x, eps);
+ }
+
+ __STATIC_INLINE__ int count_prefix_blocks(const String2TensorStorage& tensor_storage_map,
+ const std::string& prefix,
+ const std::string& marker) {
+ int max_block = -1;
+ for (const auto& [name, _] : tensor_storage_map) {
+ if (!starts_with(name, prefix)) {
+ continue;
+ }
+ size_t pos = name.find(marker);
+ if (pos == std::string::npos) {
+ continue;
+ }
+ pos += marker.size();
+ size_t end = name.find(".", pos);
+ if (end == std::string::npos) {
+ continue;
+ }
+ int block = atoi(name.substr(pos, end - pos).c_str());
+ max_block = std::max(max_block, block);
+ }
+ return max_block + 1;
+ }
+
+ __STATIC_INLINE__ std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
+ int64_t preferred_heads = -1) {
+ if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
+ return {preferred_heads, hidden_size / preferred_heads};
+ }
+ const int candidates[] = {128, 96, 80, 64, 48, 40, 32};
+ for (int head_dim : candidates) {
+ if (hidden_size % head_dim == 0) {
+ int64_t heads = hidden_size / head_dim;
+ if (heads >= 8 && heads <= 64) {
+ return {heads, head_dim};
+ }
+ }
+ }
+ return {32, hidden_size / 32};
+ }
+
+ __STATIC_INLINE__ int64_t infer_gate_heads(const String2TensorStorage& tensor_storage_map,
+ const std::string& bias_name,
+ int64_t fallback_heads) {
+ auto it = tensor_storage_map.find(bias_name);
+ if (it != tensor_storage_map.end()) {
+ return it->second.ne[0];
+ }
+ return fallback_heads;
+ }
+
+ __STATIC_INLINE__ std::vector<float> generate_freq_grid(float theta,
+ int positional_dims,
+ int dim) {
+ const int n_elem = 2 * positional_dims;
+ const int freq_count = dim / n_elem;
+
+ std::vector<float> out(freq_count);
+ if (freq_count <= 0) {
+ return out;
+ }
+ if (freq_count == 1) {
+ out[0] = 1.5707963267948966f;
+ return out;
+ }
+
+ const float half_pi = 1.5707963267948966f;
+ const float log_theta = std::log(theta);
+ for (int i = 0; i < freq_count; i++) {
+ float ratio = static_cast<float>(i) / static_cast<float>(freq_count - 1);
+ out[i] = std::exp(log_theta * ratio) * half_pi;
+ }
+ return out;
+ }
+
+ __STATIC_INLINE__ std::vector<double> generate_freq_grid_double(double theta,
+ int positional_dims,
+ int dim) {
+ const int n_elem = 2 * positional_dims;
+ const int freq_count = dim / n_elem;
+
+ std::vector<double> out(freq_count);
+ if (freq_count <= 0) {
+ return out;
+ }
+ if (freq_count == 1) {
+ out[0] = 1.5707963267948966;
+ return out;
+ }
+
+ const double half_pi = 1.5707963267948966;
+ const double log_theta = std::log(theta);
+ for (int i = 0; i < freq_count; i++) {
+ double ratio = static_cast<double>(i) / static_cast<double>(freq_count - 1);
+ out[i] = std::exp(log_theta * ratio) * half_pi;
+ }
+ return out;
+ }
+
+ __STATIC_INLINE__ std::vector<float> build_rope_matrix_from_frequencies(
+ const std::vector<std::vector<float>>& frequencies,
+ int dim) {
+ const int half_dim = dim / 2;
+ std::vector<float> out(static_cast<size_t>(frequencies.size()) * static_cast<size_t>(half_dim) * 4, 0.f);
+
+ for (size_t token = 0; token < frequencies.size(); token++) {
+ for (int i = 0; i < half_dim; i++) {
+ float angle = i < static_cast<int>(frequencies[token].size()) ? frequencies[token][i] : 0.f;
+ float c = std::cos(angle);
+ float s = std::sin(angle);
+
+ size_t base = (token * static_cast<size_t>(half_dim) + static_cast<size_t>(i)) * 4;
+ out[base + 0] = c;
+ out[base + 1] = -s;
+ out[base + 2] = s;
+ out[base + 3] = c;
+ }
+ }
+
+ return out;
+ }
+
+ __STATIC_INLINE__ std::vector<std::vector<float>> split_frequencies_by_heads(
+ const std::vector<std::vector<float>>& frequencies,
+ int inner_dim,
+ int num_heads) {
+ GGML_ASSERT(num_heads > 0);
+ GGML_ASSERT(inner_dim % num_heads == 0);
+ const int inner_half_dim = inner_dim / 2;
+ const int per_head_half_dim = inner_half_dim / num_heads;
+ GGML_ASSERT(inner_half_dim % num_heads == 0);
+
+ std::vector<std::vector<float>> out(
+ frequencies.size() * static_cast<size_t>(num_heads),
+ std::vector<float>(per_head_half_dim, 0.f));
+
+ for (size_t token = 0; token < frequencies.size(); token++) {
+ GGML_ASSERT(static_cast<int>(frequencies[token].size()) == inner_half_dim);
+ for (int head = 0; head < num_heads; head++) {
+ auto& dst = out[token * static_cast<size_t>(num_heads) + static_cast<size_t>(head)];
+ std::copy_n(frequencies[token].begin() + head * per_head_half_dim, per_head_half_dim, dst.begin());
+ }
+ }
+ return out;
+ }
+
+ __STATIC_INLINE__ ggml_tensor* apply_hidden_rope(GGMLRunnerContext* runner_ctx,
+ ggml_tensor* x,
+ ggml_tensor* pe,
+ int64_t heads,
+ int64_t dim_head,
+ bool rope_interleaved) {
+ auto ctx = runner_ctx->ggml_ctx;
+ GGML_ASSERT(x->ne[0] == heads * dim_head);
+ auto x4 = ggml_reshape_4d(ctx, x, dim_head, heads, x->ne[1], x->ne[2]);
+ if (pe != nullptr && pe->ne[3] == x->ne[1] * heads) {
+ auto x_flat = ggml_reshape_4d(ctx, x4, dim_head, 1, x->ne[1] * heads, x->ne[2]);
+ auto out_flat = Rope::apply_rope(runner_ctx, x_flat, pe, rope_interleaved);
+ auto out4 = ggml_reshape_4d(ctx, out_flat, dim_head, heads, x->ne[1], x->ne[2]);
+ return ggml_reshape_3d(ctx, out4, heads * dim_head, x->ne[1], x->ne[2]);
+ }
+ return Rope::apply_rope(runner_ctx, x4, pe, rope_interleaved);
+ }
+
+ struct TimestepEmbedder : public GGMLBlock {
+ int frequency_embedding_size;
+
+ TimestepEmbedder(int64_t hidden_size,
+ int frequency_embedding_size = 256)
+ : frequency_embedding_size(frequency_embedding_size) {
+ blocks["linear_1"] = std::make_shared<Linear>(frequency_embedding_size, hidden_size, true, true);
+ blocks["linear_2"] = std::make_shared<Linear>(hidden_size, hidden_size, true, true);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* timestep) {
+ auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
+ auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["linear_2"]);
+
+ auto t_emb = ggml_ext_timestep_embedding(ctx, timestep, frequency_embedding_size);
+ t_emb = linear_1->forward(ctx, t_emb);
+ t_emb = ggml_silu_inplace(ctx->ggml_ctx, t_emb);
+ t_emb = linear_2->forward(ctx, t_emb);
+ return t_emb;
+ }
+ };
+
+ struct CrossAttention : public GGMLBlock {
+ int64_t heads;
+ int64_t dim_head;
+ bool rope_interleaved;
+
+ CrossAttention(int64_t query_dim,
+ int64_t context_dim,
+ int64_t heads,
+ int64_t dim_head,
+ bool apply_gated_attention = false,
+ bool rope_interleaved = true)
+ : heads(heads), dim_head(dim_head), rope_interleaved(rope_interleaved) {
+ int64_t inner_dim = heads * dim_head;
+ blocks["q_norm"] = std::make_shared<RMSNorm>(inner_dim, 1e-5f);
+ blocks["k_norm"] = std::make_shared<RMSNorm>(inner_dim, 1e-5f);
+ blocks["to_q"] = std::make_shared<Linear>(query_dim, inner_dim, true);
+ blocks["to_k"] = std::make_shared<Linear>(context_dim, inner_dim, true);
+ blocks["to_v"] = std::make_shared<Linear>(context_dim, inner_dim, true);
+ if (apply_gated_attention) {
+ blocks["to_gate_logits"] = std::make_shared<Linear>(query_dim, heads, true);
+ }
+ blocks["to_out.0"] = std::make_shared<Linear>(inner_dim, query_dim, true);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx,
+ ggml_tensor* x,
+ ggml_tensor* context = nullptr,
+ ggml_tensor* mask = nullptr,
+ ggml_tensor* pe = nullptr,
+ ggml_tensor* k_pe = nullptr) {
+ if (context == nullptr) {
+ context = x;
+ }
+
+ auto to_q = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
+ auto to_k = std::dynamic_pointer_cast<Linear>(blocks["to_k"]);
+ auto to_v = std::dynamic_pointer_cast<Linear>(blocks["to_v"]);
+ auto q_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["q_norm"]);
+ auto k_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["k_norm"]);
+ auto to_out_0 = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
+
+ auto q = to_q->forward(ctx, x);
+ auto k = to_k->forward(ctx, context);
+ auto v = to_v->forward(ctx, context);
+
+ q = q_norm->forward(ctx, q);
+ k = k_norm->forward(ctx, k);
+
+ if (pe != nullptr) {
+ if (k_pe == nullptr) {
+ k_pe = pe;
+ }
+ q = apply_hidden_rope(ctx, q, pe, heads, dim_head, rope_interleaved);
+ k = apply_hidden_rope(ctx, k, k_pe, heads, dim_head, rope_interleaved);
+ }
+
+ auto out = ggml_ext_attention_ext(ctx,
+ q,
+ k,
+ v,
+ heads,
+ mask,
+ false,
+ ctx->flash_attn_enabled);
+
+ if (blocks.count("to_gate_logits") > 0) {
+ auto to_gate_logits = std::dynamic_pointer_cast<Linear>(blocks["to_gate_logits"]);
+ auto gate_logits = to_gate_logits->forward(ctx, x);
+ auto gates = ggml_sigmoid(ctx->ggml_ctx, gate_logits);
+ gates = ggml_ext_scale(ctx, gates, 2.0f, true);
+ gates = ggml_reshape_4d(ctx->ggml_ctx, gates, 1, heads, gate_logits->ne[1], gate_logits->ne[2]);
+
+ auto out4 = ggml_reshape_4d(ctx->ggml_ctx, out, dim_head, heads, out->ne[1], out->ne[2]);
+ gates = ggml_repeat(ctx->ggml_ctx, gates, out4);
+ out4 = ggml_mul(ctx->ggml_ctx, out4, gates);
+ out = ggml_reshape_3d(ctx->ggml_ctx, out4, heads * dim_head, out4->ne[2], out4->ne[3]);
+ }
+
+ return to_out_0->forward(ctx, out);
+ }
+ };
+
+} // namespace LTXV
+
+#endif // __SD_MODEL_COMMON_LTX_HPP__
diff --git a/src/model/diffusion/anima.hpp b/src/model/diffusion/anima.hpp
index 08c20fe..201a82f 100644
--- a/src/model/diffusion/anima.hpp
+++ b/src/model/diffusion/anima.hpp
@@ -718,8 +718,7 @@ namespace Anima {
const sd::Tensor<float>& context = {},
const sd::Tensor<int32_t>& t5_ids = {},
const sd::Tensor<float>& t5_weights = {},
- const std::vector<sd::Tensor<float>>& ref_latents = {},
- const RefImageParams& ref_image_params = REF_IMAGE_PRESETS.at("cosmos_reference")) {
+ const std::vector<sd::Tensor<float>>& ref_latents = {}) {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, t5_ids, t5_weights, ref_latents);
};
@@ -738,8 +737,7 @@ namespace Anima {
tensor_or_empty(diffusion_params.context),
tensor_or_empty(extra->t5_ids),
tensor_or_empty(extra->t5_weights),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents,
- diffusion_params.ref_image_params);
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents);
}
};
} // namespace Anima
diff --git a/src/model/diffusion/animatediff.hpp b/src/model/diffusion/animatediff.hpp
index 459e473..235fb4e 100644
--- a/src/model/diffusion/animatediff.hpp
+++ b/src/model/diffusion/animatediff.hpp
@@ -127,7 +127,8 @@ namespace AnimateDiff {
h = tb0->forward(ctx, h);
h = proj_out->forward(ctx, h);
h = ggml_reshape_4d(ctx->ggml_ctx, h, C, num_frames, W, H);
- h = ggml_ext_cont(ctx, ggml_permute(ctx->ggml_ctx, h, 2, 3, 0, 1));
+ // Binary kernels require a unit innermost stride even for singleton spatial dimensions.
+ h = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h, 2, 3, 0, 1));
return ggml_add(ctx->ggml_ctx, h, residual);
}
@@ -179,6 +180,37 @@ namespace AnimateDiff {
}
};
+ class MotionRunner : public GGMLRunner {
+ AnimateDiffModel model_;
+
+ public:
+ static constexpr const char* prefix = "model.diffusion_model.motion_module";
+
+ MotionRunner(ggml_backend_t backend, const String2TensorStorage& weights, MotionModuleConfig config, std::shared_ptr<RunnerWeightManager> manager)
+ : GGMLRunner(backend, std::move(manager)), model_(config) {
+ model_.init(params_ctx, weights, prefix);
+ }
+
+ std::string get_desc() override { return "animatediff"; }
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+ int max_frames() const { return model_.config.max_frames; }
+ size_t extra_graph_size() const { return weight_adapter ? weight_adapter->get_extra_graph_size() : 0; }
+
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model_.get_param_tensors(tensors, prefix); }
+
+ ggml_tensor* forward(GGMLRunnerContext& ctx, const std::string& key, ggml_tensor* activation, int frames) {
+ auto motion = model_.motion(key);
+ if (!motion)
+ return activation;
+ auto branch = ctx;
+ branch.weight_adapter = weight_adapter;
+ branch.linear_scale = linear_scale;
+ branch.attn_scale = attn_scale;
+ branch.sage_attn_enabled = sage_attn_enabled;
+ return motion->forward(&branch, activation, frames);
+ }
+ };
+
} // namespace AnimateDiff
#endif // __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__
diff --git a/src/model/diffusion/boogu.hpp b/src/model/diffusion/boogu.hpp
index c10da05..667a05e 100644
--- a/src/model/diffusion/boogu.hpp
+++ b/src/model/diffusion/boogu.hpp
@@ -836,7 +836,7 @@ namespace Boogu {
*diffusion_params.x,
*diffusion_params.timesteps,
tensor_or_empty(diffusion_params.context),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents);
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents);
}
};
} // namespace Boogu
diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp
index 2fc3dee..f8c29cd 100644
--- a/src/model/diffusion/control.hpp
+++ b/src/model/diffusion/control.hpp
@@ -450,11 +450,6 @@ struct ControlNet : public GGMLRunner {
std::vector<ggml_tensor*> control_outputs_ggml;
ggml_tensor* guided_hint_output_ggml = nullptr;
std::vector<sd::Tensor<float>> controls;
- bool guided_hint_cached = false;
-
- static const char* guided_hint_cache_name() {
- return "controlnet.guided_hint";
- }
bool is_union() const {
return config.num_control_types > 0;
@@ -462,27 +457,33 @@ struct ControlNet : public GGMLRunner {
// set the kind of the control image of the next generations, false (after saying why) when it
// can't be used: a union model needs one of the types it was trained on
- bool set_control_type(sd_control_type_t type) {
+ std::optional<int> control_index(sd_control_type_t type) const {
if (!is_union()) {
if (type != SD_CONTROL_TYPE_NONE) {
LOG_WARN("control type '%s' ignored: the ControlNet is not a union model", sd_control_type_name(type));
}
- control_type_index = -1;
- return true;
+ return -1;
}
if (type == SD_CONTROL_TYPE_NONE) {
LOG_ERROR("this ControlNet is a union model and needs a control type (openpose, depth, scribble, canny, normal, segment%s)",
config.num_control_types > 6 ? ", tile, repaint" : "");
- return false;
+ return std::nullopt;
}
// the types of a union model, in training order: the enum without NONE
const int index = static_cast<int>(type) - 1;
if (index < 0 || index >= config.num_control_types) {
LOG_ERROR("control type '%s' is not supported by this ControlNet Union (it has %d control types)",
sd_control_type_name(type), config.num_control_types);
- return false;
+ return std::nullopt;
}
- control_type_index = index;
+ return index;
+ }
+
+ bool set_control_type(sd_control_type_t type) {
+ const auto index = control_index(type);
+ if (!index)
+ return false;
+ control_type_index = *index;
return true;
}
@@ -505,7 +506,6 @@ struct ControlNet : public GGMLRunner {
void free_control_ctx() {
guided_hint_output_ggml = nullptr;
- guided_hint_cached = false;
control_outputs_ggml.clear();
controls.clear();
free_cache_ctx_and_buffer();
@@ -522,8 +522,10 @@ struct ControlNet : public GGMLRunner {
ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
const sd::Tensor<float>& hint_tensor,
const sd::Tensor<float>& timesteps_tensor,
- const sd::Tensor<float>& context_tensor = {},
- const sd::Tensor<float>& y_tensor = {}) {
+ const sd::Tensor<float>& context_tensor,
+ const sd::Tensor<float>& y_tensor,
+ const std::string& hint_cache_key,
+ int type_index) {
ggml_cgraph* gf = new_graph_custom(CONTROL_NET_GRAPH_SIZE);
ggml_tensor* x = make_input(x_tensor);
@@ -535,13 +537,7 @@ struct ControlNet : public GGMLRunner {
guided_hint_output_ggml = nullptr;
control_outputs_ggml.clear();
- ggml_tensor* guided_hint_input = nullptr;
- if (guided_hint_cached) {
- guided_hint_input = get_cache_tensor_by_name(guided_hint_cache_name());
- if (guided_hint_input == nullptr) {
- guided_hint_cached = false;
- }
- }
+ ggml_tensor* guided_hint_input = get_cache_tensor_by_name(hint_cache_key);
if (guided_hint_input == nullptr) {
hint = make_input(hint_tensor);
}
@@ -551,7 +547,7 @@ struct ControlNet : public GGMLRunner {
const int64_t batch = x_tensor.dim() >= 4 ? x_tensor.shape()[3] : 1;
control_type_tensor = sd::zeros<float>({config.num_control_types * batch});
for (int64_t i = 0; i < batch; i++) {
- control_type_tensor[i * config.num_control_types + control_type_index] = 1.f;
+ control_type_tensor[i * config.num_control_types + type_index] = 1.f;
}
control_types = make_input(control_type_tensor);
}
@@ -566,12 +562,12 @@ struct ControlNet : public GGMLRunner {
context,
y,
control_types,
- control_type_index);
+ type_index);
if (guided_hint_input == nullptr && !outs.empty()) {
guided_hint_output_ggml = outs[0];
ggml_set_output(guided_hint_output_ggml);
- runner_ctx.persist_cache_tensor(guided_hint_cache_name(), guided_hint_output_ggml);
+ runner_ctx.persist_cache_tensor(hint_cache_key, guided_hint_output_ggml);
}
control_outputs_ggml.reserve(outs.size() > 0 ? outs.size() - 1 : 0);
@@ -589,18 +585,21 @@ struct ControlNet : public GGMLRunner {
const sd::Tensor<float>& x,
const sd::Tensor<float>& hint,
const sd::Tensor<float>& timesteps,
- const sd::Tensor<float>& context = {},
- const sd::Tensor<float>& y = {}) {
+ const sd::Tensor<float>& context = {},
+ const sd::Tensor<float>& y = {},
+ const std::string& hint_cache_key = "controlnet.guided_hint",
+ std::optional<sd_control_type_t> type = {}) {
// x: [N, in_channels, h, w]
// timesteps: [N, ]
// context: [N, max_position, hidden_size]([N, 77, 768]) or [1, max_position, hidden_size]
// y: [N, adm_in_channels] or [1, adm_in_channels]
- if (is_union() && control_type_index < 0) {
+ const auto index = type ? control_index(*type) : std::optional<int>(control_type_index);
+ if (!index || (is_union() && *index < 0)) {
LOG_ERROR("ControlNet Union: no control type set");
return std::nullopt;
}
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, hint, timesteps, context, y);
+ return build_graph(x, hint, timesteps, context, y, hint_cache_key, *index);
};
auto read_outputs = [&]() {
@@ -622,7 +621,6 @@ struct ControlNet : public GGMLRunner {
controls.clear();
return std::nullopt;
}
- guided_hint_cached = get_cache_tensor_by_name(guided_hint_cache_name()) != nullptr;
return controls;
}
};
diff --git a/src/model/diffusion/fibo2.hpp b/src/model/diffusion/fibo2.hpp
index 091e74c..e4374d8 100644
--- a/src/model/diffusion/fibo2.hpp
+++ b/src/model/diffusion/fibo2.hpp
@@ -364,7 +364,7 @@ namespace Fibo2 {
GGML_ASSERT(x->ne[3] == 1 && x->ne[2] == config.channels);
int count = config.gist_count(static_cast<int>(p.context->shape()[1]));
std::vector<ggml_tensor*> refs;
- if (p.ref_latents && p.ref_image_params.pass_to_dit) {
+ if (p.ref_latents) {
for (const auto& ref : *p.ref_latents) {
refs.push_back(make_input(ref));
}
diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp
index 01b4609..5b3f443 100644
--- a/src/model/diffusion/flux.hpp
+++ b/src/model/diffusion/flux.hpp
@@ -1011,10 +1011,11 @@ namespace Flux {
ggml_tensor* y,
ggml_tensor* guidance,
ggml_tensor* pe,
- ggml_tensor* mod_index_arange = nullptr,
- std::vector<int> skip_layers = {},
- ggml_tensor* pulid_id = nullptr,
- float pulid_id_weight = 1.0f) {
+ ggml_tensor* mod_index_arange = nullptr,
+ std::vector<int> skip_layers = {},
+ ggml_tensor* pulid_id = nullptr,
+ float pulid_id_weight = 1.0f,
+ const FluxImageBlockProcessor* image_processor = nullptr) {
auto img_in = std::dynamic_pointer_cast<Linear>(blocks["img_in"]);
auto txt_in = std::dynamic_pointer_cast<Linear>(blocks["txt_in"]);
auto final_layer = std::dynamic_pointer_cast<LastLayer>(blocks["final_layer"]);
@@ -1124,6 +1125,10 @@ namespace Flux {
sd::ggml_graph_cut::mark_graph_cut(img, "flux.pulid_ca." + std::to_string(ca_idx), "img");
ca_idx++;
}
+ if (image_processor && image_processor->applies(false, i)) {
+ img = image_processor->apply(*ctx, false, i, img);
+ sd::ggml_graph_cut::mark_graph_cut(img, "flux.double_block_processor." + std::to_string(i), "img");
+ }
}
auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1); // [N, n_txt_token + n_img_token, hidden_size]
@@ -1137,9 +1142,9 @@ namespace Flux {
txt_img = block->forward(ctx, txt_img, vec, pe, txt_img_mask, ss_mods);
sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.single_blocks." + std::to_string(i), "txt_img");
- if (pulid_run && (i % config.pulid_single_interval == 0)) {
- auto pulid_ca = std::dynamic_pointer_cast<PuLIDPerceiverAttentionCA>(
- blocks["pulid_ca." + std::to_string(ca_idx)]);
+ const bool native_pulid = pulid_run && (i % config.pulid_single_interval == 0);
+ const bool process_image = image_processor && image_processor->applies(true, i);
+ if (native_pulid || process_image) {
ggml_tensor* txt_part = ggml_view_3d(ctx->ggml_ctx, txt_img,
txt_img->ne[0], n_txt_tok, txt_img->ne[2],
txt_img->nb[1], txt_img->nb[2],
@@ -1153,11 +1158,19 @@ namespace Flux {
n_txt_tok * txt_img->nb[1]);
txt_part = ggml_cont(ctx->ggml_ctx, txt_part);
img_part = ggml_cont(ctx->ggml_ctx, img_part);
- ggml_tensor* ca_out = pulid_ca->forward(ctx, pulid_id, img_part);
- img_part = ggml_add(ctx->ggml_ctx, img_part, ggml_scale(ctx->ggml_ctx, ca_out, pulid_id_weight));
- txt_img = ggml_concat(ctx->ggml_ctx, txt_part, img_part, 1);
- sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.pulid_ca." + std::to_string(ca_idx), "txt_img");
- ca_idx++;
+ if (native_pulid) {
+ auto pulid_ca = std::dynamic_pointer_cast<PuLIDPerceiverAttentionCA>(
+ blocks["pulid_ca." + std::to_string(ca_idx)]);
+ auto* ca_out = pulid_ca->forward(ctx, pulid_id, img_part);
+ img_part = ggml_add(ctx->ggml_ctx, img_part, ggml_scale(ctx->ggml_ctx, ca_out, pulid_id_weight));
+ }
+ if (process_image)
+ img_part = image_processor->apply(*ctx, true, i, img_part);
+ txt_img = ggml_concat(ctx->ggml_ctx, txt_part, img_part, 1);
+ if (native_pulid)
+ sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.pulid_ca." + std::to_string(ca_idx++), "txt_img");
+ else
+ sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.single_block_processor." + std::to_string(i), "txt_img");
}
}
@@ -1271,12 +1284,13 @@ namespace Flux {
ggml_tensor* y,
ggml_tensor* guidance,
ggml_tensor* pe,
- ggml_tensor* mod_index_arange = nullptr,
- ggml_tensor* dct = nullptr,
- std::vector<ggml_tensor*> ref_latents = {},
- std::vector<int> skip_layers = {},
- ggml_tensor* pulid_id = nullptr,
- float pulid_id_weight = 1.0f) {
+ ggml_tensor* mod_index_arange = nullptr,
+ ggml_tensor* dct = nullptr,
+ std::vector<ggml_tensor*> ref_latents = {},
+ std::vector<int> skip_layers = {},
+ ggml_tensor* pulid_id = nullptr,
+ float pulid_id_weight = 1.0f,
+ const FluxImageBlockProcessor* image_processor = nullptr) {
GGML_ASSERT(x->ne[3] == 1);
int64_t W = x->ne[0];
@@ -1324,7 +1338,7 @@ namespace Flux {
}
auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, mod_index_arange, skip_layers,
- pulid_id, pulid_id_weight); // [N, num_tokens, C * patch_size * patch_size]
+ pulid_id, pulid_id_weight, image_processor); // [N, num_tokens, C * patch_size * patch_size]
if (out->ne[1] > img_tokens) {
out = ggml_view_3d(ctx->ggml_ctx, out, out->ne[0], img_tokens, out->ne[2], out->nb[1], out->nb[2], 0);
@@ -1343,12 +1357,13 @@ namespace Flux {
ggml_tensor* y,
ggml_tensor* guidance,
ggml_tensor* pe,
- ggml_tensor* mod_index_arange = nullptr,
- ggml_tensor* dct = nullptr,
- std::vector<ggml_tensor*> ref_latents = {},
- std::vector<int> skip_layers = {},
- ggml_tensor* pulid_id = nullptr,
- float pulid_id_weight = 1.0f) {
+ ggml_tensor* mod_index_arange = nullptr,
+ ggml_tensor* dct = nullptr,
+ std::vector<ggml_tensor*> ref_latents = {},
+ std::vector<int> skip_layers = {},
+ ggml_tensor* pulid_id = nullptr,
+ float pulid_id_weight = 1.0f,
+ const FluxImageBlockProcessor* image_processor = nullptr) {
// Forward pass of DiT.
// x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
// timestep: (N,) tensor of diffusion timesteps
@@ -1388,7 +1403,8 @@ namespace Flux {
ref_latents,
skip_layers,
pulid_id,
- pulid_id_weight);
+ pulid_id_weight,
+ image_processor);
}
}
};
@@ -1397,6 +1413,8 @@ namespace Flux {
sd::PECache pe_cache_;
public:
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+
FluxConfig config;
Flux flux;
std::vector<float> mod_index_arange_vec;
@@ -1505,7 +1523,8 @@ namespace Flux {
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
std::vector<int> skip_layers = {},
const sd::Tensor<float>& pulid_id_tensor = {},
- float pulid_id_weight = 1.0f) {
+ float pulid_id_weight = 1.0f,
+ const FluxImageBlockProcessor* image_processor = nullptr) {
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
ggml_tensor* context = make_optional_input(context_tensor);
@@ -1527,7 +1546,7 @@ namespace Flux {
}
GGML_ASSERT(x->ne[3] == 1);
- ggml_cgraph* gf = new_graph_custom(FLUX_GRAPH_SIZE);
+ ggml_cgraph* gf = new_graph_custom(FLUX_GRAPH_SIZE + (image_processor ? image_processor->extra_graph_size() : 0));
ggml_tensor* mod_index_arange = nullptr;
ggml_tensor* dct = nullptr; // for chroma radiance
@@ -1600,7 +1619,8 @@ namespace Flux {
ref_latents,
skip_layers,
pulid_id,
- pulid_id_weight);
+ pulid_id_weight,
+ image_processor);
ggml_build_forward_expand(gf, out);
@@ -1618,7 +1638,8 @@ namespace Flux {
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
std::vector<int> skip_layers = std::vector<int>(),
const sd::Tensor<float>& pulid_id = {},
- float pulid_id_weight = 1.0f) {
+ float pulid_id_weight = 1.0f,
+ const FluxImageBlockProcessor* image_processor = nullptr) {
// x: [N, in_channels, h, w]
// timesteps: [N, ]
// context: [N, max_position, hidden_size]
@@ -1626,7 +1647,7 @@ namespace Flux {
// guidance: [N, ]
// pulid_id: empty (no injection) or [N, num_id_tokens=32, kv_dim=2048]
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, ref_index_mode, skip_layers, pulid_id, pulid_id_weight);
+ return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, ref_index_mode, skip_layers, pulid_id, pulid_id_weight, image_processor);
};
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
@@ -1647,11 +1668,12 @@ namespace Flux {
tensor_or_empty(diffusion_params.c_concat),
tensor_or_empty(diffusion_params.y),
tensor_or_empty(extra->guidance),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents,
- diffusion_params.ref_image_params.ref_index_mode,
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+ diffusion_params.ref_index_mode,
extra->skip_layers ? *extra->skip_layers : empty_skip_layers,
tensor_or_empty(extra->pulid_id),
- extra->pulid_id_weight);
+ extra->pulid_id_weight,
+ extra->image_processor);
}
void test() {
diff --git a/src/model/diffusion/flux3.hpp b/src/model/diffusion/flux3.hpp
index ad86d8a..2fd60cf 100644
--- a/src/model/diffusion/flux3.hpp
+++ b/src/model/diffusion/flux3.hpp
@@ -207,7 +207,7 @@ namespace Flux3 {
}
sd::Tensor<float> compute(int n_threads, const DiffusionParams& params) override {
- if (params.ref_latents && params.ref_image_params.pass_to_dit && params.ref_latents->size() > 10) {
+ if (params.ref_latents && params.ref_latents->size() > 10) {
LOG_ERROR("FLUX.3 supports at most 10 reference images");
return {};
}
@@ -222,7 +222,7 @@ namespace Flux3 {
GGML_ASSERT(x->ne[3] == 1 && text->ne[2] == 1);
std::vector<ggml_tensor*> refs;
ggml_tensor* references = nullptr;
- if (params.ref_latents && params.ref_image_params.pass_to_dit) {
+ if (params.ref_latents) {
for (const auto& ref : *params.ref_latents) {
auto r = make_input(ref);
refs.push_back(r);
diff --git a/src/model/diffusion/hidream_o1.hpp b/src/model/diffusion/hidream_o1.hpp
index 8581fba..400c10b 100644
--- a/src/model/diffusion/hidream_o1.hpp
+++ b/src/model/diffusion/hidream_o1.hpp
@@ -474,6 +474,9 @@ namespace HiDreamO1 {
};
struct HiDreamO1Conditioner : public Conditioner {
+ bool supports_prompt_formatting() const override { return true; }
+ size_t max_reference_images() const override { return std::numeric_limits<size_t>::max(); }
+
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<HiDreamO1VisionRunner> vision_runner;
@@ -529,11 +532,13 @@ namespace HiDreamO1 {
std::vector<std::array<int32_t, 3>> image_grids;
std::vector<int32_t> skip_vision_start;
- std::string prompt = "<|im_start|>user\n";
+ std::string prompt = sd::prompt_templates::with_system("<|im_start|>user\n", conditioner_params.system_prompt);
if (ref_images.empty()) {
prompt += conditioner_params.text;
prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
+ if (conditioner_params.raw)
+ prompt = conditioner_params.text;
std::vector<int> input_ids;
if (!tokenizer->encode(prompt, input_ids, nullptr)) {
return {};
@@ -628,6 +633,8 @@ namespace HiDreamO1 {
prompt += conditioner_params.text;
prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
+ if (conditioner_params.raw)
+ prompt = conditioner_params.text;
std::vector<int> input_ids;
if (!tokenizer->encode(prompt, input_ids, nullptr)) {
return {};
diff --git a/src/model/diffusion/krea2.hpp b/src/model/diffusion/krea2.hpp
index 54dd6fd..b14a65c 100644
--- a/src/model/diffusion/krea2.hpp
+++ b/src/model/diffusion/krea2.hpp
@@ -739,7 +739,8 @@ namespace Krea2 {
const sd::Tensor<float>& timesteps_tensor,
const sd::Tensor<float>& context_tensor,
const std::vector<sd::Tensor<float>>& ref_latents_tensor = {},
- const RefImageParams& ref_image_params = REF_IMAGE_PRESETS.at("krea2_ostris_edit")) {
+ Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
+ bool force_ref_timestep_zero = false) {
ggml_cgraph* gf = new_graph_custom(KREA2_GRAPH_SIZE);
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
@@ -756,7 +757,7 @@ namespace Krea2 {
sd::PECacheKey pe_key("krea2.pe");
pe_key.add_shape(x).add_shape(context).add(config.theta).add(config.axes_dim).add(config.axes_dim_sum);
pe_key.add(circular_x_enabled).add(circular_y_enabled).add(config.patch_size).add_shapes(ref_latents);
- pe_key.add(ref_image_params.ref_index_mode);
+ pe_key.add(ref_index_mode);
auto pe = make_input(pe_cache_.get_or_create(pe_key, [&]() {
auto pe_vec = finish_rope_pe(gen_krea2_pe(static_cast<int>(x->ne[1]),
static_cast<int>(x->ne[0]),
@@ -766,13 +767,13 @@ namespace Krea2 {
config.theta,
config.axes_dim,
ref_latents,
- ref_image_params.ref_index_mode));
+ ref_index_mode));
int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
return sd::Tensor<float>({2, 2, config.axes_dim_sum / 2, pos_len}, std::move(pe_vec));
}));
auto runner_ctx = get_context();
- ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pe, ref_latents, ref_image_params.force_ref_timestep_zero);
+ ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pe, ref_latents, force_ref_timestep_zero);
ggml_build_forward_expand(gf, out);
return gf;
}
@@ -782,9 +783,10 @@ namespace Krea2 {
const sd::Tensor<float>& timesteps,
const sd::Tensor<float>& context,
const std::vector<sd::Tensor<float>>& ref_latents = {},
- const RefImageParams& ref_image_params = REF_IMAGE_PRESETS.at("krea2_ostris_edit")) {
+ Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
+ bool force_ref_timestep_zero = false) {
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, timesteps, context, ref_latents, ref_image_params);
+ return build_graph(x, timesteps, context, ref_latents, ref_index_mode, force_ref_timestep_zero);
};
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
@@ -798,8 +800,8 @@ namespace Krea2 {
*diffusion_params.x,
*diffusion_params.timesteps,
tensor_or_empty(diffusion_params.context),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents,
- diffusion_params.ref_image_params);
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+ diffusion_params.ref_index_mode, diffusion_params.force_ref_timestep_zero);
}
};
} // namespace Krea2
diff --git a/src/model/diffusion/ltxv.hpp b/src/model/diffusion/ltxv.hpp
index 8a07e79..8fe4491 100644
--- a/src/model/diffusion/ltxv.hpp
+++ b/src/model/diffusion/ltxv.hpp
@@ -1,88 +1,21 @@
#ifndef __SD_MODEL_DIFFUSION_LTXV_HPP__
#define __SD_MODEL_DIFFUSION_LTXV_HPP__
-#include <algorithm>
#include <cinttypes>
-#include <cmath>
-#include <memory>
-#include <string>
+#include <fstream>
#include <tuple>
-#include <utility>
-#include <vector>
-#include "core/ggml_extend_backend.h"
-#include "core/ggml_tensor_utils.h"
-#include "core/pe_cache.h"
-#include "model/common/block.hpp"
-#include "model/common/rope.hpp"
-#include "model/diffusion/flux.hpp"
+#include "core/pe_cache.h"
+#include "model/common/ltx.hpp"
#include "model/diffusion/model.hpp"
-#include "model_loader.h"
namespace LTXV {
- constexpr int LTXAV_GRAPH_SIZE = 102400;
-
- __STATIC_INLINE__ ggml_tensor* rms_norm(ggml_context* ctx,
- ggml_tensor* x,
- float eps = 1e-6f) {
- return ggml_rms_norm(ctx, x, eps);
- }
-
- __STATIC_INLINE__ ggml_tensor* align_token_modulation(ggml_context* ctx,
- ggml_tensor* x,
- ggml_tensor* mod) {
- if (mod != nullptr && x != nullptr && mod->ne[1] == 1 && mod->ne[2] == x->ne[1] && x->ne[2] == 1) {
- return ggml_permute(ctx, mod, 0, 2, 1, 3);
- }
- return mod;
- }
-
- __STATIC_INLINE__ ggml_tensor* modulate(GGMLRunnerContext* ctx,
- ggml_tensor* x,
- ggml_tensor* shift,
- ggml_tensor* scale) {
- shift = align_token_modulation(ctx->ggml_ctx, x, shift);
- scale = align_token_modulation(ctx->ggml_ctx, x, scale);
- return Flux::modulate(ctx, x, shift, scale, true);
- }
-
- __STATIC_INLINE__ ggml_tensor* apply_gate(ggml_context* ctx,
- ggml_tensor* x,
- ggml_tensor* gate) {
- gate = align_token_modulation(ctx, x, gate);
- return ggml_mul(ctx, x, gate);
- }
-
- __STATIC_INLINE__ int count_prefix_blocks(const String2TensorStorage& tensor_storage_map,
- const std::string& prefix,
- const std::string& marker) {
- int max_block = -1;
- for (const auto& [name, _] : tensor_storage_map) {
- if (!starts_with(name, prefix)) {
- continue;
- }
- size_t pos = name.find(marker);
- if (pos == std::string::npos) {
- continue;
- }
- pos += marker.size();
- size_t end = name.find(".", pos);
- if (end == std::string::npos) {
- continue;
- }
- int block = atoi(name.substr(pos, end - pos).c_str());
- max_block = std::max(max_block, block);
- }
- return max_block + 1;
- }
-
struct LTXAVConfig {
int64_t in_channels = 128;
int64_t out_channels = 128;
int64_t hidden_size = 3840;
int64_t cross_attention_dim = 4096;
- int64_t caption_channels = 3840;
int64_t num_attention_heads = 30;
int64_t attention_head_dim = 128;
int64_t num_layers = 28;
@@ -103,33 +36,10 @@ namespace LTXV {
int64_t num_audio_channels = 8;
int64_t audio_frequency_bins = 16;
- bool use_connector = false;
- int64_t connector_hidden_size = 3840;
- int64_t connector_num_heads = 30;
- int64_t connector_head_dim = 128;
- int64_t connector_num_layers = 2;
- int64_t connector_num_registers = 128;
- bool connector_rope_interleaved = false;
- bool connector_apply_gated_attention = false;
-
- bool use_audio_connector = false;
- int64_t audio_connector_hidden_size = 2048;
- int64_t audio_connector_num_heads = 32;
- int64_t audio_connector_head_dim = 64;
- int64_t audio_connector_num_layers = 2;
- int64_t audio_connector_num_registers = 128;
- bool audio_connector_rope_interleaved = false;
- bool audio_connector_apply_gated_attention = false;
-
bool video_rope_interleaved = false;
bool use_middle_indices_grid = true;
bool cross_attention_adaln = false;
- bool use_caption_projection = true;
- bool use_audio_caption_projection = true;
- bool caption_proj_before_connector = true;
- bool caption_projection_first_linear = false;
-
bool self_attention_gated = false;
bool cross_attention_gated = false;
@@ -137,33 +47,6 @@ namespace LTXV {
bool audio_ff_bias = true;
bool use_keyframes_abs_pos_embedding = false;
- static std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
- int64_t preferred_heads = -1) {
- if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
- return {preferred_heads, hidden_size / preferred_heads};
- }
- const int candidates[] = {128, 96, 80, 64, 48, 40, 32};
- for (int head_dim : candidates) {
- if (hidden_size % head_dim == 0) {
- int64_t heads = hidden_size / head_dim;
- if (heads >= 8 && heads <= 64) {
- return {heads, head_dim};
- }
- }
- }
- return {32, hidden_size / 32};
- }
-
- static int64_t infer_gate_heads(const String2TensorStorage& tensor_storage_map,
- const std::string& bias_name,
- int64_t fallback_heads) {
- auto it = tensor_storage_map.find(bias_name);
- if (it != tensor_storage_map.end()) {
- return it->second.ne[0];
- }
- return fallback_heads;
- }
-
static LTXAVConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, const std::string& prefix) {
LTXAVConfig config;
auto patchify_proj_iter = tensor_storage_map.find(prefix + ".patchify_proj.weight");
@@ -228,56 +111,8 @@ namespace LTXV {
if (tensor_storage_map.find(prefix + ".keyframes_abs_pos_embedding") != tensor_storage_map.end()) {
config.use_keyframes_abs_pos_embedding = true;
}
- if (tensor_storage_map.find(prefix + ".caption_projection.linear_1.weight") == tensor_storage_map.end() &&
- tensor_storage_map.find(prefix + ".caption_projection.linear_2.weight") == tensor_storage_map.end()) {
- config.use_caption_projection = false;
- }
- if (tensor_storage_map.find(prefix + ".audio_caption_projection.linear_1.weight") == tensor_storage_map.end() &&
- tensor_storage_map.find(prefix + ".audio_caption_projection.linear_2.weight") == tensor_storage_map.end()) {
- config.use_audio_caption_projection = false;
- }
-
config.num_layers = count_prefix_blocks(tensor_storage_map, prefix + ".", "transformer_blocks.");
- auto connector_iter = tensor_storage_map.find(prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight");
- if (connector_iter != tensor_storage_map.end()) {
- config.use_connector = true;
- config.connector_hidden_size = connector_iter->second.ne[1];
- int64_t connector_heads = infer_gate_heads(tensor_storage_map,
- prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.bias",
- 32);
- auto connector_layout = infer_attention_layout(config.connector_hidden_size, connector_heads);
- config.connector_num_heads = connector_layout.first;
- config.connector_head_dim = connector_layout.second;
- config.connector_num_layers = count_prefix_blocks(tensor_storage_map, prefix + ".video_embeddings_connector.", "transformer_1d_blocks.");
- auto register_iter = tensor_storage_map.find(prefix + ".video_embeddings_connector.learnable_registers");
- if (register_iter != tensor_storage_map.end()) {
- config.connector_num_registers = register_iter->second.ne[1];
- }
- if (tensor_storage_map.find(prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.weight") != tensor_storage_map.end()) {
- config.connector_apply_gated_attention = true;
- }
- }
-
- auto audio_connector_iter = tensor_storage_map.find(prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight");
- if (audio_connector_iter != tensor_storage_map.end()) {
- config.use_audio_connector = true;
- config.audio_connector_hidden_size = audio_connector_iter->second.ne[1];
- int64_t connector_heads = infer_gate_heads(tensor_storage_map,
- prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.bias",
- 32);
- auto connector_layout = infer_attention_layout(config.audio_connector_hidden_size, connector_heads);
- config.audio_connector_num_heads = connector_layout.first;
- config.audio_connector_head_dim = connector_layout.second;
- config.audio_connector_num_layers = count_prefix_blocks(tensor_storage_map, prefix + ".audio_embeddings_connector.", "transformer_1d_blocks.");
- auto register_iter = tensor_storage_map.find(prefix + ".audio_embeddings_connector.learnable_registers");
- if (register_iter != tensor_storage_map.end()) {
- config.audio_connector_num_registers = register_iter->second.ne[1];
- }
- if (tensor_storage_map.find(prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.weight") != tensor_storage_map.end()) {
- config.audio_connector_apply_gated_attention = true;
- }
- }
LOG_VERBOSE("ltxav: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_attention_heads = %" PRId64 ", audio_hidden_size = %" PRId64 ", audio_num_attention_heads = %" PRId64,
config.num_layers,
config.hidden_size,
@@ -288,99 +123,29 @@ namespace LTXV {
}
};
- __STATIC_INLINE__ std::vector<float> generate_freq_grid(float theta,
- int positional_dims,
- int dim) {
- const int n_elem = 2 * positional_dims;
- const int freq_count = dim / n_elem;
-
- std::vector<float> out(freq_count);
- if (freq_count <= 0) {
- return out;
- }
- if (freq_count == 1) {
- out[0] = 1.5707963267948966f;
- return out;
- }
-
- const float half_pi = 1.5707963267948966f;
- const float log_theta = std::log(theta);
- for (int i = 0; i < freq_count; i++) {
- float ratio = static_cast<float>(i) / static_cast<float>(freq_count - 1);
- out[i] = std::exp(log_theta * ratio) * half_pi;
- }
- return out;
- }
-
- __STATIC_INLINE__ std::vector<double> generate_freq_grid_double(double theta,
- int positional_dims,
- int dim) {
- const int n_elem = 2 * positional_dims;
- const int freq_count = dim / n_elem;
-
- std::vector<double> out(freq_count);
- if (freq_count <= 0) {
- return out;
- }
- if (freq_count == 1) {
- out[0] = 1.5707963267948966;
- return out;
- }
-
- const double half_pi = 1.5707963267948966;
- const double log_theta = std::log(theta);
- for (int i = 0; i < freq_count; i++) {
- double ratio = static_cast<double>(i) / static_cast<double>(freq_count - 1);
- out[i] = std::exp(log_theta * ratio) * half_pi;
+ __STATIC_INLINE__ ggml_tensor* align_token_modulation(ggml_context* ctx,
+ ggml_tensor* x,
+ ggml_tensor* mod) {
+ if (mod != nullptr && x != nullptr && mod->ne[1] == 1 && mod->ne[2] == x->ne[1] && x->ne[2] == 1) {
+ return ggml_permute(ctx, mod, 0, 2, 1, 3);
}
- return out;
+ return mod;
}
- __STATIC_INLINE__ std::vector<float> build_rope_matrix_from_frequencies(
- const std::vector<std::vector<float>>& frequencies,
- int dim) {
- const int half_dim = dim / 2;
- std::vector<float> out(static_cast<size_t>(frequencies.size()) * static_cast<size_t>(half_dim) * 4, 0.f);
-
- for (size_t token = 0; token < frequencies.size(); token++) {
- for (int i = 0; i < half_dim; i++) {
- float angle = i < static_cast<int>(frequencies[token].size()) ? frequencies[token][i] : 0.f;
- float c = std::cos(angle);
- float s = std::sin(angle);
-
- size_t base = (token * static_cast<size_t>(half_dim) + static_cast<size_t>(i)) * 4;
- out[base + 0] = c;
- out[base + 1] = -s;
- out[base + 2] = s;
- out[base + 3] = c;
- }
- }
-
- return out;
+ __STATIC_INLINE__ ggml_tensor* modulate(GGMLRunnerContext* ctx,
+ ggml_tensor* x,
+ ggml_tensor* shift,
+ ggml_tensor* scale) {
+ shift = align_token_modulation(ctx->ggml_ctx, x, shift);
+ scale = align_token_modulation(ctx->ggml_ctx, x, scale);
+ return ggml_ext_modulate(ctx, x, shift, scale);
}
- __STATIC_INLINE__ std::vector<std::vector<float>> split_frequencies_by_heads(
- const std::vector<std::vector<float>>& frequencies,
- int inner_dim,
- int num_heads) {
- GGML_ASSERT(num_heads > 0);
- GGML_ASSERT(inner_dim % num_heads == 0);
- const int inner_half_dim = inner_dim / 2;
- const int per_head_half_dim = inner_half_dim / num_heads;
- GGML_ASSERT(inner_half_dim % num_heads == 0);
-
- std::vector<std::vector<float>> out(
- frequencies.size() * static_cast<size_t>(num_heads),
- std::vector<float>(per_head_half_dim, 0.f));
-
- for (size_t token = 0; token < frequencies.size(); token++) {
- GGML_ASSERT(static_cast<int>(frequencies[token].size()) == inner_half_dim);
- for (int head = 0; head < num_heads; head++) {
- auto& dst = out[token * static_cast<size_t>(num_heads) + static_cast<size_t>(head)];
- std::copy_n(frequencies[token].begin() + head * per_head_half_dim, per_head_half_dim, dst.begin());
- }
- }
- return out;
+ __STATIC_INLINE__ ggml_tensor* apply_gate(ggml_context* ctx,
+ ggml_tensor* x,
+ ggml_tensor* gate) {
+ gate = align_token_modulation(ctx, x, gate);
+ return ggml_mul(ctx, x, gate);
}
__STATIC_INLINE__ std::vector<float> build_video_rope_matrix(int64_t width,
@@ -510,85 +275,6 @@ namespace LTXV {
return build_rope_matrix_from_frequencies(freqs, dim);
}
- __STATIC_INLINE__ std::vector<float> build_1d_rope_matrix(int64_t seq_len,
- int dim,
- int num_heads = 1,
- float theta = 10000.f,
- float positional_scale = 4096.f,
- bool double_precision = false) {
- GGML_ASSERT(dim % num_heads == 0);
- const std::vector<float> indices = double_precision ? std::vector<float>() : generate_freq_grid(theta, 1, dim);
- const std::vector<double> indices_d =
- double_precision ? generate_freq_grid_double(static_cast<double>(theta), 1, dim) : std::vector<double>();
- const int half_dim = dim / 2;
- const int pad_size = half_dim - static_cast<int>(double_precision ? indices_d.size() : indices.size());
-
- std::vector<std::vector<float>> freqs(static_cast<size_t>(seq_len), std::vector<float>(half_dim, 0.f));
- for (int64_t pos = 0; pos < seq_len; pos++) {
- int out_idx = 0;
- for (int i = 0; i < pad_size; i++) {
- freqs[static_cast<size_t>(pos)][out_idx++] = 0.f;
- }
-
- if (double_precision) {
- double coord = static_cast<double>(pos) / static_cast<double>(positional_scale);
- for (double index : indices_d) {
- freqs[static_cast<size_t>(pos)][out_idx++] = static_cast<float>(index * (coord * 2.0 - 1.0));
- }
- } else {
- float coord = static_cast<float>(pos) / positional_scale;
- for (float index : indices) {
- freqs[static_cast<size_t>(pos)][out_idx++] = index * (coord * 2.f - 1.f);
- }
- }
- }
-
- if (num_heads > 1) {
- return build_rope_matrix_from_frequencies(split_frequencies_by_heads(freqs, dim, num_heads), dim / num_heads);
- }
- return build_rope_matrix_from_frequencies(freqs, dim);
- }
-
- __STATIC_INLINE__ ggml_tensor* apply_hidden_rope(GGMLRunnerContext* runner_ctx,
- ggml_tensor* x,
- ggml_tensor* pe,
- int64_t heads,
- int64_t dim_head,
- bool rope_interleaved) {
- auto ctx = runner_ctx->ggml_ctx;
- GGML_ASSERT(x->ne[0] == heads * dim_head);
- auto x4 = ggml_reshape_4d(ctx, x, dim_head, heads, x->ne[1], x->ne[2]);
- if (pe != nullptr && pe->ne[3] == x->ne[1] * heads) {
- auto x_flat = ggml_reshape_4d(ctx, x4, dim_head, 1, x->ne[1] * heads, x->ne[2]);
- auto out_flat = Rope::apply_rope(runner_ctx, x_flat, pe, rope_interleaved);
- auto out4 = ggml_reshape_4d(ctx, out_flat, dim_head, heads, x->ne[1], x->ne[2]);
- return ggml_reshape_3d(ctx, out4, heads * dim_head, x->ne[1], x->ne[2]);
- }
- return Rope::apply_rope(runner_ctx, x4, pe, rope_interleaved);
- }
-
- struct TimestepEmbedder : public GGMLBlock {
- int frequency_embedding_size;
-
- TimestepEmbedder(int64_t hidden_size,
- int frequency_embedding_size = 256)
- : frequency_embedding_size(frequency_embedding_size) {
- blocks["linear_1"] = std::make_shared<Linear>(frequency_embedding_size, hidden_size, true, true);
- blocks["linear_2"] = std::make_shared<Linear>(hidden_size, hidden_size, true, true);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* timestep) {
- auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
- auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["linear_2"]);
-
- auto t_emb = ggml_ext_timestep_embedding(ctx, timestep, frequency_embedding_size);
- t_emb = linear_1->forward(ctx, t_emb);
- t_emb = ggml_silu_inplace(ctx->ggml_ctx, t_emb);
- t_emb = linear_2->forward(ctx, t_emb);
- return t_emb;
- }
- };
-
struct AdaLayerNormSingle : public GGMLBlock {
int64_t embedding_dim;
int64_t embedding_coefficient;
@@ -615,128 +301,6 @@ namespace LTXV {
}
};
- struct PixArtAlphaTextProjection : public GGMLBlock {
- PixArtAlphaTextProjection(int64_t in_features,
- int64_t hidden_size,
- int64_t out_features = -1) {
- if (out_features < 0) {
- out_features = hidden_size;
- }
- blocks["linear_1"] = std::make_shared<Linear>(in_features, hidden_size, true, true);
- blocks["linear_2"] = std::make_shared<Linear>(hidden_size, out_features, true, true);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* caption) {
- auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
- auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["linear_2"]);
-
- caption = linear_1->forward(ctx, caption);
- caption = ggml_ext_gelu(ctx, caption, true);
- caption = linear_2->forward(ctx, caption);
- return caption;
- }
- };
-
- struct NormSingleLinearTextProjection : public GGMLBlock {
- int64_t in_features;
- int64_t hidden_size;
-
- NormSingleLinearTextProjection(int64_t in_features,
- int64_t hidden_size)
- : in_features(in_features), hidden_size(hidden_size) {
- blocks["linear_1"] = std::make_shared<Linear>(in_features, hidden_size, true, true);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* caption) {
- auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
- caption = ggml_rms_norm(ctx->ggml_ctx, caption, 1e-6f);
- caption = ggml_ext_scale(ctx, caption, std::sqrt(static_cast<float>(hidden_size) / static_cast<float>(in_features)));
- return linear_1->forward(ctx, caption);
- }
- };
-
- struct CrossAttention : public GGMLBlock {
- int64_t heads;
- int64_t dim_head;
- bool rope_interleaved;
-
- CrossAttention(int64_t query_dim,
- int64_t context_dim,
- int64_t heads,
- int64_t dim_head,
- bool apply_gated_attention = false,
- bool rope_interleaved = true)
- : heads(heads), dim_head(dim_head), rope_interleaved(rope_interleaved) {
- int64_t inner_dim = heads * dim_head;
- blocks["q_norm"] = std::make_shared<RMSNorm>(inner_dim, 1e-5f);
- blocks["k_norm"] = std::make_shared<RMSNorm>(inner_dim, 1e-5f);
- blocks["to_q"] = std::make_shared<Linear>(query_dim, inner_dim, true);
- blocks["to_k"] = std::make_shared<Linear>(context_dim, inner_dim, true);
- blocks["to_v"] = std::make_shared<Linear>(context_dim, inner_dim, true);
- if (apply_gated_attention) {
- blocks["to_gate_logits"] = std::make_shared<Linear>(query_dim, heads, true);
- }
- blocks["to_out.0"] = std::make_shared<Linear>(inner_dim, query_dim, true);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx,
- ggml_tensor* x,
- ggml_tensor* context = nullptr,
- ggml_tensor* mask = nullptr,
- ggml_tensor* pe = nullptr,
- ggml_tensor* k_pe = nullptr) {
- if (context == nullptr) {
- context = x;
- }
-
- auto to_q = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
- auto to_k = std::dynamic_pointer_cast<Linear>(blocks["to_k"]);
- auto to_v = std::dynamic_pointer_cast<Linear>(blocks["to_v"]);
- auto q_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["q_norm"]);
- auto k_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["k_norm"]);
- auto to_out_0 = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
-
- auto q = to_q->forward(ctx, x);
- auto k = to_k->forward(ctx, context);
- auto v = to_v->forward(ctx, context);
-
- q = q_norm->forward(ctx, q);
- k = k_norm->forward(ctx, k);
-
- if (pe != nullptr) {
- if (k_pe == nullptr) {
- k_pe = pe;
- }
- q = apply_hidden_rope(ctx, q, pe, heads, dim_head, rope_interleaved);
- k = apply_hidden_rope(ctx, k, k_pe, heads, dim_head, rope_interleaved);
- }
-
- auto out = ggml_ext_attention_ext(ctx,
- q,
- k,
- v,
- heads,
- mask,
- false,
- ctx->flash_attn_enabled);
-
- if (blocks.count("to_gate_logits") > 0) {
- auto to_gate_logits = std::dynamic_pointer_cast<Linear>(blocks["to_gate_logits"]);
- auto gate_logits = to_gate_logits->forward(ctx, x);
- auto gates = ggml_sigmoid(ctx->ggml_ctx, gate_logits);
- gates = ggml_ext_scale(ctx, gates, 2.0f, true);
- gates = ggml_reshape_4d(ctx->ggml_ctx, gates, 1, heads, gate_logits->ne[1], gate_logits->ne[2]);
-
- auto out4 = ggml_reshape_4d(ctx->ggml_ctx, out, dim_head, heads, out->ne[1], out->ne[2]);
- gates = ggml_repeat(ctx->ggml_ctx, gates, out4);
- out4 = ggml_mul(ctx->ggml_ctx, out4, gates);
- out = ggml_reshape_3d(ctx->ggml_ctx, out4, heads * dim_head, out4->ne[2], out4->ne[3]);
- }
-
- return to_out_0->forward(ctx, out);
- }
- };
-
struct BasicTransformerBlock : public GGMLBlock {
int64_t dim;
bool cross_attention_adaln;
@@ -854,133 +418,6 @@ namespace LTXV {
}
};
- struct BasicTransformerBlock1D : public GGMLBlock {
- BasicTransformerBlock1D(int64_t dim,
- int64_t n_heads,
- int64_t d_head,
- bool rope_interleaved,
- bool apply_gated_attention = false) {
- blocks["attn1"] = std::make_shared<CrossAttention>(dim, dim, n_heads, d_head, apply_gated_attention, rope_interleaved);
- blocks["ff"] = std::make_shared<FeedForward>(dim, dim, 4, FeedForward::Activation::GELU);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx,
- ggml_tensor* x,
- ggml_tensor* pe,
- ggml_tensor* attention_mask = nullptr) {
- auto attn1 = std::dynamic_pointer_cast<CrossAttention>(blocks["attn1"]);
- auto ff = std::dynamic_pointer_cast<FeedForward>(blocks["ff"]);
-
- auto h = rms_norm(ctx->ggml_ctx, x);
- h = attn1->forward(ctx, h, nullptr, attention_mask, pe);
- x = ggml_add(ctx->ggml_ctx, x, h);
-
- h = rms_norm(ctx->ggml_ctx, x);
- h = ff->forward(ctx, h);
- x = ggml_add(ctx->ggml_ctx, x, h);
- return x;
- }
- };
-
- struct Embeddings1DConnector : public GGMLBlock {
- int64_t hidden_size;
- int64_t num_attention_heads;
- int64_t attention_head_dim;
- int64_t num_layers;
- int64_t num_learnable_registers;
- bool rope_interleaved;
- bool apply_gated_attention;
-
- void init_params(ggml_context* ctx,
- const String2TensorStorage& tensor_storage_map = {},
- const std::string prefix = "") override {
- if (num_learnable_registers > 0) {
- params["learnable_registers"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_learnable_registers);
- }
- }
-
- Embeddings1DConnector(int64_t hidden_size,
- int64_t num_attention_heads = 30,
- int64_t attention_head_dim = 128,
- int64_t num_layers = 2,
- int64_t num_learnable_registers = 128,
- bool rope_interleaved = false,
- bool apply_gated_attention = false)
- : hidden_size(hidden_size),
- num_attention_heads(num_attention_heads),
- attention_head_dim(attention_head_dim),
- num_layers(num_layers),
- num_learnable_registers(num_learnable_registers),
- rope_interleaved(rope_interleaved),
- apply_gated_attention(apply_gated_attention) {
- for (int i = 0; i < num_layers; i++) {
- blocks["transformer_1d_blocks." + std::to_string(i)] =
- std::make_shared<BasicTransformerBlock1D>(hidden_size,
- num_attention_heads,
- attention_head_dim,
- rope_interleaved,
- apply_gated_attention);
- }
- }
-
- ggml_tensor* append_registers(GGMLRunnerContext* ctx,
- ggml_tensor* hidden_states) {
- if (num_learnable_registers <= 0 || params.count("learnable_registers") == 0) {
- return hidden_states;
- }
-
- int64_t seq_len = hidden_states->ne[1];
- int64_t target_len = std::max<int64_t>(1024, seq_len);
- int64_t duplications = (target_len + num_learnable_registers - 1) / num_learnable_registers;
- int64_t total_to_keep = duplications * num_learnable_registers - seq_len;
- if (total_to_keep <= 0) {
- return hidden_states;
- }
-
- auto regs = ggml_reshape_3d(ctx->ggml_ctx, params["learnable_registers"], hidden_size, num_learnable_registers, 1);
- auto temp = ggml_new_tensor_3d(ctx->ggml_ctx, regs->type, regs->ne[0], regs->ne[1], hidden_states->ne[2]);
- regs = ggml_repeat(ctx->ggml_ctx, regs, temp);
-
- auto regs_full = regs;
- for (int64_t i = 1; i < duplications; i++) {
- regs_full = ggml_concat(ctx->ggml_ctx, regs_full, regs, 1);
- }
- regs_full = ggml_ext_slice(ctx, regs_full, 1, seq_len, seq_len + total_to_keep);
- return ggml_concat(ctx->ggml_ctx, hidden_states, regs_full, 1);
- }
-
- ggml_tensor* forward(GGMLRunnerContext* ctx,
- ggml_tensor* hidden_states,
- ggml_tensor* pe,
- ggml_tensor* attention_mask = nullptr) {
- hidden_states = append_registers(ctx, hidden_states);
-
- for (int i = 0; i < num_layers; i++) {
- auto block = std::dynamic_pointer_cast<BasicTransformerBlock1D>(blocks["transformer_1d_blocks." + std::to_string(i)]);
- hidden_states = block->forward(ctx, hidden_states, pe, attention_mask);
- }
-
- return ggml_rms_norm(ctx->ggml_ctx, hidden_states, 1e-6f);
- }
- };
-
- __STATIC_INLINE__ std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
- int64_t preferred_heads = -1) {
- if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
- return {preferred_heads, hidden_size / preferred_heads};
- }
- const int candidates[] = {128, 96, 80, 64, 48, 40, 32};
- for (int head_dim : candidates) {
- if (hidden_size % head_dim == 0) {
- int64_t heads = hidden_size / head_dim;
- if (heads >= 8 && heads <= 64) {
- return {heads, head_dim};
- }
- }
- }
- return {32, hidden_size / 32};
- }
-
__STATIC_INLINE__ std::vector<float> build_1d_rope_matrix_from_coords(const std::vector<float>& coords,
int dim,
int num_heads = 1,
@@ -1365,44 +802,6 @@ namespace LTXV {
blocks["av_ca_audio_scale_shift_adaln_single"] = std::make_shared<AdaLayerNormSingle>(config.audio_hidden_size, 4);
blocks["av_ca_v2a_gate_adaln_single"] = std::make_shared<AdaLayerNormSingle>(config.audio_hidden_size, 1);
- if (config.use_caption_projection) {
- if (config.caption_proj_before_connector) {
- if (config.caption_projection_first_linear) {
- blocks["caption_projection"] = std::make_shared<NormSingleLinearTextProjection>(config.caption_channels, config.hidden_size);
- }
- } else {
- blocks["caption_projection"] = std::make_shared<PixArtAlphaTextProjection>(config.caption_channels, config.hidden_size, config.hidden_size);
- }
- }
- if (config.use_audio_caption_projection) {
- if (config.caption_proj_before_connector) {
- if (config.caption_projection_first_linear) {
- blocks["audio_caption_projection"] = std::make_shared<NormSingleLinearTextProjection>(config.caption_channels, config.audio_hidden_size);
- }
- } else {
- blocks["audio_caption_projection"] = std::make_shared<PixArtAlphaTextProjection>(config.caption_channels, config.audio_hidden_size, config.audio_hidden_size);
- }
- }
-
- if (config.use_connector) {
- blocks["video_embeddings_connector"] = std::make_shared<Embeddings1DConnector>(config.connector_hidden_size,
- config.connector_num_heads,
- config.connector_head_dim,
- config.connector_num_layers,
- config.connector_num_registers,
- config.connector_rope_interleaved,
- config.connector_apply_gated_attention);
- }
- if (config.use_audio_connector) {
- blocks["audio_embeddings_connector"] = std::make_shared<Embeddings1DConnector>(config.audio_connector_hidden_size,
- config.audio_connector_num_heads,
- config.audio_connector_head_dim,
- config.audio_connector_num_layers,
- config.audio_connector_num_registers,
- config.audio_connector_rope_interleaved,
- config.audio_connector_apply_gated_attention);
- }
-
for (int i = 0; i < config.num_layers; i++) {
blocks["transformer_blocks." + std::to_string(i)] = std::make_shared<BasicAVTransformerBlock>(config.hidden_size,
config.audio_hidden_size,
@@ -1466,104 +865,6 @@ namespace LTXV {
return ax;
}
- std::pair<ggml_tensor*, ggml_tensor*> preprocess_contexts(GGMLRunnerContext* ctx,
- ggml_tensor* context,
- ggml_tensor* video_connector_pe,
- ggml_tensor* audio_connector_pe,
- bool process_audio_context) {
- if (context == nullptr) {
- return {nullptr, nullptr};
- }
-
- bool is_fully_processed_context =
- context->ne[0] == config.cross_attention_dim + config.audio_cross_attention_dim &&
- context->ne[1] >= 1024;
- bool is_unprocessed_dual_context =
- context->ne[0] == config.cross_attention_dim + config.audio_cross_attention_dim &&
- context->ne[1] < 1024;
-
- if (is_fully_processed_context) {
- auto v_context = ggml_ext_slice(ctx, context, 0, 0, config.cross_attention_dim);
- ggml_tensor* a_context = nullptr;
- if (process_audio_context) {
- a_context = ggml_ext_slice(ctx, context, 0, config.cross_attention_dim, config.cross_attention_dim + config.audio_cross_attention_dim);
- }
- return {v_context, a_context};
- }
-
- ggml_tensor* v_context = context;
- ggml_tensor* a_context = process_audio_context ? context : nullptr;
- if (is_unprocessed_dual_context) {
- v_context = ggml_ext_slice(ctx, context, 0, 0, config.cross_attention_dim);
- if (process_audio_context) {
- a_context = ggml_ext_slice(ctx, context, 0, config.cross_attention_dim, config.cross_attention_dim + config.audio_cross_attention_dim);
- }
- } else if (context->ne[0] == config.caption_channels * 2) {
- v_context = ggml_ext_slice(ctx, context, 0, 0, config.caption_channels);
- if (process_audio_context) {
- a_context = ggml_ext_slice(ctx, context, 0, config.caption_channels, config.caption_channels * 2);
- }
- }
-
- if (config.caption_proj_before_connector) {
- if (config.use_caption_projection &&
- blocks.count("caption_projection") > 0 &&
- v_context != nullptr &&
- v_context->ne[0] == config.caption_channels) {
- auto caption_projection = std::dynamic_pointer_cast<NormSingleLinearTextProjection>(blocks["caption_projection"]);
- if (caption_projection != nullptr) {
- v_context = caption_projection->forward(ctx, v_context);
- }
- }
- if (process_audio_context &&
- config.use_audio_caption_projection &&
- blocks.count("audio_caption_projection") > 0 &&
- a_context != nullptr &&
- a_context->ne[0] == config.caption_channels) {
- auto caption_projection = std::dynamic_pointer_cast<NormSingleLinearTextProjection>(blocks["audio_caption_projection"]);
- if (caption_projection != nullptr) {
- a_context = caption_projection->forward(ctx, a_context);
- }
- }
- }
-
- if (config.use_connector && v_context != nullptr && v_context->ne[0] == config.connector_hidden_size) {
- auto connector = std::dynamic_pointer_cast<Embeddings1DConnector>(blocks["video_embeddings_connector"]);
- v_context = connector->forward(ctx, v_context, video_connector_pe);
- }
- if (process_audio_context &&
- config.use_audio_connector &&
- a_context != nullptr &&
- a_context->ne[0] == config.audio_connector_hidden_size) {
- auto connector = std::dynamic_pointer_cast<Embeddings1DConnector>(blocks["audio_embeddings_connector"]);
- a_context = connector->forward(ctx, a_context, audio_connector_pe);
- }
-
- if (!config.caption_proj_before_connector &&
- config.use_caption_projection &&
- blocks.count("caption_projection") > 0 &&
- v_context != nullptr &&
- v_context->ne[0] == config.caption_channels) {
- auto caption_projection = std::dynamic_pointer_cast<PixArtAlphaTextProjection>(blocks["caption_projection"]);
- if (caption_projection != nullptr) {
- v_context = caption_projection->forward(ctx, v_context);
- }
- }
- if (process_audio_context &&
- !config.caption_proj_before_connector &&
- config.use_audio_caption_projection &&
- blocks.count("audio_caption_projection") > 0 &&
- a_context != nullptr &&
- a_context->ne[0] == config.caption_channels) {
- auto caption_projection = std::dynamic_pointer_cast<PixArtAlphaTextProjection>(blocks["audio_caption_projection"]);
- if (caption_projection != nullptr) {
- a_context = caption_projection->forward(ctx, a_context);
- }
- }
-
- return {v_context, a_context};
- }
-
// The video encoder is causal, so the first latent frame covers a single pixel frame while
// every later one covers temporal_scale_factor. LTX 2.5 marks that token class with a
// learned embedding added right after patchify_proj.
@@ -1617,8 +918,7 @@ namespace LTXV {
ggml_tensor* a_pe,
ggml_tensor* v_cross_pe,
ggml_tensor* a_cross_pe,
- ggml_tensor* video_connector_pe,
- ggml_tensor* audio_connector_pe) {
+ ggml_tensor* audio_context = nullptr) {
auto patchify_proj = std::dynamic_pointer_cast<Linear>(blocks["patchify_proj"]);
auto audio_patchify_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patchify_proj"]);
auto adaln_single = std::dynamic_pointer_cast<AdaLayerNormSingle>(blocks["adaln_single"]);
@@ -1646,12 +946,8 @@ namespace LTXV {
}
bool run_ax = ax != nullptr && ggml_nelements(ax) > 0 && audio_time > 0;
- auto contexts = preprocess_contexts(ctx, context, video_connector_pe, audio_connector_pe, run_ax);
- auto v_context = contexts.first;
- auto a_context = contexts.second != nullptr ? contexts.second : contexts.first;
- if (contexts.second != nullptr) {
- a_context = ggml_cont(ctx->ggml_ctx, a_context);
- }
+ auto v_context = context;
+ auto a_context = run_ax ? ggml_cont(ctx->ggml_ctx, audio_context) : nullptr;
auto v_timestep_scaled = ggml_ext_scale(ctx, timestep, config.timestep_scale_multiplier);
auto v_pair = adaln_single->forward(ctx, v_timestep_scaled);
@@ -1828,7 +1124,8 @@ namespace LTXV {
const sd::Tensor<float>& audio_timesteps_tensor = {},
int audio_length = 0,
float frame_rate = 24.f,
- const sd::Tensor<float>& video_positions_tensor = {}) {
+ const sd::Tensor<float>& video_positions_tensor = {},
+ const sd::Tensor<float>& audio_context_tensor = {}) {
auto split_inputs = split_av_latents(x_tensor, audio_length);
vx_input_cache = split_inputs.first;
if (!audio_x_tensor.empty()) {
@@ -1837,11 +1134,22 @@ namespace LTXV {
ax_input_cache = split_inputs.second;
}
- ggml_tensor* vx = make_input(vx_input_cache);
- ggml_tensor* ax = make_optional_input(ax_input_cache);
- ggml_tensor* timesteps = make_input(timesteps_tensor);
- ggml_tensor* a_timestep = make_optional_input(audio_timesteps_tensor);
- ggml_tensor* context = make_optional_input(context_tensor);
+ {
+ const auto valid = [](const sd::Tensor<float>& tensor, int64_t features) {
+ return !tensor.empty() && tensor.dim() >= 2 && tensor.dim() <= 3 && tensor.shape()[0] == features;
+ };
+ if (!valid(context_tensor, config.cross_attention_dim))
+ throw std::runtime_error("LTX diffusion requires the prepared video condition returned by the text encoder");
+ if (!ax_input_cache.empty() && !valid(audio_context_tensor, config.audio_cross_attention_dim))
+ throw std::runtime_error("LTX audio diffusion requires the prepared audio condition in extra_crossattn");
+ }
+
+ ggml_tensor* vx = make_input(vx_input_cache);
+ ggml_tensor* ax = make_optional_input(ax_input_cache);
+ ggml_tensor* timesteps = make_input(timesteps_tensor);
+ ggml_tensor* a_timestep = make_optional_input(audio_timesteps_tensor);
+ ggml_tensor* context = make_optional_input(context_tensor);
+ ggml_tensor* audio_context = make_optional_input(audio_context_tensor);
ggml_cgraph* gf = new_graph_custom(LTXAV_GRAPH_SIZE);
@@ -1945,55 +1253,6 @@ namespace LTXV {
}));
}
- bool needs_video_connector_pe =
- config.use_connector &&
- context != nullptr &&
- (context->ne[0] == config.connector_hidden_size ||
- ((context->ne[0] == config.cross_attention_dim + config.audio_cross_attention_dim ||
- context->ne[0] == config.caption_channels * 2) &&
- context->ne[1] < 1024));
- ggml_tensor* video_connector_pe = nullptr;
- if (needs_video_connector_pe) {
- int64_t seq_len = context->ne[1];
- int64_t target_len = std::max<int64_t>(1024, seq_len);
- int64_t duplications = (target_len + config.connector_num_registers - 1) / config.connector_num_registers;
- int64_t full_len = seq_len + duplications * config.connector_num_registers - seq_len;
- sd::PECacheKey video_connector_pe_key("ltxav_video_connector_pe");
- video_connector_pe_key.add(full_len).add(config.connector_hidden_size).add(config.connector_num_heads);
- video_connector_pe_key.add(config.connector_head_dim);
- video_connector_pe = make_input(pe_cache_.get_or_create(video_connector_pe_key, [&]() {
- auto connector_pe_vec = build_1d_rope_matrix(full_len, static_cast<int>(config.connector_hidden_size), static_cast<int>(config.connector_num_heads), 10000.f, 4096.f, true);
- return sd::Tensor<float>({2, 2, config.connector_head_dim / 2, full_len * config.connector_num_heads}, std::move(connector_pe_vec));
- }));
- }
-
- bool run_audio_context =
- ax != nullptr &&
- ggml_nelements(ax) > 0 &&
- ax->ne[1] > 0;
- bool needs_audio_connector_pe =
- run_audio_context &&
- config.use_audio_connector &&
- context != nullptr &&
- (context->ne[0] == config.audio_connector_hidden_size ||
- ((context->ne[0] == config.cross_attention_dim + config.audio_cross_attention_dim ||
- context->ne[0] == config.caption_channels * 2) &&
- context->ne[1] < 1024));
- ggml_tensor* audio_connector_pe = nullptr;
- if (needs_audio_connector_pe) {
- int64_t seq_len = context->ne[1];
- int64_t target_len = std::max<int64_t>(1024, seq_len);
- int64_t duplications = (target_len + config.audio_connector_num_registers - 1) / config.audio_connector_num_registers;
- int64_t full_len = seq_len + duplications * config.audio_connector_num_registers - seq_len;
- sd::PECacheKey audio_connector_pe_key("ltxav_audio_connector_pe");
- audio_connector_pe_key.add(full_len).add(config.audio_connector_hidden_size).add(config.audio_connector_num_heads);
- audio_connector_pe_key.add(config.audio_connector_head_dim);
- audio_connector_pe = make_input(pe_cache_.get_or_create(audio_connector_pe_key, [&]() {
- auto audio_connector_pe_vec = build_1d_rope_matrix(full_len, static_cast<int>(config.audio_connector_hidden_size), static_cast<int>(config.audio_connector_num_heads), 10000.f, 4096.f, true);
- return sd::Tensor<float>({2, 2, config.audio_connector_head_dim / 2, full_len * config.audio_connector_num_heads}, std::move(audio_connector_pe_vec));
- }));
- }
-
auto runner_ctx = get_context();
auto out_pair = model.forward(&runner_ctx,
vx,
@@ -2005,8 +1264,7 @@ namespace LTXV {
audio_pe,
video_cross_pe,
audio_cross_pe,
- video_connector_pe,
- audio_connector_pe);
+ audio_context);
auto out = merge_av_latents(&runner_ctx, out_pair.first, out_pair.second);
ggml_build_forward_expand(gf, out);
return gf;
@@ -2020,9 +1278,10 @@ namespace LTXV {
const sd::Tensor<float>& audio_timesteps = {},
int audio_length = 0,
float frame_rate = 24.f,
- const sd::Tensor<float>& video_positions = {}) {
+ const sd::Tensor<float>& video_positions = {},
+ const sd::Tensor<float>& audio_context = {}) {
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions);
+ return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions, audio_context);
};
auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
return out;
@@ -2041,14 +1300,16 @@ namespace LTXV {
tensor_or_empty(extra->audio_timesteps),
extra->audio_length,
extra->frame_rate,
- tensor_or_empty(extra->video_positions));
+ tensor_or_empty(extra->video_positions),
+ tensor_or_empty(extra->audio_context));
}
void test(const std::string& x_path,
const std::string& timesteps_path = "",
const std::string& context_path = "",
const std::string& audio_x_path = "",
- const std::string& audio_timesteps_path = "") {
+ const std::string& audio_timesteps_path = "",
+ const std::string& audio_context_path = "") {
auto x = sd::load_tensor_from_file_as_tensor<float>(x_path);
GGML_ASSERT(!x.empty());
print_sd_tensor(x, false, "ltxav_x");
@@ -2090,8 +1351,15 @@ namespace LTXV {
print_sd_tensor(audio_timesteps, false, "ltxav_audio_timesteps");
}
+ sd::Tensor<float> audio_context;
+ if (!audio_context_path.empty()) {
+ audio_context = sd::load_tensor_from_file_as_tensor<float>(audio_context_path);
+ GGML_ASSERT(!audio_context.empty());
+ print_sd_tensor(audio_context, false, "ltxav_audio_context");
+ }
+
int64_t t0 = ggml_time_ms();
- auto out_opt = compute(8, x, timesteps, context, audio_x, audio_timesteps, audio_length);
+ auto out_opt = compute(8, x, timesteps, context, audio_x, audio_timesteps, audio_length, 24.f, {}, audio_context);
int64_t t1 = ggml_time_ms();
GGML_ASSERT(!out_opt.empty());
@@ -2103,9 +1371,9 @@ namespace LTXV {
const std::string& x_path,
const std::string& timesteps_path = "",
const std::string& context_path = "",
- const std::string& embeddings_path = "",
const std::string& audio_x_path = "",
- const std::string& audio_timesteps_path = "") {
+ const std::string& audio_timesteps_path = "",
+ const std::string& audio_context_path = "") {
// ggml_backend_t backend = ggml_backend_cuda_init(0);
ggml_backend_t backend = sd_backend_cpu_init();
LOG_INFO("loading ltxav from '%s'", model_path.c_str());
@@ -2116,14 +1384,6 @@ namespace LTXV {
LOG_ERROR("init model loader from file failed: '%s'", model_path.c_str());
return;
}
- if (!embeddings_path.empty()) {
- LOG_INFO("loading ltxav embeddings from '%s'", embeddings_path.c_str());
- if (!model_loader.init_from_file(embeddings_path)) {
- LOG_ERROR("init embeddings model loader from file failed: '%s'", embeddings_path.c_str());
- return;
- }
- }
-
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
std::shared_ptr<LTXAVRunner> ltxav = std::make_shared<LTXAVRunner>(backend,
tensor_storage_map,
@@ -2143,7 +1403,7 @@ namespace LTXV {
}
LOG_INFO("ltxav model loaded");
- ltxav->test(x_path, timesteps_path, context_path, audio_x_path, audio_timesteps_path);
+ ltxav->test(x_path, timesteps_path, context_path, audio_x_path, audio_timesteps_path, audio_context_path);
}
};
diff --git a/src/model/diffusion/mage_flow.hpp b/src/model/diffusion/mage_flow.hpp
index 415c0f7..aaba749 100644
--- a/src/model/diffusion/mage_flow.hpp
+++ b/src/model/diffusion/mage_flow.hpp
@@ -160,7 +160,7 @@ namespace MageFlow {
*diffusion_params.x,
*diffusion_params.timesteps,
tensor_or_empty(diffusion_params.context),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents);
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents);
}
};
} // namespace MageFlow
diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp
index ed20c80..b717192 100644
--- a/src/model/diffusion/model.hpp
+++ b/src/model/diffusion/model.hpp
@@ -10,61 +10,14 @@
#include "model/common/rope_circular.hpp"
#include "model_manager.h"
-enum class RefImageResizeMode {
- NONE,
- LONGEST_SIDE,
- AREA,
-};
-
-enum class RefImageSizeMode {
- MATCH,
- MAX,
-};
-
-struct RefImageParams {
- bool pass_to_vlm = false;
- bool pass_to_dit = true;
- Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED;
- bool force_ref_timestep_zero = false;
- bool resize_before_vae = true;
- int vae_input_max_pixels = -1;
- RefImageResizeMode vlm_resize_mode = RefImageResizeMode::AREA;
- int vlm_min_size = -1;
- int vlm_max_size = -1;
- bool resize_vae_to_target = false;
- bool resize_single_ref_to_target = false;
- bool vae_size_buckets = false;
- bool vae_center_crop = false;
- bool vlm_use_vae_input = false;
- RefImageSizeMode ref_image_size = RefImageSizeMode::MATCH;
-};
-
-const std::unordered_map<std::string, RefImageParams> REF_IMAGE_PRESETS = {
- {"fibo2", [] {
- RefImageParams params;
- params.pass_to_vlm = true;
- params.ref_index_mode = Rope::RefIndexMode::INCREASE;
- params.vlm_resize_mode = RefImageResizeMode::LONGEST_SIDE;
- params.vlm_max_size = 768;
- params.resize_single_ref_to_target = true;
- params.vae_size_buckets = true;
- params.vae_center_crop = true;
- params.vlm_use_vae_input = true;
- return params;
- }()},
- {"flux_kontext", {false, true, Rope::RefIndexMode::FIXED, false, true, -1, RefImageResizeMode::NONE, -1, -1}},
- {"longcat", {true, true, Rope::RefIndexMode::FIXED, false, true, -1, RefImageResizeMode::AREA, -1, -1}},
- {"flux2", {false, true, Rope::RefIndexMode::INCREASE, false, true, -1, RefImageResizeMode::NONE, -1, -1}},
- {"qwen", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, RefImageResizeMode::AREA, -1, -1}},
- {"qwen_layered", {true, true, Rope::RefIndexMode::DECREASE, false, true, -1, RefImageResizeMode::AREA, -1, -1}},
- {"mage_flow", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, RefImageResizeMode::LONGEST_SIDE, -1, 384, true}},
- {"z_image_omni", {true, true, Rope::RefIndexMode::FIXED, false, true, -1, RefImageResizeMode::AREA, -1, -1}},
- {"krea2_ostris_edit", {true, true, Rope::RefIndexMode::INCREASE, true, true, -1, RefImageResizeMode::AREA, -1, -1}},
- {"krea2_edit", {true, true, Rope::RefIndexMode::INCREASE, false, true, -1, RefImageResizeMode::LONGEST_SIDE, 768, 768}},
- // pass_to_vlm routes the reference image to the conditioner, which is where LLaDA-Image's
- // SigVQ encoder lives; it does its own half-resolution resize.
- {"llada_image", {true, true, Rope::RefIndexMode::FIXED, true, true, -1, RefImageResizeMode::NONE, -1, -1, true}},
- {"cosmos_reference", {false, true, Rope::RefIndexMode::INCREASE, false, false, -1, RefImageResizeMode::NONE, -1, -1}},
+enum class UNetBlockStage { Down,
+ Middle,
+ Up };
+
+struct UNetBlockProcessor {
+ virtual ~UNetBlockProcessor() = default;
+ virtual ggml_tensor* apply(GGMLRunnerContext& ctx, UNetBlockStage stage, int level, int block, ggml_tensor* activation, int frames) const = 0;
+ virtual size_t extra_graph_size() const = 0;
};
struct UNetDiffusionExtra {
@@ -73,17 +26,27 @@ struct UNetDiffusionExtra {
float control_strength = 0.f;
const sd::Tensor<float>* ip_context = nullptr;
float ip_scale = 1.f;
+ const AttentionProcessor* attn2_processor = nullptr;
+ const UNetBlockProcessor* block_processor = nullptr;
};
struct SkipLayerDiffusionExtra {
const std::vector<int>* skip_layers = nullptr;
};
+struct FluxImageBlockProcessor {
+ virtual ~FluxImageBlockProcessor() = default;
+ virtual bool applies(bool single_stream, int block) const = 0;
+ virtual ggml_tensor* apply(GGMLRunnerContext& ctx, bool single_stream, int block, ggml_tensor* image) const = 0;
+ virtual size_t extra_graph_size() const = 0;
+};
+
struct FluxDiffusionExtra {
- const sd::Tensor<float>* guidance = nullptr;
- const std::vector<int>* skip_layers = nullptr;
- const sd::Tensor<float>* pulid_id = nullptr;
- float pulid_id_weight = 1.0f;
+ const sd::Tensor<float>* guidance = nullptr;
+ const std::vector<int>* skip_layers = nullptr;
+ const sd::Tensor<float>* pulid_id = nullptr;
+ float pulid_id_weight = 1.0f;
+ const FluxImageBlockProcessor* image_processor = nullptr;
};
struct Flux3DiffusionExtra {
@@ -122,6 +85,7 @@ struct LTXAVDiffusionExtra {
int audio_length = 0;
float frame_rate = 24.f;
const sd::Tensor<float>* video_positions = nullptr;
+ const sd::Tensor<float>* audio_context = nullptr;
};
enum class MiniMaxH3ReferenceKind : int32_t {
@@ -172,9 +136,20 @@ struct MingImageDiffusionExtra {
const sd::Tensor<float>* direct_context = nullptr;
};
+namespace ZImage {
+ struct ControlNetRunner;
+}
+
+struct ZImageControlInput {
+ const ZImage::ControlNetRunner* runner;
+ const sd::Tensor<float>* latent;
+ float strength;
+};
+
struct ZImageDiffusionExtra {
- const sd::Tensor<float>* control_latent = nullptr;
- float strength = 1.f;
+ const sd::Tensor<float>* control_latent = nullptr;
+ float strength = 1.f;
+ const std::vector<ZImageControlInput>* controls = nullptr;
};
using DiffusionExtraParams = std::variant<std::monostate,
@@ -202,7 +177,8 @@ struct DiffusionParams {
const sd::Tensor<float>* c_concat = nullptr;
const sd::Tensor<float>* y = nullptr;
const std::vector<sd::Tensor<float>>* ref_latents = nullptr;
- RefImageParams ref_image_params = {false, false, Rope::RefIndexMode::FIXED, false};
+ Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED;
+ bool force_ref_timestep_zero = false;
DiffusionExtraParams extra = std::monostate{};
};
diff --git a/src/model/diffusion/qwen_image.hpp b/src/model/diffusion/qwen_image.hpp
index a9aa41b..95c0da8 100644
--- a/src/model/diffusion/qwen_image.hpp
+++ b/src/model/diffusion/qwen_image.hpp
@@ -723,8 +723,8 @@ namespace Qwen {
*diffusion_params.x,
*diffusion_params.timesteps,
tensor_or_empty(diffusion_params.context),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents,
- diffusion_params.ref_image_params.ref_index_mode);
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+ diffusion_params.ref_index_mode);
}
void test() {
diff --git a/src/model/diffusion/qwen_image_2_1.hpp b/src/model/diffusion/qwen_image_2_1.hpp
index 00b88f4..5122ae6 100644
--- a/src/model/diffusion/qwen_image_2_1.hpp
+++ b/src/model/diffusion/qwen_image_2_1.hpp
@@ -460,7 +460,7 @@ namespace Qwen {
return {};
}
static const std::vector<sd::Tensor<float>> empty_refs;
- const auto& refs = inputs.ref_latents && inputs.ref_image_params.pass_to_dit ? *inputs.ref_latents : empty_refs;
+ const auto& refs = inputs.ref_latents ? *inputs.ref_latents : empty_refs;
std::vector<std::pair<int64_t, int64_t>> shapes;
for (const auto& ref : refs) {
if (ref.dim() != 4 || ref.shape()[2] != config.in_channels || ref.shape()[3] != 1) {
diff --git a/src/model/diffusion/relay_vsr.cpp b/src/model/diffusion/relay_vsr.cpp
new file mode 100644
index 0000000..2a50ca4
--- /dev/null
+++ b/src/model/diffusion/relay_vsr.cpp
@@ -0,0 +1,80 @@
+#include "relay_vsr.hpp"
+
+namespace RelayVSR {
+
+ static int64_t aligned_vsr_size(int64_t size, int scale) {
+ const int alignment = 32 / scale;
+ return (size + alignment - 1) / alignment * alignment;
+ }
+
+ static sd::Tensor<float> pad_vsr_frame(sd::Tensor<float> result, int64_t width, int64_t height) {
+ const auto w = result.shape()[0], h = result.shape()[1];
+ if (width > w) {
+ auto edge = sd::ops::slice(result, 0, w - 1, w);
+ auto padding = sd::zeros<float>({width - w, h, 3, 1}) + edge;
+ result = sd::ops::concat(result, padding, 0);
+ }
+ if (height > h) {
+ auto edge = sd::ops::slice(result, 1, h - 1, h);
+ auto padding = sd::zeros<float>({width, height - h, 3, 1}) + edge;
+ result = sd::ops::concat(result, padding, 1);
+ }
+ return result;
+ }
+
+ static sd::Tensor<float> prepare_vsr_lq(const sd::Tensor<float>& frame, int scale) {
+ auto input = frame;
+ if (scale == 4)
+ input = sd::ops::interpolate(frame, {frame.shape()[0] * 4, frame.shape()[1] * 4, 3, 1}, sd::ops::InterpolateMode::Bilinear);
+ const int patch = 4 * scale;
+ const int width = int(input.shape()[0]), height = int(input.shape()[1]);
+ input.reshape_({patch, width / patch, patch, height / patch, 3, 1});
+ input.permute_({1, 3, 0, 2, 4, 5});
+ input.reshape_({width / patch, height / patch, 3 * patch * patch, 1});
+ return input;
+ }
+
+ static std::vector<int64_t> vsr_latent_shape(int scale, int64_t width, int64_t height, int64_t frames) {
+ const int stride = 16 / scale;
+ return {aligned_vsr_size(width, scale) / stride, aligned_vsr_size(height, scale) / stride, frames, 48, 1};
+ }
+
+ sd::Tensor<float> WanVSRRunner::compute(int n_threads, const DiffusionParams& params) {
+ const auto& noise = *params.x;
+ if (!params.ref_latents || params.ref_latents->size() != 1)
+ throw std::runtime_error("RelayVSR requires one low-quality keyframe tensor");
+ const auto& lq = params.ref_latents->front();
+ if (lq.dim() != 5 || lq.shape()[3] != 3 || lq.shape()[4] != 1)
+ throw std::runtime_error("RelayVSR requires lq_frames with shape [width, height, keyframes, 3, 1]");
+ const auto w = lq.shape()[0], h = lq.shape()[1], count = lq.shape()[2];
+ const int scale = config.upscale_factor;
+ if (noise.shape() != vsr_latent_shape(scale, w, h, count))
+ throw std::runtime_error("RelayVSR noise shape does not match the low-quality keyframes");
+ if (!params.context || params.context->shape() != std::vector<int64_t>{config.text_dim, config.text_len})
+ throw std::runtime_error("RelayVSR requires a compatible positive embedding");
+ reset();
+ struct Reset {
+ WanVSRRunner& model;
+ ~Reset() { model.reset(); }
+ } cleanup{*this};
+ const auto time = time_embedding((*params.timesteps)[0]);
+ const auto pw = aligned_vsr_size(w, scale), ph = aligned_vsr_size(h, scale);
+ sd::Tensor<float> output(noise.shape());
+ for (int64_t i = 0; i < count; ++i) {
+ auto input = sd::ops::slice(lq, 2, i, i + 1);
+ input.reshape_({w, h, 3, 1});
+ input *= 2.f;
+ input -= 1.f;
+ input = pad_vsr_frame(std::move(input), pw, ph);
+ auto sample = sd::ops::slice(noise, 2, i, i + 1);
+ sample.reshape_({noise.shape()[0], noise.shape()[1], 1, 48});
+ auto velocity = predict_frame(n_threads, sample, prepare_vsr_lq(input, scale), time, params.context);
+ if (velocity.empty())
+ return {};
+ velocity.reshape_({noise.shape()[0], noise.shape()[1], 1, 48, 1});
+ sd::ops::slice_assign(&output, 2, i, i + 1, velocity);
+ }
+ return output;
+ }
+
+} // namespace RelayVSR
diff --git a/src/model/diffusion/relay_vsr.hpp b/src/model/diffusion/relay_vsr.hpp
index 12c7a1a..fe0186c 100644
--- a/src/model/diffusion/relay_vsr.hpp
+++ b/src/model/diffusion/relay_vsr.hpp
@@ -2,6 +2,7 @@
#define __SD_MODEL_DIFFUSION_RELAY_VSR_HPP__
#include "model/common/vsr.hpp"
+#include "model/diffusion/model.hpp"
namespace RelayVSR {
@@ -194,7 +195,7 @@ namespace RelayVSR {
blocks["proj_out"] = std::make_shared<Linear>(config.dim, config.in_channels * 4);
}
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* latent, ggml_tensor* lq, ggml_tensor* timestep, ggml_tensor* qpe, ggml_tensor* kpe, const VSR::LocalAttention& local) {
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* latent, ggml_tensor* lq, ggml_tensor* timestep, ggml_tensor* qpe, ggml_tensor* kpe, const VSR::LocalAttention& local, ggml_tensor* prompt) {
auto patch_embedding = std::dynamic_pointer_cast<Conv3d>(blocks["patch_embedding"]);
auto condition_embedder_time_embedder_linear_1 = std::dynamic_pointer_cast<Linear>(blocks["condition_embedder.time_embedder.linear_1"]);
auto condition_embedder_time_embedder_linear_2 = std::dynamic_pointer_cast<Linear>(blocks["condition_embedder.time_embedder.linear_2"]);
@@ -212,7 +213,7 @@ namespace RelayVSR {
e = condition_embedder_time_embedder_linear_2->forward(ctx, ggml_silu(c, e));
auto e0 = condition_embedder_time_proj->forward(ctx, ggml_silu(c, e));
e0 = ggml_reshape_3d(c, e0, config_.dim, 6, 1);
- auto text = condition_embedder_text_embedder_linear_1->forward(ctx, params["prompt_embeds"]);
+ auto text = condition_embedder_text_embedder_linear_1->forward(ctx, prompt ? prompt : params["prompt_embeds"]);
text = condition_embedder_text_embedder_linear_2->forward(ctx, ggml_ext_gelu(ctx, text));
for (int i = 0; i < config_.num_layers; ++i) {
auto name = "blocks." + std::to_string(i);
@@ -230,7 +231,7 @@ namespace RelayVSR {
}
};
- struct WanVSRRunner : GGMLRunner {
+ struct WanVSRRunner : DiffusionModelRunner {
WanVSRConfig config;
WanTransformer3DModel model;
VSR::GraphInputs inputs;
@@ -239,17 +240,24 @@ namespace RelayVSR {
int frame = 0;
WanVSRRunner(ggml_backend_t backend, const String2TensorStorage& weights, int scale, std::shared_ptr<DeviceResidencyManager> manager)
- : GGMLRunner(backend, manager), config(WanVSRConfig::detect_from_weights(weights, "wan", scale)), model(config), timestep({256}) {
+ : DiffusionModelRunner(backend, "wan", manager), config(WanVSRConfig::detect_from_weights(weights, "wan", scale)), model(config), timestep(time_embedding(1000.f)) {
model.init(params_ctx, weights, "wan");
+ }
+
+ static sd::Tensor<float> time_embedding(float time) {
+ sd::Tensor<float> embedding({256});
for (int i = 0; i < 128; ++i) {
- double angle = 1000.0 * std::pow(10000.0, -double(i) / 128);
- timestep.data()[i] = float(std::cos(angle));
- timestep.data()[i + 128] = float(std::sin(angle));
+ double angle = double(time) * std::pow(10000.0, -double(i) / 128);
+ embedding.data()[i] = float(std::cos(angle));
+ embedding.data()[i + 128] = float(std::sin(angle));
}
+ return embedding;
}
std::string get_desc() override { return "RelayVSR Wan"; }
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model.get_param_tensors(tensors, "wan"); }
+ using DiffusionModelRunner::get_param_tensors;
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override { model.get_param_tensors(tensors, prefix); }
+ sd::Tensor<float> compute(int n_threads, const DiffusionParams& params) override;
void reset() {
runner_end();
free_cache_ctx_and_buffer();
@@ -258,7 +266,7 @@ namespace RelayVSR {
frame = 0;
}
- sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& noise, const sd::Tensor<float>& lq) {
+ sd::Tensor<float> predict_frame(int n_threads, const sd::Tensor<float>& noise, const sd::Tensor<float>& lq, const sd::Tensor<float>& time, const sd::Tensor<float>* prompt = nullptr) {
int width = int(noise.shape()[0]) / 2, height = int(noise.shape()[1]) / 2;
int count = std::min(frame + 1, 4);
if (!attention.count(count))
@@ -274,7 +282,7 @@ namespace RelayVSR {
times.push_back(i - origin);
auto qpe = inputs.rope(&ctx, width, height, {frame - origin}, {44, 42, 42});
auto kpe = inputs.rope(&ctx, width, height, times, {44, 42, 42});
- auto out = model.forward(&ctx, make_input(noise), make_input(lq), make_input(timestep), qpe, kpe, *attention[count]);
+ auto out = model.forward(&ctx, make_input(noise), make_input(lq), make_input(time), qpe, kpe, *attention[count], prompt ? make_input(*prompt) : nullptr);
ggml_build_forward_expand(gf, out);
return gf;
};
@@ -282,6 +290,13 @@ namespace RelayVSR {
if (prediction.empty())
return {};
++frame;
+ return prediction;
+ }
+
+ sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& noise, const sd::Tensor<float>& lq) {
+ auto prediction = predict_frame(n_threads, noise, lq, timestep);
+ if (prediction.empty())
+ return {};
for (int64_t i = 0; i < prediction.numel(); ++i)
prediction.data()[i] = noise.data()[i] - prediction.data()[i];
return prediction;
diff --git a/src/model/diffusion/seedvr2.hpp b/src/model/diffusion/seedvr2.hpp
index dad1e7a..2fded24 100644
--- a/src/model/diffusion/seedvr2.hpp
+++ b/src/model/diffusion/seedvr2.hpp
@@ -2,6 +2,7 @@
#define __SD_MODEL_DIFFUSION_SEEDVR2_HPP__
#include "model/common/vsr.hpp"
+#include "model/diffusion/model.hpp"
namespace SeedVR2 {
@@ -272,10 +273,10 @@ namespace SeedVR2 {
blocks["vid_out_ada"] = std::make_shared<AdaSingle>(config.dim, true);
}
}
- ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* input, ggml_tensor* timestep, const std::vector<WindowPlan>& plans, VSR::GraphInputs& inputs) {
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* input, ggml_tensor* timestep, const std::vector<WindowPlan>& plans, VSR::GraphInputs& inputs, ggml_tensor* context) {
auto linear = [&](const std::string& name, ggml_tensor* x) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x); };
auto vid = linear("vid_in.proj", input);
- auto txt = linear("txt_in", params["positive_conditioning"]);
+ auto txt = linear("txt_in", context ? context : params["positive_conditioning"]);
auto emb = linear("emb_in.proj_in", timestep);
emb = linear("emb_in.proj_hid", ggml_silu(ctx->ggml_ctx, emb));
emb = linear("emb_in.proj_out", ggml_silu(ctx->ggml_ctx, emb));
@@ -293,24 +294,39 @@ namespace SeedVR2 {
}
};
- struct SeedVR2Runner : GGMLRunner {
+ struct SeedVR2Runner : DiffusionModelRunner {
SeedVR2Config config;
NaDiT model;
sd::Tensor<float> timestep;
- SeedVR2Runner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<DeviceResidencyManager> manager)
- : GGMLRunner(backend, manager), config(SeedVR2Config::detect_from_weights(weights, "diffusion")), model(config), timestep({256}) {
- model.init(params_ctx, weights, "diffusion");
+ SeedVR2Runner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<RunnerWeightManager> manager, const std::string& prefix = "diffusion")
+ : DiffusionModelRunner(backend, prefix, manager),
+ config(SeedVR2Config::detect_from_weights(weights, prefix)),
+ model(config),
+ timestep(timestep_embedding(1000.0)) {
+ model.init(params_ctx, weights, prefix);
+ }
+
+ static sd::Tensor<float> timestep_embedding(double time) {
+ sd::Tensor<float> embedding({256});
for (int i = 0; i < 128; ++i) {
- const double angle = 1000.0 * std::pow(10000.0, -double(i) / 128);
- timestep.data()[i] = float(std::sin(angle));
- timestep.data()[i + 128] = float(std::cos(angle));
+ const double angle = time * std::pow(10000.0, -double(i) / 128);
+ embedding[i] = float(std::sin(angle));
+ embedding[i + 128] = float(std::cos(angle));
}
+ return embedding;
}
+
std::string get_desc() override { return "SeedVR2 DiT"; }
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model.get_param_tensors(tensors, "diffusion"); }
+ int get_patch_size() const override { return 2; }
+ using DiffusionModelRunner::get_param_tensors;
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+ model.get_param_tensors(tensors, prefix);
+ }
+ std::string positive_embedding_weight() const { return prefix + ".positive_conditioning"; }
- sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& input, int width, int height, int frames) {
+ sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& input, int width, int height, int frames, const sd::Tensor<float>* context = nullptr, double time = 1000.0) {
+ const auto embedding = time == 1000.0 ? timestep : timestep_embedding(time);
std::vector<WindowPlan> plans;
for (bool shifted : {false, true})
plans.emplace_back(width, height, frames, config.text_len, shifted, config.swiglu);
@@ -319,12 +335,44 @@ namespace SeedVR2 {
inputs.begin_graph();
auto gf = new_graph_custom(MAX_GRAPH_SIZE);
auto ctx = get_context(gf);
- auto out = model.forward(&ctx, make_input(input), make_input(timestep), plans, inputs);
+ auto out = model.forward(&ctx, make_input(input), make_input(embedding), plans, inputs, context ? make_input(*context) : nullptr);
ggml_build_forward_expand(gf, out);
return gf;
};
auto output = GGMLRunner::compute(graph, n_threads);
- return output ? *output : sd::Tensor<float>();
+ return take_or_empty(std::move(output));
+ }
+
+ sd::Tensor<float> compute(int n_threads, const DiffusionParams& params) override {
+ const auto& x = tensor_or_empty(params.x);
+ const auto& condition = tensor_or_empty(params.c_concat);
+ const auto& times = tensor_or_empty(params.timesteps);
+ const auto& context = tensor_or_empty(params.context);
+ if (x.empty() || (x.dim() != 4 && x.dim() != 5) || x.shape().back() != 1 ||
+ x.shape()[0] % 2 || x.shape()[1] % 2 || x.shape()[x.dim() - 2] != 16 ||
+ times.numel() != 1 || !std::isfinite(times[0])) {
+ throw std::runtime_error("SeedVR2 requires a single 16-channel latent, even width and height, and one timestep");
+ }
+ const auto channels = x.dim() - 2;
+ const auto w = x.shape()[0], h = x.shape()[1], t = x.dim() == 5 ? x.shape()[2] : 1;
+ auto expected_condition = x.shape();
+ expected_condition[channels] = 17;
+ if (condition.shape() != expected_condition)
+ throw std::runtime_error("SeedVR2 concat conditioning requires 16 latent channels and a one-channel task mask");
+ if (context.shape() != std::vector<int64_t>{config.text_dim, config.text_len})
+ throw std::runtime_error("SeedVR2 requires a matching crossattn embedding");
+
+ auto input = sd::ops::concat(x, condition, channels);
+ input.reshape_({2, w / 2, 2, h / 2, t, 33});
+ input.permute_({5, 0, 2, 1, 3, 4});
+ input.reshape_({132, (w / 2) * (h / 2) * t});
+ auto output = compute(n_threads, input, int(w / 2), int(h / 2), int(t), &context, times[0]);
+ if (output.empty())
+ return {};
+ output.reshape_({16, 2, 2, w / 2, h / 2, t});
+ output.permute_({1, 3, 2, 4, 5, 0});
+ output.reshape_(x.shape());
+ return output;
}
};
}
diff --git a/src/model/diffusion/unet.hpp b/src/model/diffusion/unet.hpp
index 767dfc1..85a8e8a 100644
--- a/src/model/diffusion/unet.hpp
+++ b/src/model/diffusion/unet.hpp
@@ -528,11 +528,12 @@ public:
ggml_tensor* x,
ggml_tensor* timesteps,
ggml_tensor* context,
- ggml_tensor* c_concat = nullptr,
- ggml_tensor* y = nullptr,
- int num_video_frames = -1,
- std::vector<ggml_tensor*> controls = {},
- float control_strength = 0.f) {
+ ggml_tensor* c_concat = nullptr,
+ ggml_tensor* y = nullptr,
+ int num_video_frames = -1,
+ std::vector<ggml_tensor*> controls = {},
+ float control_strength = 0.f,
+ const UNetBlockProcessor* block_processor = nullptr) {
// x: [N, in_channels, h, w] or [N, in_channels/2, h, w]
// timesteps: [N,]
// context: [N, max_position, hidden_size] or [1, max_position, hidden_size]. for example, [N, 77, 768]
@@ -603,7 +604,7 @@ public:
ggml_set_name(h, "bench-start");
hs.push_back(h);
- auto motion_root = config.enable_animatediff && num_video_frames > 1
+ auto motion_root = !block_processor && config.enable_animatediff && num_video_frames > 1
? std::dynamic_pointer_cast<AnimateDiff::AnimateDiffModel>(blocks["motion_module"])
: nullptr;
auto apply_motion_input = [&](int input_block_idx, ggml_tensor* h_in) -> ggml_tensor* {
@@ -653,6 +654,8 @@ public:
h = attention_layer_forward(name, ctx, h, context, num_video_frames); // [N, mult*model_channels, h, w]
}
h = apply_motion_input(input_block_idx, h);
+ if (block_processor && num_video_frames > 1)
+ h = block_processor->apply(*ctx, UNetBlockStage::Down, int(i), j, h, num_video_frames);
sd::ggml_graph_cut::mark_graph_cut(h, "unet.input_blocks." + std::to_string(input_block_idx), "h");
hs.push_back(h);
}
@@ -681,6 +684,8 @@ public:
h = resblock_forward("middle_block.2", ctx, h, emb, num_video_frames); // [N, 4*model_channels, h/8, w/8]
}
h = apply_motion_mid(h);
+ if (block_processor && num_video_frames > 1)
+ h = block_processor->apply(*ctx, UNetBlockStage::Middle, 0, 0, h, num_video_frames);
}
sd::ggml_graph_cut::mark_graph_cut(h, "unet.middle_block", "h");
if (controls.size() > 0) {
@@ -720,6 +725,8 @@ public:
}
h = apply_motion_output(output_block_idx, h);
+ if (block_processor && num_video_frames > 1)
+ h = block_processor->apply(*ctx, UNetBlockStage::Up, int(len_mults) - 1 - i, j, h, num_video_frames);
if (i > 0 && j == num_res_blocks) {
if (tiny_unet) {
@@ -769,6 +776,8 @@ struct UNetModelRunner : public DiffusionModelRunner {
return "unet";
}
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
unet.get_param_tensors(tensors, prefix);
}
@@ -782,8 +791,10 @@ struct UNetModelRunner : public DiffusionModelRunner {
const std::vector<sd::Tensor<float>>& controls_tensor = {},
float control_strength = 0.f,
const sd::Tensor<float>& ip_context_tensor = {},
- float ip_scale = 1.f) {
- ggml_cgraph* gf = new_graph_custom(UNET_GRAPH_SIZE);
+ float ip_scale = 1.f,
+ const AttentionProcessor* attn2_processor = nullptr,
+ const UNetBlockProcessor* block_processor = nullptr) {
+ ggml_cgraph* gf = new_graph_custom(UNET_GRAPH_SIZE + (block_processor ? block_processor->extra_graph_size() : 0));
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
@@ -801,9 +812,10 @@ struct UNetModelRunner : public DiffusionModelRunner {
num_video_frames = static_cast<int>(x->ne[3]);
}
- auto runner_ctx = get_context();
- runner_ctx.ip_context = ip_context;
- runner_ctx.ip_scale = ip_scale;
+ auto runner_ctx = get_context();
+ runner_ctx.ip_context = ip_context;
+ runner_ctx.ip_scale = ip_scale;
+ runner_ctx.attn2_processor = attn2_processor;
ggml_tensor* out = unet.forward(&runner_ctx,
x,
@@ -813,7 +825,8 @@ struct UNetModelRunner : public DiffusionModelRunner {
y,
num_video_frames,
controls,
- control_strength);
+ control_strength,
+ block_processor);
ggml_build_forward_expand(gf, out);
@@ -830,14 +843,17 @@ struct UNetModelRunner : public DiffusionModelRunner {
const std::vector<sd::Tensor<float>>& controls = {},
float control_strength = 0.f,
const sd::Tensor<float>& ip_context = {},
- float ip_scale = 1.f) {
+ float ip_scale = 1.f,
+ const AttentionProcessor* attn2_processor = nullptr,
+ const UNetBlockProcessor* block_processor = nullptr) {
// x: [N, in_channels, h, w]
// timesteps: [N, ]
// context: [N, max_position, hidden_size]([N, 77, 768]) or [1, max_position, hidden_size]
// c_concat: [N, in_channels, h, w] or [1, in_channels, h, w]
// y: [N, adm_in_channels] or [1, adm_in_channels]
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength, ip_context, ip_scale);
+ return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls,
+ control_strength, ip_context, ip_scale, attn2_processor, block_processor);
};
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
@@ -859,7 +875,9 @@ struct UNetModelRunner : public DiffusionModelRunner {
extra->controls ? *extra->controls : empty_controls,
extra->control_strength,
extra->ip_context ? *extra->ip_context : sd::Tensor<float>{},
- extra->ip_scale);
+ extra->ip_scale,
+ extra->attn2_processor,
+ extra->block_processor);
}
void test() {
diff --git a/src/model/diffusion/z_image.hpp b/src/model/diffusion/z_image.hpp
index 43df4d3..de63c5d 100644
--- a/src/model/diffusion/z_image.hpp
+++ b/src/model/diffusion/z_image.hpp
@@ -542,24 +542,203 @@ namespace ZImage {
}
};
+ class ControlNet : public GGMLBlock {
+ ZImageConfig config_;
+ int num_layers = 0;
+ int num_refiner_layers = 0;
+
+ public:
+ static bool has_weights(const String2TensorStorage& weights) {
+ return weights.count("control_all_x_embedder.2-1.weight") != 0;
+ }
+
+ static ZImageConfig detect_config(const String2TensorStorage& weights) {
+ ZImageConfig config;
+ const auto projection = weights.find("control_all_x_embedder.2-1.weight");
+ const auto q = weights.find("control_layers.0.attention.to_q.weight");
+ const auto k = weights.find("control_layers.0.attention.to_k.weight");
+ const auto norm = weights.find("control_layers.0.attention.norm_q.weight");
+ if (projection == weights.end() || q == weights.end() || k == weights.end() ||
+ projection->second.logical_ne0() != config.patch_size * config.patch_size * CONTROL_IN_CHANNELS)
+ throw std::runtime_error("invalid Z-Image ControlNet weights");
+ config.hidden_size = projection->second.ne[1];
+ config.qk_norm = norm != weights.end();
+ if (config.qk_norm)
+ config.head_dim = norm->second.ne[0];
+ if (config.head_dim <= 0 || config.hidden_size <= 0 ||
+ q->second.ne[1] % config.head_dim || k->second.ne[1] % config.head_dim ||
+ q->second.ne[1] == 0 || k->second.ne[1] == 0)
+ throw std::runtime_error("invalid Z-Image ControlNet attention dimensions");
+ config.num_heads = q->second.ne[1] / config.head_dim;
+ config.num_kv_heads = k->second.ne[1] / config.head_dim;
+ return config;
+ }
+
+ ControlNet(const ZImageConfig& config, const String2TensorStorage& tensor_storage_map)
+ : config_(config) {
+ for (const auto& pair : tensor_storage_map) {
+ if (starts_with(pair.first, "control_layers.")) {
+ auto items = split_string(pair.first, '.');
+ if (items.size() > 1) {
+ num_layers = std::max(num_layers, atoi(items[1].c_str()) + 1);
+ }
+ } else if (starts_with(pair.first, "control_noise_refiner.")) {
+ auto items = split_string(pair.first, '.');
+ if (items.size() > 1) {
+ num_refiner_layers = std::max(num_refiner_layers, atoi(items[1].c_str()) + 1);
+ }
+ }
+ }
+
+ if (num_layers <= 0 || num_refiner_layers <= 0)
+ throw std::runtime_error("Z-Image ControlNet requires main and noise refiner blocks");
+
+ blocks["control_all_x_embedder.2-1"] = std::make_shared<Linear>(config.patch_size * config.patch_size * CONTROL_IN_CHANNELS, config.hidden_size);
+
+ for (int i = 0; i < num_refiner_layers; i++) {
+ std::string name = "control_noise_refiner." + std::to_string(i);
+ auto block = std::make_shared<ZImageControlTransformerBlock>(i,
+ config.hidden_size,
+ config.head_dim,
+ config.num_heads,
+ config.num_kv_heads,
+ config.multiple_of,
+ config.ffn_dim_multiplier,
+ config.norm_eps,
+ config.qk_norm,
+ tensor_storage_map.find(name + ".before_proj.weight") != tensor_storage_map.end());
+
+ blocks[name] = block;
+ }
+
+ for (int i = 0; i < num_layers; i++) {
+ std::string name = "control_layers." + std::to_string(i);
+ auto block = std::make_shared<ZImageControlTransformerBlock>(i,
+ config.hidden_size,
+ config.head_dim,
+ config.num_heads,
+ config.num_kv_heads,
+ config.multiple_of,
+ config.ffn_dim_multiplier,
+ config.norm_eps,
+ config.qk_norm,
+ tensor_storage_map.find(name + ".before_proj.weight") != tensor_storage_map.end());
+
+ blocks[name] = block;
+ }
+ }
+
+ bool compatible(const ZImageConfig& config) const {
+ return config_.patch_size == config.patch_size && config_.hidden_size == config.hidden_size &&
+ config_.head_dim == config.head_dim && config_.num_heads == config.num_heads &&
+ config_.num_kv_heads == config.num_kv_heads && num_refiner_layers == config.num_refiner_layers &&
+ num_layers <= (config.num_layers + 1) / 2;
+ }
+
+ ggml_tensor* forward_refiner(GGMLRunnerContext* ctx,
+ ggml_tensor* control,
+ ggml_tensor* img,
+ ggml_tensor* img_pe,
+ ggml_tensor* img_pad_token,
+ ggml_tensor* t_emb,
+ float strength,
+ const std::string& group,
+ std::vector<ggml_tensor*>& ref_hints) const {
+ auto all_x_embedder = std::dynamic_pointer_cast<Linear>(blocks.at("control_all_x_embedder.2-1"));
+
+ auto c = all_x_embedder->forward(ctx, control);
+ sd::ggml_graph_cut::mark_graph_cut(c, group + ".prelude", "control");
+
+ int64_t N = c->ne[2];
+ int64_t n_token = c->ne[1];
+
+ int64_t n_pad_token = Rope::bound_mod(static_cast<int>(n_token), SEQ_MULTI_OF);
+ if (n_pad_token > 0) {
+ auto ctrl_pad_tokens = ggml_repeat_4d(ctx->ggml_ctx, img_pad_token, img_pad_token->ne[0], n_pad_token, N, 1);
+ c = ggml_concat(ctx->ggml_ctx, c, ctrl_pad_tokens, 1);
+ }
+
+ for (int i = 0; i < num_refiner_layers; i++) {
+ auto block = std::dynamic_pointer_cast<ZImageControlTransformerBlock>(blocks.at("control_noise_refiner." + std::to_string(i)));
+
+ auto out = block->forward(ctx, c, img_pe, img, t_emb);
+ c = out.second;
+ ref_hints.push_back(ggml_ext_scale(ctx, out.first, strength));
+ sd::ggml_graph_cut::mark_graph_cut(c, group + ".noise_refiner." + std::to_string(i), "control");
+ }
+
+ return c;
+ }
+
+ std::vector<ggml_tensor*> forward_main(GGMLRunnerContext* ctx,
+ ggml_tensor* control_ctx,
+ ggml_tensor* img_c,
+ ggml_tensor* txt,
+ ggml_tensor* pe,
+ ggml_tensor* t_emb,
+ float strength,
+ const std::string& group) const {
+ auto unified_c = ggml_concat(ctx->ggml_ctx, txt, img_c, 1);
+ auto ctrl_unified = ggml_concat(ctx->ggml_ctx, txt, control_ctx, 1);
+
+ std::vector<ggml_tensor*> main_hints;
+ main_hints.reserve(num_layers);
+
+ for (int i = 0; i < num_layers; i++) {
+ auto block = std::dynamic_pointer_cast<ZImageControlTransformerBlock>(blocks.at("control_layers." + std::to_string(i)));
+
+ auto out = block->forward(ctx, ctrl_unified, pe, unified_c, t_emb);
+ ctrl_unified = out.second;
+ main_hints.push_back(ggml_ext_scale(ctx, out.first, strength));
+ sd::ggml_graph_cut::mark_graph_cut(ctrl_unified, group + ".layers." + std::to_string(i), "control");
+ }
+
+ return main_hints;
+ }
+ };
+
+ struct ControlNetRunner : public GGMLRunner {
+ ControlNet model;
+
+ ControlNetRunner(ggml_backend_t backend, const String2TensorStorage& weights, const std::shared_ptr<RunnerWeightManager>& manager)
+ : GGMLRunner(backend, manager), model(ControlNet::detect_config(weights), weights) {
+ model.init(params_ctx, weights, "");
+ }
+
+ std::string get_desc() override { return "z_image_controlnet"; }
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model.get_param_tensors(tensors, ""); }
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+ size_t extra_graph_size() const { return weight_adapter ? weight_adapter->get_extra_graph_size() : 0; }
+ void configure_branch(GGMLRunnerContext& ctx) const {
+ ctx.weight_adapter = weight_adapter;
+ ctx.flash_attn_enabled = flash_attn_enabled;
+ ctx.linear_scale = linear_scale;
+ ctx.attn_scale = attn_scale;
+ }
+ };
+
+ struct GraphControl {
+ const ControlNet* model;
+ ggml_tensor* hint;
+ float strength;
+ const ControlNetRunner* runner = nullptr;
+ };
+
class ZImageModel : public GGMLBlock {
protected:
ZImageConfig config;
- GGMLBlockMap control_blocks;
- int control_num_layers = 0;
- int control_num_refiner_layers = 0;
+ std::shared_ptr<ControlNet> embedded_control_;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
params["cap_pad_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, config.hidden_size);
params["x_pad_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, config.hidden_size);
- for (auto& [name, block] : control_blocks) {
- block->init(ctx, tensor_storage_map, name);
- }
+ if (embedded_control_)
+ embedded_control_->init(ctx, tensor_storage_map, "");
}
public:
- bool control_enabled = false;
+ bool control_enabled() const { return bool(embedded_control_); }
ZImageModel() = default;
ZImageModel(ZImageConfig config, const String2TensorStorage& tensor_storage_map = {})
@@ -616,107 +795,21 @@ namespace ZImage {
blocks["final_layer"] = std::make_shared<FinalLayer>(config.hidden_size, config.patch_size, config.out_channels);
- if (tensor_storage_map.find("control_all_x_embedder.2-1.weight") != tensor_storage_map.end()) {
- control_enabled = true;
-
- for (const auto& pair : tensor_storage_map) {
- if (starts_with(pair.first, "control_layers.")) {
- auto items = split_string(pair.first, '.');
- if (items.size() > 1) {
- control_num_layers = std::max(control_num_layers, atoi(items[1].c_str()) + 1);
- }
- } else if (starts_with(pair.first, "control_noise_refiner.")) {
- auto items = split_string(pair.first, '.');
- if (items.size() > 1) {
- control_num_refiner_layers = std::max(control_num_refiner_layers, atoi(items[1].c_str()) + 1);
- }
- }
- }
-
- GGML_ASSERT(control_num_layers > 0);
- GGML_ASSERT(control_num_refiner_layers == config.num_refiner_layers);
-
- control_blocks["control_all_x_embedder.2-1"] = std::make_shared<Linear>(config.patch_size * config.patch_size * CONTROL_IN_CHANNELS, config.hidden_size);
-
- for (int i = 0; i < control_num_refiner_layers; i++) {
- std::string name = "control_noise_refiner." + std::to_string(i);
- auto block = std::make_shared<ZImageControlTransformerBlock>(i,
- config.hidden_size,
- config.head_dim,
- config.num_heads,
- config.num_kv_heads,
- config.multiple_of,
- config.ffn_dim_multiplier,
- config.norm_eps,
- config.qk_norm,
- tensor_storage_map.find(name + ".before_proj.weight") != tensor_storage_map.end());
-
- control_blocks[name] = block;
- }
-
- for (int i = 0; i < control_num_layers; i++) {
- std::string name = "control_layers." + std::to_string(i);
- auto block = std::make_shared<ZImageControlTransformerBlock>(i,
- config.hidden_size,
- config.head_dim,
- config.num_heads,
- config.num_kv_heads,
- config.multiple_of,
- config.ffn_dim_multiplier,
- config.norm_eps,
- config.qk_norm,
- tensor_storage_map.find(name + ".before_proj.weight") != tensor_storage_map.end());
-
- control_blocks[name] = block;
- }
- }
+ if (ControlNet::has_weights(tensor_storage_map))
+ embedded_control_ = std::make_shared<ControlNet>(config, tensor_storage_map);
}
void get_control_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
- for (auto& [name, block] : control_blocks) {
- block->get_param_tensors(tensors, name);
- }
- }
-
- ggml_tensor* forward_control_refiner(GGMLRunnerContext* ctx,
- ggml_tensor* control,
- ggml_tensor* img,
- ggml_tensor* img_pe,
- ggml_tensor* t_emb,
- float strength,
- std::vector<ggml_tensor*>& ref_hints) {
- auto all_x_embedder = std::dynamic_pointer_cast<Linear>(control_blocks["control_all_x_embedder.2-1"]);
-
- auto c = all_x_embedder->forward(ctx, control);
- sd::ggml_graph_cut::mark_graph_cut(c, "z_image.control.prelude", "control");
-
- int64_t N = c->ne[2];
- int64_t n_token = c->ne[1];
-
- int64_t n_pad_token = Rope::bound_mod(static_cast<int>(n_token), SEQ_MULTI_OF);
- if (n_pad_token > 0) {
- auto img_pad_token = params["x_pad_token"];
- auto ctrl_pad_tokens = ggml_repeat_4d(ctx->ggml_ctx, img_pad_token, img_pad_token->ne[0], n_pad_token, N, 1);
- c = ggml_concat(ctx->ggml_ctx, c, ctrl_pad_tokens, 1);
- }
-
- for (int i = 0; i < control_num_refiner_layers; i++) {
- auto block = std::dynamic_pointer_cast<ZImageControlTransformerBlock>(control_blocks["control_noise_refiner." + std::to_string(i)]);
-
- auto out = block->forward(ctx, c, img_pe, img, t_emb);
- c = out.second;
- ref_hints.push_back(ggml_ext_scale(ctx, out.first, strength));
- sd::ggml_graph_cut::mark_graph_cut(c, "z_image.control.noise_refiner." + std::to_string(i), "control");
- }
-
- return c;
+ if (embedded_control_)
+ embedded_control_->get_param_tensors(tensors, "");
}
ggml_tensor* forward_noise_refiner_hinted(GGMLRunnerContext* ctx,
ggml_tensor* img,
ggml_tensor* img_pe,
ggml_tensor* t_emb,
- const std::vector<ggml_tensor*>& ref_hints) {
+ const std::vector<ggml_tensor*>& ref_hints,
+ const std::string& group) {
GGML_ASSERT(ref_hints.size() == static_cast<size_t>(config.num_refiner_layers));
auto c = img;
@@ -725,44 +818,18 @@ namespace ZImage {
c = block->forward(ctx, c, img_pe, nullptr, t_emb);
c = ggml_add(ctx->ggml_ctx, c, ref_hints[i]);
- sd::ggml_graph_cut::mark_graph_cut(c, "z_image.noise_refiner_hinted." + std::to_string(i), "img");
+ sd::ggml_graph_cut::mark_graph_cut(c, group + ".noise_refiner_hinted." + std::to_string(i), "img");
}
return c;
}
- std::vector<ggml_tensor*> forward_control_main(GGMLRunnerContext* ctx,
- ggml_tensor* control_ctx,
- ggml_tensor* img_c,
- ggml_tensor* txt,
- ggml_tensor* pe,
- ggml_tensor* t_emb,
- float strength) {
- auto unified_c = ggml_concat(ctx->ggml_ctx, txt, img_c, 1);
- auto ctrl_unified = ggml_concat(ctx->ggml_ctx, txt, control_ctx, 1);
-
- std::vector<ggml_tensor*> main_hints;
- main_hints.reserve(control_num_layers);
-
- for (int i = 0; i < control_num_layers; i++) {
- auto block = std::dynamic_pointer_cast<ZImageControlTransformerBlock>(control_blocks["control_layers." + std::to_string(i)]);
-
- auto out = block->forward(ctx, ctrl_unified, pe, unified_c, t_emb);
- ctrl_unified = out.second;
- main_hints.push_back(ggml_ext_scale(ctx, out.first, strength));
- sd::ggml_graph_cut::mark_graph_cut(ctrl_unified, "z_image.control.layers." + std::to_string(i), "control");
- }
-
- return main_hints;
- }
-
ggml_tensor* forward_core(GGMLRunnerContext* ctx,
ggml_tensor* x,
ggml_tensor* timestep,
ggml_tensor* context,
ggml_tensor* pe,
- ggml_tensor* control = nullptr,
- float strength = 0.f) {
+ const std::vector<GraphControl>& controls) {
auto x_embedder = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"]);
auto t_embedder = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"]);
auto cap_embedder_0 = std::dynamic_pointer_cast<RMSNorm>(blocks["cap_embedder.0"]);
@@ -802,10 +869,21 @@ namespace ZImage {
auto txt_pe = ggml_ext_slice(ctx, pe, 3, 0, txt->ne[1]);
auto img_pe = ggml_ext_slice(ctx, pe, 3, txt->ne[1], pe->ne[3]);
- std::vector<ggml_tensor*> ref_hints;
- ggml_tensor* control_ctx = nullptr;
- if (control != nullptr) {
- control_ctx = forward_control_refiner(ctx, control, img, img_pe, t_emb, strength, ref_hints);
+ struct ControlFeatures {
+ ggml_tensor* context;
+ std::vector<ggml_tensor*> refiner_hints;
+ std::string group;
+ };
+ std::vector<ControlFeatures> features;
+ features.reserve(controls.size());
+ for (const auto& control : controls) {
+ auto& feature = features.emplace_back();
+ feature.group = controls.size() == 1 ? "z_image.control" : "z_image.control." + std::to_string(features.size() - 1);
+ auto control_ctx = *ctx;
+ if (control.runner)
+ control.runner->configure_branch(control_ctx);
+ feature.context = control.model->forward_refiner(&control_ctx, control.hint, img, img_pe, img_pad_token,
+ t_emb, control.strength, feature.group, feature.refiner_hints);
}
ggml_tensor* img_pre_refiner = img;
@@ -824,19 +902,24 @@ namespace ZImage {
sd::ggml_graph_cut::mark_graph_cut(img, "z_image.noise_refiner." + std::to_string(i), "img");
}
- ggml_tensor* img_c = nullptr;
- if (control_ctx != nullptr) {
- img_c = forward_noise_refiner_hinted(ctx, img_pre_refiner, img_pe, t_emb, ref_hints);
- }
-
auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1); // [N, n_txt_token + n_txt_pad_token + n_img_token + n_img_pad_token, hidden_size]
sd::ggml_graph_cut::mark_graph_cut(txt_img, "z_image.prelude", "txt_img");
std::vector<ggml_tensor*> main_hints;
- if (control_ctx != nullptr) {
- main_hints = forward_control_main(ctx, control_ctx, img_c, txt, pe, t_emb, strength);
+ for (size_t i = 0; i < controls.size(); ++i) {
+ const auto& control = controls[i];
+ const auto& feature = features[i];
+ auto* img_c = forward_noise_refiner_hinted(ctx, img_pre_refiner, img_pe, t_emb,
+ feature.refiner_hints, feature.group);
+ auto control_ctx = *ctx;
+ if (control.runner)
+ control.runner->configure_branch(control_ctx);
+ auto hints = control.model->forward_main(&control_ctx, feature.context, img_c, txt, pe, t_emb,
+ control.strength, feature.group);
+ main_hints.resize(std::max(main_hints.size(), hints.size()), nullptr);
+ for (size_t j = 0; j < hints.size(); ++j)
+ main_hints[j] = main_hints[j] ? ggml_add(ctx->ggml_ctx, main_hints[j], hints[j]) : hints[j];
}
- GGML_ASSERT(main_hints.empty() || main_hints.size() == static_cast<size_t>(control_num_layers));
for (int i = 0; i < config.num_layers; i++) {
auto block = std::dynamic_pointer_cast<JointTransformerBlock>(blocks["layers." + std::to_string(i)]);
@@ -862,7 +945,8 @@ namespace ZImage {
ggml_tensor* pe,
std::vector<ggml_tensor*> ref_latents = {},
ggml_tensor* control_latent = nullptr,
- float control_strength = 0.f) {
+ float control_strength = 0.f,
+ std::vector<GraphControl> controls = {}) {
// Forward pass of DiT.
// x: [N, C, H, W]
// timestep: [N,]
@@ -887,16 +971,20 @@ namespace ZImage {
}
}
- ggml_tensor* control = nullptr;
if (control_latent != nullptr) {
- GGML_ASSERT(control_enabled);
- GGML_ASSERT(control_latent->ne[2] == CONTROL_IN_CHANNELS);
- GGML_ASSERT(control_latent->ne[0] == W && control_latent->ne[1] == H && control_latent->ne[3] == N);
- GGML_ASSERT(ref_latents.empty());
- control = DiT::pad_and_patchify(ctx, control_latent, patch_size, patch_size, false);
+ GGML_ASSERT(embedded_control_ != nullptr);
+ controls.push_back({embedded_control_.get(), control_latent, control_strength});
+ }
+ for (auto& control : controls) {
+ if (!control.model->compatible(config) || control.hint->ne[2] != CONTROL_IN_CHANNELS ||
+ control.hint->ne[0] != W || control.hint->ne[1] != H || control.hint->ne[3] != N)
+ throw std::runtime_error("Z-Image ControlNet is incompatible with the model or latent shape");
+ if (!ref_latents.empty())
+ throw std::runtime_error("Z-Image ControlNet cannot be combined with reference latents");
+ control.hint = DiT::pad_and_patchify(ctx, control.hint, patch_size, patch_size, false);
}
- auto out = forward_core(ctx, img, timestep, context, pe, control, control_strength);
+ auto out = forward_core(ctx, img, timestep, context, pe, controls);
out = ggml_ext_slice(ctx, out, 1, 0, n_img_token); // [N, n_img_token, ph*pw*C]
out = DiT::unpatchify_and_crop(ctx, out, H, W, patch_size, patch_size, false); // [N, C, H, W]
@@ -936,9 +1024,11 @@ namespace ZImage {
}
bool control_enabled() const {
- return z_image.control_enabled;
+ return z_image.control_enabled();
}
+ ggml_backend_t compute_backend() const { return runtime_backend; }
+
void get_control_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
z_image.get_control_param_tensors(tensors);
}
@@ -949,8 +1039,15 @@ namespace ZImage {
const std::vector<sd::Tensor<float>>& ref_latents_tensor = {},
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
const sd::Tensor<float>& control_latent_tensor = {},
- float control_strength = 0.f) {
- ggml_cgraph* gf = new_graph_custom(Z_IMAGE_GRAPH_SIZE);
+ float control_strength = 0.f,
+ const std::vector<ZImageControlInput>* external_controls = nullptr) {
+ size_t graph_size = Z_IMAGE_GRAPH_SIZE;
+ if (external_controls) {
+ graph_size += external_controls->size() * size_t(Z_IMAGE_GRAPH_SIZE);
+ for (const auto& control : *external_controls)
+ graph_size += control.runner->extra_graph_size();
+ }
+ ggml_cgraph* gf = new_graph_custom(graph_size);
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
GGML_ASSERT(x->ne[3] == 1);
@@ -963,6 +1060,12 @@ namespace ZImage {
}
ggml_tensor* control_latent = make_optional_input(control_latent_tensor);
+ std::vector<GraphControl> controls;
+ if (external_controls) {
+ controls.reserve(external_controls->size());
+ for (const auto& control : *external_controls)
+ controls.push_back({&control.runner->model, make_input(*control.latent), control.strength, control.runner});
+ }
sd::PECacheKey pe_key("z_image.pe");
pe_key.add_shape(x).add_shape(context).add(config.theta).add(config.axes_dim).add(config.axes_dim_sum);
@@ -991,7 +1094,8 @@ namespace ZImage {
pe,
ref_latents,
control_latent,
- control_strength);
+ control_strength,
+ std::move(controls));
ggml_build_forward_expand(gf, out);
@@ -1005,12 +1109,13 @@ namespace ZImage {
const std::vector<sd::Tensor<float>>& ref_latents = {},
Rope::RefIndexMode ref_index_mode = Rope::RefIndexMode::FIXED,
const sd::Tensor<float>& control_latent = {},
- float control_strength = 0.f) {
+ float control_strength = 0.f,
+ const std::vector<ZImageControlInput>* controls = nullptr) {
// x: [N, in_channels, h, w]
// timesteps: [N, ]
// context: [N, max_position, hidden_size]
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(x, timesteps, context, ref_latents, ref_index_mode, control_latent, control_strength);
+ return build_graph(x, timesteps, context, ref_latents, ref_index_mode, control_latent, control_strength, controls);
};
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
@@ -1032,10 +1137,11 @@ namespace ZImage {
*diffusion_params.x,
*diffusion_params.timesteps,
tensor_or_empty(diffusion_params.context),
- diffusion_params.ref_latents && diffusion_params.ref_image_params.pass_to_dit ? *diffusion_params.ref_latents : empty_ref_latents,
- diffusion_params.ref_image_params.ref_index_mode,
+ diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+ diffusion_params.ref_index_mode,
control_latent,
- control_strength);
+ control_strength,
+ z_image_extra ? z_image_extra->controls : nullptr);
}
void test() {
diff --git a/src/model/integrations/anima.cpp b/src/model/integrations/anima.cpp
index 2f9b2ba..6227c9c 100644
--- a/src/model/integrations/anima.cpp
+++ b/src/model/integrations/anima.cpp
@@ -1,21 +1,25 @@
#include "model/diffusion/anima.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_anima(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -26,8 +30,8 @@ const std::vector<ModelDefinition>& anima_definitions() {
ModelDefinition definition;
definition.version = VERSION_ANIMA;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "cosmos_reference";
definition.name = "Anima";
+ definition.id = "anima";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
diff --git a/src/model/integrations/boogu_image.cpp b/src/model/integrations/boogu_image.cpp
index 3d6e8c0..e8c6e68 100644
--- a/src/model/integrations/boogu_image.cpp
+++ b/src/model/integrations/boogu_image.cpp
@@ -1,25 +1,29 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/boogu.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_boogu_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- true,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ true,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager);
+ }
return true;
}
@@ -30,8 +34,8 @@ const std::vector<ModelDefinition>& boogu_image_definitions() {
ModelDefinition definition;
definition.version = VERSION_BOOGU_IMAGE;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "z_image_omni";
definition.name = "Boogu Image";
+ definition.id = "boogu_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLOW_PRED, 3.16f};
diff --git a/src/model/integrations/ernie_image.cpp b/src/model/integrations/ernie_image.cpp
index f1d185c..9375b29 100644
--- a/src/model/integrations/ernie_image.cpp
+++ b/src/model/integrations/ernie_image.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/ernie_image.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_ernie_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -30,6 +34,7 @@ const std::vector<ModelDefinition>& ernie_image_definitions() {
definition.version = VERSION_ERNIE_IMAGE;
definition.geometry = {1, 128, 3};
definition.name = "Ernie Image";
+ definition.id = "ernie_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLOW_PRED, 4.f};
diff --git a/src/model/integrations/fibo2.cpp b/src/model/integrations/fibo2.cpp
index d16a0cf..82ef70f 100644
--- a/src/model/integrations/fibo2.cpp
+++ b/src/model/integrations/fibo2.cpp
@@ -1,18 +1,22 @@
#include "model/diffusion/fibo2.hpp"
#include "conditioning/conditioner.hpp"
#include "conditioning/fibo2.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_fibo2(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<Fibo2Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map, weight_manager, tokenizers);
- result.diffusion = std::make_shared<Fibo2::Fibo2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map, "model.diffusion_model", weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<Fibo2Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map, weight_manager, tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Fibo2::Fibo2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map, "model.diffusion_model", weight_manager);
+ }
return true;
}
@@ -23,8 +27,8 @@ const std::vector<ModelDefinition>& fibo2_definitions() {
ModelDefinition definition;
definition.version = VERSION_FIBO2;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "fibo2";
definition.name = "Fibo-2.0";
+ definition.id = "fibo2";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLOW_PRED, 3.f};
diff --git a/src/model/integrations/flux.cpp b/src/model/integrations/flux.cpp
index fc96da7..2a84205 100644
--- a/src/model/integrations/flux.cpp
+++ b/src/model/integrations/flux.cpp
@@ -1,8 +1,8 @@
#include "model/diffusion/flux.hpp"
#include "conditioning/conditioner.hpp"
#include "model/integrations/flux.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_flux(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
@@ -10,33 +10,41 @@ static bool build_flux(const sd::model_builders::Context& ctx, const TokenizerCo
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
if (version == VERSION_CHROMA || version == VERSION_CHROMA_RADIANCE) {
- result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- false,
- 1,
- false,
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ false,
+ 1,
+ false,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
} else if (version == VERSION_OVIS_IMAGE) {
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
} else {
- result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- weight_manager,
- tokenizers);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ weight_manager,
+ tokenizers);
+ }
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
}
- result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager,
- sd_ctx_params->model_args);
return true;
}
@@ -47,8 +55,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Flux";
+ definition.id = "flux";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLUX_FLOW_PRED, 1.0f};
@@ -62,8 +70,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX_FILL;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Flux Fill";
+ definition.id = "flux_fill";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.traits.is_inpaint = true;
@@ -78,8 +86,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX_CONTROLS;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Flux Control";
+ definition.id = "flux_control";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.traits.is_control = true;
@@ -94,8 +102,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLEX_2;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Flex.2";
+ definition.id = "flex2";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.traits.is_inpaint = true;
@@ -111,8 +119,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_CHROMA_RADIANCE;
definition.geometry = {1, 3, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Chroma Radiance";
+ definition.id = "chroma_radiance";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.traits.fake_vae = true;
@@ -127,8 +135,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_CHROMA;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Chroma";
+ definition.id = "chroma";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLUX_FLOW_PRED, 1.0f};
@@ -142,8 +150,8 @@ const std::vector<ModelDefinition>& flux_definitions() {
ModelDefinition definition;
definition.version = VERSION_OVIS_IMAGE;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "flux_kontext";
definition.name = "Ovis Image";
+ definition.id = "ovis_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLUX_FLOW_PRED, 1.0f};
diff --git a/src/model/integrations/flux2.cpp b/src/model/integrations/flux2.cpp
index d895df2..02a060a 100644
--- a/src/model/integrations/flux2.cpp
+++ b/src/model/integrations/flux2.cpp
@@ -1,8 +1,8 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/flux.hpp"
#include "model/integrations/flux.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_flux2(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
@@ -10,19 +10,23 @@ static bool build_flux2(const sd::model_builders::Context& ctx, const TokenizerC
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
bool is_chroma = false;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
return true;
}
@@ -33,8 +37,8 @@ const std::vector<ModelDefinition>& flux2_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX2;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "flux2";
definition.name = "Flux.2";
+ definition.id = "flux2";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLUX_FLOW_PRED, 1.0f};
@@ -48,8 +52,8 @@ const std::vector<ModelDefinition>& flux2_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX2_KLEIN;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "flux2";
definition.name = "Flux.2 klein";
+ definition.id = "flux2_klein";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLUX_FLOW_PRED, 1.0f};
@@ -63,8 +67,8 @@ const std::vector<ModelDefinition>& flux2_definitions() {
ModelDefinition definition;
definition.version = VERSION_SEFI_IMAGE;
definition.geometry = {1, 144, 3};
- definition.ref_image_preset = "flux2";
definition.name = "SeFi-Image";
+ definition.id = "sefi_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {SEFI_FLOW_PRED, INFINITY};
diff --git a/src/model/integrations/flux3.cpp b/src/model/integrations/flux3.cpp
index 9ee8d3d..73db452 100644
--- a/src/model/integrations/flux3.cpp
+++ b/src/model/integrations/flux3.cpp
@@ -1,17 +1,21 @@
#include "model/diffusion/flux3.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_flux3(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map, version, "", false, weight_manager, tokenizers);
- result.diffusion = std::make_shared<Flux3::Flux3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map, "model.diffusion_model", weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map, version, "", false, weight_manager, tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Flux3::Flux3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map, "model.diffusion_model", weight_manager);
+ }
return true;
}
@@ -22,8 +26,8 @@ const std::vector<ModelDefinition>& flux3_definitions() {
ModelDefinition definition;
definition.version = VERSION_FLUX3;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "flux2";
definition.name = "Flux.3 Image Klein";
+ definition.id = "flux3_klein";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLUX3_FLOW_PRED, 6.93f};
diff --git a/src/model/integrations/hidream_o1.cpp b/src/model/integrations/hidream_o1.cpp
index 66f6a9f..ca56e02 100644
--- a/src/model/integrations/hidream_o1.cpp
+++ b/src/model/integrations/hidream_o1.cpp
@@ -1,21 +1,25 @@
#include "model/diffusion/hidream_o1.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_hidream_o1(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model",
+ weight_manager);
+ }
return true;
}
@@ -27,6 +31,7 @@ const std::vector<ModelDefinition>& hidream_o1_definitions() {
definition.version = VERSION_HIDREAM_O1;
definition.geometry = {1, 3, 3};
definition.name = "HiDream O1";
+ definition.id = "hidream_o1";
definition.traits.is_dit = true;
definition.traits.fake_vae = true;
definition.sampling = {FLOW_PRED, 3.f};
diff --git a/src/model/integrations/hunyuan_video.cpp b/src/model/integrations/hunyuan_video.cpp
index 85bb741..321a554 100644
--- a/src/model/integrations/hunyuan_video.cpp
+++ b/src/model/integrations/hunyuan_video.cpp
@@ -1,25 +1,29 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/hunyuan.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_hunyuan_video(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager);
+ }
return true;
}
@@ -31,6 +35,7 @@ const std::vector<ModelDefinition>& hunyuan_video_definitions() {
definition.version = VERSION_HUNYUAN_VIDEO;
definition.geometry = {1, 32, 3};
definition.name = "Hunyuan Video";
+ definition.id = "hunyuan_video";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::HUNYUAN;
diff --git a/src/model/integrations/ideogram4.cpp b/src/model/integrations/ideogram4.cpp
index 77baef2..0a63b81 100644
--- a/src/model/integrations/ideogram4.cpp
+++ b/src/model/integrations/ideogram4.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/ideogram4.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_ideogram4(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -30,6 +34,7 @@ const std::vector<ModelDefinition>& ideogram4_definitions() {
definition.version = VERSION_IDEOGRAM4;
definition.geometry = {1, 128, 3};
definition.name = "Ideogram 4";
+ definition.id = "ideogram4";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLOW_PRED, 1.0f};
diff --git a/src/model/integrations/iris.cpp b/src/model/integrations/iris.cpp
index a75c88c..ed638a2 100644
--- a/src/model/integrations/iris.cpp
+++ b/src/model/integrations/iris.cpp
@@ -1,17 +1,21 @@
#include "model/diffusion/iris.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_iris(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map, version, "", false, weight_manager, tokenizers);
- result.diffusion = std::make_shared<Iris::IrisRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map, "model.diffusion_model", weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map, version, "", false, weight_manager, tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Iris::IrisRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map, "model.diffusion_model", weight_manager);
+ }
return true;
}
@@ -23,6 +27,7 @@ const std::vector<ModelDefinition>& iris_definitions() {
definition.version = VERSION_IRIS;
definition.geometry = {16, 3, 3};
definition.name = "Iris-3B";
+ definition.id = "iris_3b";
definition.traits.is_dit = true;
definition.traits.fake_vae = true;
definition.sampling = {FLOW_PRED, 4.f};
diff --git a/src/model/integrations/krea2.cpp b/src/model/integrations/krea2.cpp
index 3670445..8560f97 100644
--- a/src/model/integrations/krea2.cpp
+++ b/src/model/integrations/krea2.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/krea2.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_krea2(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- true,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ true,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -29,8 +33,8 @@ const std::vector<ModelDefinition>& krea2_definitions() {
ModelDefinition definition;
definition.version = VERSION_KREA2;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "krea2_ostris_edit";
definition.name = "Krea2";
+ definition.id = "krea2";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
diff --git a/src/model/integrations/lens.cpp b/src/model/integrations/lens.cpp
index da66d2c..5df9d52 100644
--- a/src/model/integrations/lens.cpp
+++ b/src/model/integrations/lens.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/lens.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_lens(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -30,6 +34,7 @@ const std::vector<ModelDefinition>& lens_definitions() {
definition.version = VERSION_LENS;
definition.geometry = {1, 128, 3};
definition.name = "Lens";
+ definition.id = "lens";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLUX_FLOW_PRED, 1.83f};
diff --git a/src/model/integrations/lingbot_video.cpp b/src/model/integrations/lingbot_video.cpp
index d4cc6cc..e0319cc 100644
--- a/src/model/integrations/lingbot_video.cpp
+++ b/src/model/integrations/lingbot_video.cpp
@@ -1,7 +1,7 @@
#include "model/diffusion/lingbot_video.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_lingbot_video(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
@@ -15,18 +15,22 @@ static bool build_lingbot_video(const sd::model_builders::Context& ctx, const To
break;
}
}
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- enable_vision,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ enable_vision,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
return true;
}
@@ -38,6 +42,7 @@ const std::vector<ModelDefinition>& lingbot_video_definitions() {
definition.version = VERSION_LINGBOT_VIDEO;
definition.geometry = {2, 16, 3};
definition.name = "LingBot Video";
+ definition.id = "lingbot_video";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
diff --git a/src/model/integrations/llada_image.cpp b/src/model/integrations/llada_image.cpp
index 58c47e8..c51ba5e 100644
--- a/src/model/integrations/llada_image.cpp
+++ b/src/model/integrations/llada_image.cpp
@@ -1,25 +1,29 @@
#include "model/diffusion/llada_image.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_llada_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLaDAImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLaDAImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ "text_encoders.llm",
+ "text_encoders.connector.queryformer",
+ "text_encoders.connector.text_projection",
+ "text_encoders.connector.sigvq",
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<LLaDAImage::LLaDAImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
- "text_encoders.llm",
- "queryformer",
- "text_projection",
- "sigvq",
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<LLaDAImage::LLaDAImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -30,8 +34,8 @@ const std::vector<ModelDefinition>& llada_image_definitions() {
ModelDefinition definition;
definition.version = VERSION_LLADA_IMAGE;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "llada_image";
definition.name = "LLaDA-Image";
+ definition.id = "llada_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLOW_PRED, 1.0f};
diff --git a/src/model/integrations/longcat.cpp b/src/model/integrations/longcat.cpp
index d371710..d34fdc0 100644
--- a/src/model/integrations/longcat.cpp
+++ b/src/model/integrations/longcat.cpp
@@ -1,26 +1,30 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/flux.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_longcat(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- true,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ true,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
return true;
}
@@ -31,8 +35,8 @@ const std::vector<ModelDefinition>& longcat_definitions() {
ModelDefinition definition;
definition.version = VERSION_LONGCAT;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "longcat";
definition.name = "Longcat-Image";
+ definition.id = "longcat_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLUX_FLOW_PRED, 3.0f};
diff --git a/src/model/integrations/ltxav.cpp b/src/model/integrations/ltxav.cpp
index 9995239..6c315a0 100644
--- a/src/model/integrations/ltxav.cpp
+++ b/src/model/integrations/ltxav.cpp
@@ -1,23 +1,27 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/ltxv.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_ltxav(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LTXAVEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- "text_encoders.llm",
- "text_embedding_projection",
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LTXAVEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ "text_encoders.llm",
+ "text_encoders.connector.text_embedding_projection",
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -29,6 +33,7 @@ const std::vector<ModelDefinition>& ltxav_definitions() {
definition.version = VERSION_LTXAV;
definition.geometry = {1, 128, 3};
definition.name = "LTXAV";
+ definition.id = "ltxav";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::LTXAV;
diff --git a/src/model/integrations/mage_flow.cpp b/src/model/integrations/mage_flow.cpp
index 000fa15..32660e6 100644
--- a/src/model/integrations/mage_flow.cpp
+++ b/src/model/integrations/mage_flow.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/mage_flow.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_mage_flow(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- true,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ true,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -29,8 +33,8 @@ const std::vector<ModelDefinition>& mage_flow_definitions() {
ModelDefinition definition;
definition.version = VERSION_MAGE_FLOW;
definition.geometry = {1, 128, 3};
- definition.ref_image_preset = "mage_flow";
definition.name = "Mage Flow";
+ definition.id = "mage_flow";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::MAGE;
definition.sampling = {FLOW_PRED, 6.f};
diff --git a/src/model/integrations/ming_image.cpp b/src/model/integrations/ming_image.cpp
index 8587df8..f8fd25b 100644
--- a/src/model/integrations/ming_image.cpp
+++ b/src/model/integrations/ming_image.cpp
@@ -1,17 +1,21 @@
#include "model/diffusion/ming_image.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_ming_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<MingImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map, weight_manager, tokenizers);
- result.diffusion = std::make_shared<MingImage::MingImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map, "model.diffusion_model", weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<MingImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map, weight_manager, tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<MingImage::MingImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map, "model.diffusion_model", weight_manager);
+ }
return true;
}
@@ -23,6 +27,7 @@ const std::vector<ModelDefinition>& ming_image_definitions() {
definition.version = VERSION_MING_IMAGE;
definition.geometry = {2, 16, 4};
definition.name = "Ming-Image";
+ definition.id = "ming_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
diff --git a/src/model/integrations/minimax_h3.cpp b/src/model/integrations/minimax_h3.cpp
index c4a7ae8..e6ca906 100644
--- a/src/model/integrations/minimax_h3.cpp
+++ b/src/model/integrations/minimax_h3.cpp
@@ -1,25 +1,29 @@
#include "model/diffusion/minimax_h3.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_minimax_h3(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- true,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ true,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
return true;
}
@@ -31,6 +35,7 @@ const std::vector<ModelDefinition>& minimax_h3_definitions() {
definition.version = VERSION_MINIMAX_H3;
definition.geometry = {2, 24, 3};
definition.name = "MiniMax-H3";
+ definition.id = "minimax_h3";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::MINIMAX_H3;
@@ -45,18 +50,18 @@ const std::vector<ModelDefinition>& minimax_h3_definitions() {
ModelDefinition definition = result.front();
definition.version = VERSION_MINIMAX_H3_IMAGE;
definition.name = "MiniMax-H3-Image";
+ definition.id = "minimax_h3_image";
definition.traits.supports_video_generation = false;
definition.sampling.default_flow_shift = 1.f;
- definition.ref_image_preset = "qwen";
result.push_back(std::move(definition));
}
{
ModelDefinition definition = result.front();
definition.version = VERSION_P_H3_IMAGE;
definition.name = "P-H3-Image";
+ definition.id = "p_h3_image";
definition.traits.supports_video_generation = false;
definition.sampling.default_flow_shift = 1.f;
- definition.ref_image_preset = "qwen";
result.push_back(std::move(definition));
}
return result;
diff --git a/src/model/integrations/minit2i.cpp b/src/model/integrations/minit2i.cpp
index fb30594..5966aeb 100644
--- a/src/model/integrations/minit2i.cpp
+++ b/src/model/integrations/minit2i.cpp
@@ -1,20 +1,24 @@
#include "model/diffusion/minit2i.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_minit2i(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<MiniT2IConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- weight_manager);
- result.diffusion = std::make_shared<MiniT2I::MiniT2IRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model.model.net",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<MiniT2IConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ weight_manager);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<MiniT2I::MiniT2IRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model.model.net",
+ weight_manager);
+ }
return true;
}
@@ -26,6 +30,7 @@ const std::vector<ModelDefinition>& minit2i_definitions() {
definition.version = VERSION_MINIT2I;
definition.geometry = {1, 3, 3};
definition.name = "MiniT2I";
+ definition.id = "minit2i";
definition.traits.is_dit = true;
definition.traits.fake_vae = true;
definition.sampling = {MINIT2I_FLOW_PRED, INFINITY};
diff --git a/src/model/integrations/moss_image.cpp b/src/model/integrations/moss_image.cpp
index 6a2114d..8226426 100644
--- a/src/model/integrations/moss_image.cpp
+++ b/src/model/integrations/moss_image.cpp
@@ -2,16 +2,20 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/moss_image.hpp"
-#include "pipeline/model_builders.h"
+#include "model_build_context.h"
static bool build_moss_image(const sd::model_builders::Context& ctx,
const TokenizerConfig& tokenizers,
sd::model_builders::CoreRunners& runners) {
- runners.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- ctx.tensor_storage_map, ctx.version, "", true,
- ctx.weight_manager, tokenizers);
- runners.diffusion = std::make_shared<MOSSImage::MOSSImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- ctx.tensor_storage_map, "model.diffusion_model", ctx.weight_manager);
+ if (ctx.build_conditioner) {
+ runners.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ ctx.tensor_storage_map, ctx.version, "", true,
+ ctx.weight_manager, tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ runners.diffusion = std::make_shared<MOSSImage::MOSSImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ ctx.tensor_storage_map, "model.diffusion_model", ctx.weight_manager);
+ }
return true;
}
@@ -21,6 +25,7 @@ const std::vector<ModelDefinition>& moss_image_definitions() {
definition.version = VERSION_MOSS_IMAGE;
definition.geometry = {1, 128, 3};
definition.name = "MOSS-Image";
+ definition.id = "moss_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX2;
definition.sampling = {FLOW_PRED, 5.f};
diff --git a/src/model/integrations/pid.cpp b/src/model/integrations/pid.cpp
index d307fb6..5f0858d 100644
--- a/src/model/integrations/pid.cpp
+++ b/src/model/integrations/pid.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/pid.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_pid(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model.net",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model.net",
+ weight_manager);
+ }
return true;
}
@@ -30,6 +34,7 @@ const std::vector<ModelDefinition>& pid_definitions() {
definition.version = VERSION_PID;
definition.geometry = {1, 3, 3};
definition.name = "PiD";
+ definition.id = "pid";
definition.traits.is_dit = true;
definition.traits.vae_format_override = true;
definition.sampling = {FLOW_PRED, 1.5f};
diff --git a/src/model/integrations/pixart.cpp b/src/model/integrations/pixart.cpp
index d0a4aa7..2851ae7 100644
--- a/src/model/integrations/pixart.cpp
+++ b/src/model/integrations/pixart.cpp
@@ -1,8 +1,8 @@
#include "model/diffusion/pixart.hpp"
#include "conditioning/conditioner.hpp"
#include "model/vae/auto_encoder_kl.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static void configure_pixart_vae(const sd::model_builders::Context& ctx,
const sd::model_builders::VAEOptions&,
@@ -12,7 +12,7 @@ static void configure_pixart_vae(const sd::model_builders::Context& ctx,
if (ctx.tensor_storage_map.count("model.diffusion_model.csize_embedder.mlp.0.weight") != 0) {
model.scale_factor = 0.18215f;
}
- for (const auto& [key, value] : parse_key_value_args(ctx.params.model_args, "model arg")) {
+ for (const auto& [key, value] : parse_key_value_args(ctx.params.model_args.c_str(), "model arg")) {
if (key == "pixart_vae_scale_factor") {
float parsed = 0.f;
if (parse_strict_float(value, parsed) && std::isfinite(parsed) && parsed > 0.f) {
@@ -30,18 +30,22 @@ static bool build_pixart(const sd::model_builders::Context& ctx, const Tokenizer
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- true,
- 0,
- false,
- weight_manager,
- sd_ctx_params->model_args);
- result.diffusion = std::make_shared<PixArt::PixArtRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ true,
+ 0,
+ false,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<PixArt::PixArtRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
return true;
}
@@ -53,6 +57,7 @@ const std::vector<ModelDefinition>& pixart_definitions() {
definition.version = VERSION_PIXART;
definition.geometry = {2, 4, 3};
definition.name = "PixArt";
+ definition.id = "pixart";
definition.traits.is_dit = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
diff --git a/src/model/integrations/qwen_image.cpp b/src/model/integrations/qwen_image.cpp
index 49505bb..c45c948 100644
--- a/src/model/integrations/qwen_image.cpp
+++ b/src/model/integrations/qwen_image.cpp
@@ -1,8 +1,8 @@
#include "model/diffusion/qwen_image.hpp"
#include "conditioning/conditioner.hpp"
#include "model/diffusion/qwen_image_2_1.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_qwen_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
@@ -10,26 +10,32 @@ static bool build_qwen_image(const sd::model_builders::Context& ctx, const Token
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
bool enable_vision = version != VERSION_QWEN_IMAGE_LAYERED;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- enable_vision,
- weight_manager,
- tokenizers);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ enable_vision,
+ weight_manager,
+ tokenizers);
+ }
if (version == VERSION_QWEN_IMAGE_2_1) {
- result.diffusion = std::make_shared<Qwen::QwenImage21Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Qwen::QwenImage21Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
} else {
- result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager,
- sd_ctx_params->model_args);
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager,
+ sd_ctx_params->model_args.c_str());
+ }
}
return true;
}
@@ -41,8 +47,8 @@ const std::vector<ModelDefinition>& qwen_image_definitions() {
ModelDefinition definition;
definition.version = VERSION_QWEN_IMAGE;
definition.geometry = {1, 16, 3};
- definition.ref_image_preset = "qwen";
definition.name = "Qwen Image";
+ definition.id = "qwen_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
@@ -56,8 +62,8 @@ const std::vector<ModelDefinition>& qwen_image_definitions() {
ModelDefinition definition;
definition.version = VERSION_QWEN_IMAGE_LAYERED;
definition.geometry = {1, 16, 4};
- definition.ref_image_preset = "qwen_layered";
definition.name = "Qwen Image Layered";
+ definition.id = "qwen_image_layered";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
@@ -71,8 +77,8 @@ const std::vector<ModelDefinition>& qwen_image_definitions() {
ModelDefinition definition;
definition.version = VERSION_QWEN_IMAGE_2_1;
definition.geometry = {2, 64, 4};
- definition.ref_image_preset = "qwen";
definition.name = "Qwen Image 2.1";
+ definition.id = "qwen_image_2_1";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.sampling = {FLOW_PRED, 3.f};
diff --git a/src/model/integrations/sd.cpp b/src/model/integrations/sd.cpp
index 24a9acf..837e647 100644
--- a/src/model/integrations/sd.cpp
+++ b/src/model/integrations/sd.cpp
@@ -2,13 +2,13 @@
#include "core/option_types.h"
#include "model/diffusion/unet.hpp"
#include "model/vae/auto_encoder_kl.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static void configure_sdxl_vae(const sd::model_builders::Context& ctx,
const sd::model_builders::VAEOptions& options,
VAE& vae) {
- if (strlen(SAFE_STR(ctx.params.vae_path)) == 0 || ctx.params.force_sdxl_vae_conv_scale || options.external_vae_is_invalid) {
+ if (options.apply_default_conv_scale) {
float vae_conv_2d_scale = 1.f / 32.f;
LOG_WARN(
"No valid VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
@@ -24,21 +24,21 @@ static bool build_sd(const sd::model_builders::Context& ctx, const TokenizerConf
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
// SD1.x SD2.x SDXL
- std::map<std::string, std::string> embbeding_map;
- for (uint32_t i = 0; i < sd_ctx_params->embedding_count; i++) {
- embbeding_map.emplace(SAFE_STR(sd_ctx_params->embeddings[i].name), SAFE_STR(sd_ctx_params->embeddings[i].path));
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<FrozenCLIPEmbedderWithCustomWords>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ sd_ctx_params->embeddings,
+ version,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager);
}
- result.conditioner = std::make_shared<FrozenCLIPEmbedderWithCustomWords>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- embbeding_map,
- version,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager);
return true;
}
static prediction_t detect_sdxl_prediction(const String2TensorStorage& weights) {
@@ -55,6 +55,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD1;
definition.geometry = {8, 4, 3};
definition.name = "SD 1.x";
+ definition.id = "sd1";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -67,6 +68,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD1_INPAINT;
definition.geometry = {8, 4, 3};
definition.name = "SD 1.x Inpaint";
+ definition.id = "sd1_inpaint";
definition.traits.is_inpaint = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -79,6 +81,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD1_PIX2PIX;
definition.geometry = {8, 4, 3};
definition.name = "Instruct-Pix2Pix";
+ definition.id = "sd1_pix2pix";
definition.traits.is_unet_edit = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -91,6 +94,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD1_TINY_UNET;
definition.geometry = {8, 4, 3};
definition.name = "SD 1.x Tiny UNet";
+ definition.id = "sd1_tiny_unet";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -103,6 +107,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD2;
definition.geometry = {8, 4, 3};
definition.name = "SD 2.x";
+ definition.id = "sd2";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -116,6 +121,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD2_INPAINT;
definition.geometry = {8, 4, 3};
definition.name = "SD 2.x Inpaint";
+ definition.id = "sd2_inpaint";
definition.traits.is_inpaint = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -129,6 +135,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SD2_TINY_UNET;
definition.geometry = {8, 4, 3};
definition.name = "SD 2.x Tiny UNet";
+ definition.id = "sd2_tiny_unet";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -142,6 +149,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXS_512_DS;
definition.geometry = {8, 4, 3};
definition.name = "SDXS (512-DS)";
+ definition.id = "sdxs_512_ds";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -154,6 +162,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXS_09;
definition.geometry = {8, 4, 3};
definition.name = "SDXS (09)";
+ definition.id = "sdxs_09";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -167,6 +176,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXL;
definition.geometry = {8, 4, 3};
definition.name = "SDXL";
+ definition.id = "sdxl";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -181,6 +191,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXL_INPAINT;
definition.geometry = {8, 4, 3};
definition.name = "SDXL Inpaint";
+ definition.id = "sdxl_inpaint";
definition.traits.is_inpaint = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -195,6 +206,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXL_PIX2PIX;
definition.geometry = {8, 4, 3};
definition.name = "SDXL Instruct-Pix2Pix";
+ definition.id = "sdxl_pix2pix";
definition.traits.is_unet_edit = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -209,6 +221,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXL_VEGA;
definition.geometry = {8, 4, 3};
definition.name = "SDXL (Vega)";
+ definition.id = "sdxl_vega";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -223,6 +236,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SDXL_SSD1B;
definition.geometry = {8, 4, 3};
definition.name = "SDXL (SSD1B)";
+ definition.id = "sdxl_ssd1b";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -237,6 +251,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_SVD;
definition.geometry = {8, 4, 3};
definition.name = "SVD";
+ definition.id = "svd";
definition.traits.supports_video_generation = true;
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
@@ -249,6 +264,7 @@ const std::vector<ModelDefinition>& sd_definitions() {
definition.version = VERSION_ESRGAN;
definition.geometry = {8, 4, 3};
definition.name = "ESRGAN";
+ definition.id = "esrgan";
definition.sampling = {EPS_PRED, INFINITY};
definition.sampling.default_sample_method = EULER_A_SAMPLE_METHOD;
diff --git a/src/model/integrations/sd3.cpp b/src/model/integrations/sd3.cpp
index f9c2ad7..3756100 100644
--- a/src/model/integrations/sd3.cpp
+++ b/src/model/integrations/sd3.cpp
@@ -1,21 +1,25 @@
#include "conditioning/conditioner.hpp"
#include "model/diffusion/mmdit.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_sd3(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -27,6 +31,7 @@ const std::vector<ModelDefinition>& sd3_definitions() {
definition.version = VERSION_SD3;
definition.geometry = {1, 16, 3};
definition.name = "SD3.x";
+ definition.id = "sd3";
definition.traits.is_dit = true;
definition.sampling = {FLOW_PRED, 3.f};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
diff --git a/src/model/integrations/seedvr2.cpp b/src/model/integrations/seedvr2.cpp
new file mode 100644
index 0000000..b36d5a9
--- /dev/null
+++ b/src/model/integrations/seedvr2.cpp
@@ -0,0 +1,29 @@
+#include "model/diffusion/seedvr2.hpp"
+#include "model_build_context.h"
+#include "model_definition.h"
+
+static bool build_seedvr2(const sd::model_builders::Context& ctx, const TokenizerConfig&, sd::model_builders::CoreRunners& runners) {
+ if (!ctx.build_diffusion || ctx.build_conditioner)
+ return false;
+ const auto& weights = ctx.tensor_storage_map;
+ const char* prefix = weights.count("model.diffusion_model.vid_in.proj.weight") ? "model.diffusion_model" : "diffusion";
+ runners.diffusion = std::make_shared<SeedVR2::SeedVR2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ weights, ctx.weight_manager, prefix);
+ return true;
+}
+
+const std::vector<ModelDefinition>& seedvr2_definitions() {
+ static const std::vector<ModelDefinition> definitions = [] {
+ ModelDefinition definition;
+ definition.version = VERSION_SEEDVR2;
+ definition.name = "SeedVR2";
+ definition.id = "seedvr2";
+ definition.geometry = {2, 16, 3};
+ definition.traits.is_dit = true;
+ definition.sampling = {FLOW_PRED, 1.f};
+ definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
+ definition.build = build_seedvr2;
+ return std::vector<ModelDefinition>{definition};
+ }();
+ return definitions;
+}
diff --git a/src/model/integrations/sensenova_u1.cpp b/src/model/integrations/sensenova_u1.cpp
index 900f222..cc98374 100644
--- a/src/model/integrations/sensenova_u1.cpp
+++ b/src/model/integrations/sensenova_u1.cpp
@@ -1,18 +1,22 @@
#include "model/diffusion/sensenova_u1.h"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_sensenova_u1(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<SenseNovaU1Conditioner>();
- result.diffusion = std::make_shared<SenseNovaU1::SenseNovaU1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<SenseNovaU1Conditioner>();
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<SenseNovaU1::SenseNovaU1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "",
+ weight_manager);
+ }
return true;
}
@@ -24,6 +28,7 @@ const std::vector<ModelDefinition>& sensenova_u1_definitions() {
definition.version = VERSION_SENSENOVA_U1_5;
definition.geometry = {32, 3, 3};
definition.name = "SenseNova U1.5";
+ definition.id = "sensenova_u1_5";
definition.traits.is_dit = true;
definition.traits.fake_vae = true;
definition.sampling = {SENSENOVA_U1_FLOW_PRED, 3.f};
diff --git a/src/model/integrations/wan.cpp b/src/model/integrations/wan.cpp
index 2daada2..fc14fb6 100644
--- a/src/model/integrations/wan.cpp
+++ b/src/model/integrations/wan.cpp
@@ -1,45 +1,51 @@
#include "model/diffusion/wan.hpp"
#include "conditioning/conditioner.hpp"
#include "model/audio/wav2vec2.hpp"
+#include "model/diffusion/relay_vsr.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_wan(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- true,
- 0,
- true,
- weight_manager);
- result.diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager);
- if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ true,
+ 0,
+ true,
+ weight_manager);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager);
+ }
+ if (ctx.build_auxiliary && sd_ctx_params->build_high_noise) {
result.high_noise_diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.high_noise_diffusion_model",
version,
weight_manager);
}
- if (result.diffusion->get_desc() == "Wan2.1-I2V-14B" ||
- result.diffusion->get_desc() == "Wan2.1-FLF2V-14B" ||
- result.diffusion->get_desc() == "Wan2.1-I2V-1.3B") {
- if (!sd::model_builders::ensure_backend_pair(ctx.backends, SDBackendModule::CLIP_VISION)) {
+ if (ctx.build_auxiliary && result.diffusion &&
+ (result.diffusion->get_desc() == "Wan2.1-I2V-14B" ||
+ result.diffusion->get_desc() == "Wan2.1-FLF2V-14B" ||
+ result.diffusion->get_desc() == "Wan2.1-I2V-1.3B")) {
+ if (!sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::CLIP_VISION)) {
return false;
}
result.clip_vision = std::make_shared<FrozenCLIPVisionEmbedder>(ctx.backends.runtime_backend(SDBackendModule::CLIP_VISION),
tensor_storage_map,
weight_manager);
}
- if (version == VERSION_WAN2_2_S2V &&
+ if (ctx.build_auxiliary && version == VERSION_WAN2_2_S2V &&
tensor_storage_map.count("wav2vec2.encoder.layer_norm.bias") > 0) {
- if (!sd::model_builders::ensure_backend_pair(ctx.backends, SDBackendModule::AUDIO_ENCODER)) {
+ if (!sd::model_builders::ensure_model_backend_pair(ctx.backends, SDBackendModule::AUDIO_ENCODER)) {
return false;
}
result.audio_encoder = std::make_shared<Wav2Vec2::Wav2Vec2ModelRunner>(ctx.backends.runtime_backend(SDBackendModule::AUDIO_ENCODER),
@@ -53,11 +59,33 @@ static bool build_wan(const sd::model_builders::Context& ctx, const TokenizerCon
const std::vector<ModelDefinition>& wan_definitions() {
static const std::vector<ModelDefinition> definitions = [] {
std::vector<ModelDefinition> result;
+ {
+ ModelDefinition definition;
+ definition.version = VERSION_RELAY_VSR;
+ definition.name = "RelayVSR";
+ definition.id = "relayvsr";
+ definition.geometry = {2, 48, 3};
+ definition.traits.is_dit = true;
+ definition.sampling = {FLOW_PRED, 1.f};
+ definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
+ definition.build = [](const sd::model_builders::Context& ctx, const TokenizerConfig&, sd::model_builders::CoreRunners& runners) {
+ if (!ctx.build_diffusion || ctx.build_conditioner)
+ return false;
+ const auto channels = ctx.tensor_storage_map.at("wan.lq_proj.conv1.weight").ne[2];
+ const int scale = channels == 192 ? 2 : channels == 768 ? 4
+ : 0;
+ runners.diffusion = std::make_shared<RelayVSR::WanVSRRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ ctx.tensor_storage_map, scale, ctx.weight_manager);
+ return runners.diffusion != nullptr;
+ };
+ result.push_back(std::move(definition));
+ }
{
ModelDefinition definition;
definition.version = VERSION_WAN2;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.x";
+ definition.id = "wan2";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
@@ -73,6 +101,7 @@ const std::vector<ModelDefinition>& wan_definitions() {
definition.version = VERSION_WAN2_2_I2V;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.2 I2V";
+ definition.id = "wan2_2_i2v";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
@@ -88,6 +117,7 @@ const std::vector<ModelDefinition>& wan_definitions() {
definition.version = VERSION_WAN2_2_TI2V;
definition.geometry = {2, 48, 3};
definition.name = "Wan 2.2 TI2V";
+ definition.id = "wan2_2_ti2v";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
@@ -103,6 +133,7 @@ const std::vector<ModelDefinition>& wan_definitions() {
definition.version = VERSION_WAN2_2_S2V;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.2 S2V";
+ definition.id = "wan2_2_s2v";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
diff --git a/src/model/integrations/z_image.cpp b/src/model/integrations/z_image.cpp
index 29054d4..d3a40b9 100644
--- a/src/model/integrations/z_image.cpp
+++ b/src/model/integrations/z_image.cpp
@@ -1,25 +1,29 @@
#include "model/diffusion/z_image.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_z_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- version,
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ version,
+ weight_manager);
+ }
return true;
}
@@ -31,6 +35,7 @@ const std::vector<ModelDefinition>& z_image_definitions() {
definition.version = VERSION_Z_IMAGE;
definition.geometry = {1, 16, 3};
definition.name = "Z-Image";
+ definition.id = "z_image";
definition.traits.is_dit = true;
definition.traits.vae_kind = ModelVAEKind::FLUX;
definition.sampling = {FLOW_PRED, 3.f};
diff --git a/src/model/integrations/z_image_l2p.cpp b/src/model/integrations/z_image_l2p.cpp
index d8104f8..fbcd22b 100644
--- a/src/model/integrations/z_image_l2p.cpp
+++ b/src/model/integrations/z_image_l2p.cpp
@@ -1,24 +1,28 @@
#include "model/diffusion/z_image_l2p.hpp"
#include "conditioning/conditioner.hpp"
+#include "model_build_context.h"
#include "model_definition.h"
-#include "pipeline/model_builders.h"
static bool build_z_image_l2p(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
- result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
- tensor_storage_map,
- version,
- "",
- false,
- weight_manager,
- tokenizers);
- result.diffusion = std::make_shared<ZImageL2P::ZImageL2PRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
- tensor_storage_map,
- "model.diffusion_model",
- weight_manager);
+ if (ctx.build_conditioner) {
+ result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+ tensor_storage_map,
+ version,
+ "",
+ false,
+ weight_manager,
+ tokenizers);
+ }
+ if (ctx.build_diffusion) {
+ result.diffusion = std::make_shared<ZImageL2P::ZImageL2PRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+ tensor_storage_map,
+ "model.diffusion_model",
+ weight_manager);
+ }
return true;
}
@@ -30,6 +34,7 @@ const std::vector<ModelDefinition>& z_image_l2p_definitions() {
definition.version = VERSION_Z_IMAGE_L2P;
definition.geometry = {16, 3, 3};
definition.name = "Z-Image L2P";
+ definition.id = "z_image_l2p";
definition.traits.is_dit = true;
definition.traits.fake_vae = true;
definition.sampling = {FLOW_PRED, 3.f};
diff --git a/src/model/te/clip.hpp b/src/model/te/clip.hpp
index 858895d..113f974 100644
--- a/src/model/te/clip.hpp
+++ b/src/model/te/clip.hpp
@@ -166,7 +166,9 @@ public:
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* input_ids,
- ggml_tensor* custom_embed_weight) {
+ ggml_tensor* custom_embed_weight,
+ ggml_tensor* inline_vectors = nullptr,
+ ggml_tensor* inline_positions = nullptr) {
// input_ids: [N, n_token]
auto token_embed_weight = params["token_embedding.weight"];
auto position_embed_weight = params["position_embedding.weight"];
@@ -176,6 +178,13 @@ public:
auto token_embedding = ggml_get_rows(ctx->ggml_ctx, custom_embed_weight != nullptr ? custom_embed_weight : token_embed_weight, input_ids);
token_embedding = ggml_reshape_3d(ctx->ggml_ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
+ if (inline_vectors) {
+ const int64_t tokens = token_embedding->ne[1], batch = token_embedding->ne[2];
+ token_embedding = ggml_reshape_2d(ctx->ggml_ctx, token_embedding, token_embedding->ne[0], tokens * batch);
+ token_embedding = ggml_set_rows(ctx->ggml_ctx, token_embedding, inline_vectors, inline_positions);
+ token_embedding = ggml_reshape_3d(ctx->ggml_ctx, token_embedding, token_embedding->ne[0], tokens, batch);
+ }
+
// token_embedding + position_embedding
auto x = ggml_add(ctx->ggml_ctx,
token_embedding,
@@ -305,16 +314,18 @@ public:
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* input_ids,
ggml_tensor* tkn_embeddings,
- ggml_tensor* mask = nullptr,
- size_t max_token_idx = 0,
- bool return_pooled = false,
- int clip_skip = -1) {
+ ggml_tensor* mask = nullptr,
+ size_t max_token_idx = 0,
+ bool return_pooled = false,
+ int clip_skip = -1,
+ ggml_tensor* inline_vectors = nullptr,
+ ggml_tensor* inline_positions = nullptr) {
// input_ids: [N, n_token]
auto embeddings = std::dynamic_pointer_cast<CLIPEmbeddings>(blocks["embeddings"]);
auto encoder = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
auto final_layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer_norm"]);
- auto x = embeddings->forward(ctx, input_ids, tkn_embeddings); // [N, n_token, hidden_size]
+ auto x = embeddings->forward(ctx, input_ids, tkn_embeddings, inline_vectors, inline_positions); // [N, n_token, hidden_size]
sd::ggml_graph_cut::mark_graph_cut(x, "clip_text.prelude", "x");
x = encoder->forward(ctx, x, mask, return_pooled ? -1 : clip_skip, "clip_text");
if (return_pooled || with_final_ln) {
@@ -474,6 +485,9 @@ struct CLIPTextModelRunner : public GGMLRunner {
CLIPTextModel model;
std::vector<float> attention_mask_vec;
+ sd::Tensor<int32_t> inline_ids_;
+ sd::Tensor<float> inline_vectors_;
+ sd::Tensor<int64_t> inline_positions_;
CLIPTextModelRunner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map,
@@ -509,9 +523,11 @@ struct CLIPTextModelRunner : public GGMLRunner {
ggml_tensor* input_ids,
ggml_tensor* embeddings,
ggml_tensor* mask,
- size_t max_token_idx = 0,
- bool return_pooled = false,
- int clip_skip = -1) {
+ size_t max_token_idx = 0,
+ bool return_pooled = false,
+ int clip_skip = -1,
+ ggml_tensor* inline_vectors = nullptr,
+ ggml_tensor* inline_positions = nullptr) {
size_t N = input_ids->ne[1];
size_t n_token = input_ids->ne[0];
if (input_ids->ne[0] > model.n_token) {
@@ -519,17 +535,46 @@ struct CLIPTextModelRunner : public GGMLRunner {
input_ids = ggml_reshape_2d(ctx->ggml_ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
}
- return model.forward(ctx, input_ids, embeddings, mask, max_token_idx, return_pooled, clip_skip);
+ return model.forward(ctx, input_ids, embeddings, mask, max_token_idx, return_pooled, clip_skip, inline_vectors, inline_positions);
}
ggml_cgraph* build_graph(const sd::Tensor<int32_t>& input_ids_tensor,
- int num_custom_embeddings = 0,
- void* custom_embeddings_data = nullptr,
- size_t max_token_idx = 0,
- bool return_pooled = false,
- int clip_skip = -1) {
- ggml_cgraph* gf = new_graph_custom(2048);
- ggml_tensor* input_ids = make_input(input_ids_tensor);
+ int num_custom_embeddings = 0,
+ void* custom_embeddings_data = nullptr,
+ size_t max_token_idx = 0,
+ bool return_pooled = false,
+ int clip_skip = -1,
+ const sd::Tensor<float>* inversion = nullptr) {
+ ggml_cgraph* gf = new_graph_custom(2048);
+ ggml_tensor* inline_vectors = nullptr;
+ ggml_tensor* inline_positions = nullptr;
+ const auto* ids = &input_ids_tensor;
+ if (inversion) {
+ std::vector<int64_t> positions;
+ std::vector<float> vectors;
+ for (int64_t i = 0; i < input_ids_tensor.numel(); ++i) {
+ const int row = input_ids_tensor.values()[i] - model.vocab_size;
+ if (row < 0)
+ continue;
+ if (inversion->dim() != 2 || inversion->shape()[0] != model.hidden_size || row >= inversion->shape()[1])
+ throw std::runtime_error("invalid inline CLIP embedding row");
+ positions.push_back(i);
+ const float* begin = inversion->data() + row * model.hidden_size;
+ vectors.insert(vectors.end(), begin, begin + model.hidden_size);
+ }
+ if (!positions.empty()) {
+ inline_ids_ = input_ids_tensor;
+ for (auto i : positions)
+ inline_ids_.values()[i] = 0;
+ ids = &inline_ids_;
+ const int64_t count = positions.size();
+ inline_vectors_ = sd::Tensor<float>({model.hidden_size, count}, std::move(vectors));
+ inline_positions_ = sd::Tensor<int64_t>({count}, std::move(positions));
+ inline_vectors = make_input(inline_vectors_);
+ inline_positions = make_input(inline_positions_);
+ }
+ }
+ ggml_tensor* input_ids = make_input(*ids);
ggml_tensor* embeddings = nullptr;
@@ -561,7 +606,7 @@ struct CLIPTextModelRunner : public GGMLRunner {
auto runner_ctx = get_context();
- ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, embeddings, attention_mask, max_token_idx, return_pooled, clip_skip);
+ ggml_tensor* hidden_states = forward(&runner_ctx, input_ids, embeddings, attention_mask, max_token_idx, return_pooled, clip_skip, inline_vectors, inline_positions);
ggml_build_forward_expand(gf, hidden_states);
@@ -575,9 +620,10 @@ struct CLIPTextModelRunner : public GGMLRunner {
size_t max_token_idx,
bool return_pooled,
int clip_skip,
- bool auto_runner_end = true) {
+ bool auto_runner_end = true,
+ const sd::Tensor<float>* inversion = nullptr) {
auto get_graph = [&]() -> ggml_cgraph* {
- return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip);
+ return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip, inversion);
};
auto result = GGMLRunner::compute(get_graph, n_threads, auto_runner_end);
if (return_pooled) {
diff --git a/src/model/te/ltx_connector.hpp b/src/model/te/ltx_connector.hpp
new file mode 100644
index 0000000..088f0d7
--- /dev/null
+++ b/src/model/te/ltx_connector.hpp
@@ -0,0 +1,495 @@
+#ifndef __SD_MODEL_TE_LTX_CONNECTOR_HPP__
+#define __SD_MODEL_TE_LTX_CONNECTOR_HPP__
+
+#include <cinttypes>
+
+#include "core/pe_cache.h"
+#include "core/tensor_ggml.hpp"
+#include "model/common/ltx.hpp"
+
+namespace LTXV {
+ struct LTXConditioningConfig {
+ int64_t hidden_size = 3840;
+ int64_t audio_hidden_size = 2048;
+ int64_t caption_channels = 3840;
+
+ bool use_connector = false;
+ int64_t connector_hidden_size = 3840;
+ int64_t connector_num_heads = 30;
+ int64_t connector_head_dim = 128;
+ int64_t connector_num_layers = 2;
+ int64_t connector_num_registers = 128;
+ bool connector_rope_interleaved = false;
+ bool connector_apply_gated_attention = false;
+
+ bool use_audio_connector = false;
+ int64_t audio_connector_hidden_size = 2048;
+ int64_t audio_connector_num_heads = 32;
+ int64_t audio_connector_head_dim = 64;
+ int64_t audio_connector_num_layers = 2;
+ int64_t audio_connector_num_registers = 128;
+ bool audio_connector_rope_interleaved = false;
+ bool audio_connector_apply_gated_attention = false;
+
+ bool use_caption_projection = true;
+ bool use_audio_caption_projection = true;
+ bool caption_proj_before_connector = true;
+ bool caption_projection_first_linear = false;
+
+ static LTXConditioningConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, const std::string& prefix) {
+ LTXConditioningConfig config;
+ auto connector_iter = tensor_storage_map.find(prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight");
+ if (connector_iter != tensor_storage_map.end()) {
+ config.use_connector = true;
+ config.connector_hidden_size = connector_iter->second.ne[1];
+ int64_t connector_heads = infer_gate_heads(tensor_storage_map,
+ prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.bias",
+ 32);
+ auto connector_layout = infer_attention_layout(config.connector_hidden_size, connector_heads);
+ config.connector_num_heads = connector_layout.first;
+ config.connector_head_dim = connector_layout.second;
+ config.connector_num_layers = count_prefix_blocks(tensor_storage_map, prefix + ".video_embeddings_connector.", "transformer_1d_blocks.");
+ auto register_iter = tensor_storage_map.find(prefix + ".video_embeddings_connector.learnable_registers");
+ if (register_iter != tensor_storage_map.end()) {
+ config.connector_num_registers = register_iter->second.ne[1];
+ }
+ if (tensor_storage_map.find(prefix + ".video_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.weight") != tensor_storage_map.end()) {
+ config.connector_apply_gated_attention = true;
+ }
+ }
+
+ auto audio_connector_iter = tensor_storage_map.find(prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight");
+ if (audio_connector_iter != tensor_storage_map.end()) {
+ config.use_audio_connector = true;
+ config.audio_connector_hidden_size = audio_connector_iter->second.ne[1];
+ int64_t connector_heads = infer_gate_heads(tensor_storage_map,
+ prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.bias",
+ 32);
+ auto connector_layout = infer_attention_layout(config.audio_connector_hidden_size, connector_heads);
+ config.audio_connector_num_heads = connector_layout.first;
+ config.audio_connector_head_dim = connector_layout.second;
+ config.audio_connector_num_layers = count_prefix_blocks(tensor_storage_map, prefix + ".audio_embeddings_connector.", "transformer_1d_blocks.");
+ auto register_iter = tensor_storage_map.find(prefix + ".audio_embeddings_connector.learnable_registers");
+ if (register_iter != tensor_storage_map.end()) {
+ config.audio_connector_num_registers = register_iter->second.ne[1];
+ }
+ if (tensor_storage_map.find(prefix + ".audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_gate_logits.weight") != tensor_storage_map.end()) {
+ config.audio_connector_apply_gated_attention = true;
+ }
+ }
+
+ const auto video_first = tensor_storage_map.find(prefix + ".caption_projection.linear_1.weight");
+ const auto video_last = tensor_storage_map.find(prefix + ".caption_projection.linear_2.weight");
+ const auto audio_first = tensor_storage_map.find(prefix + ".audio_caption_projection.linear_1.weight");
+ const auto audio_last = tensor_storage_map.find(prefix + ".audio_caption_projection.linear_2.weight");
+ config.use_caption_projection = video_first != tensor_storage_map.end();
+ config.use_audio_caption_projection = audio_first != tensor_storage_map.end();
+ const bool has_video_last = video_last != tensor_storage_map.end();
+ const bool has_audio_last = audio_last != tensor_storage_map.end();
+ if ((has_video_last && !config.use_caption_projection) || (has_audio_last && !config.use_audio_caption_projection))
+ throw std::runtime_error("LTX caption projection requires its first linear layer");
+ if (config.use_caption_projection && config.use_audio_caption_projection &&
+ (has_video_last != has_audio_last || video_first->second.logical_ne0() != audio_first->second.logical_ne0()))
+ throw std::runtime_error("LTX video and audio caption projections require compatible layouts");
+ config.caption_proj_before_connector = !(has_video_last || has_audio_last);
+ config.caption_projection_first_linear = config.use_caption_projection || config.use_audio_caption_projection;
+ if (config.use_caption_projection) {
+ config.caption_channels = video_first->second.logical_ne0();
+ config.hidden_size = (has_video_last ? video_last : video_first)->second.ne[1];
+ } else {
+ config.hidden_size = config.connector_hidden_size;
+ }
+ if (config.use_audio_caption_projection) {
+ config.caption_channels = audio_first->second.logical_ne0();
+ config.audio_hidden_size = (has_audio_last ? audio_last : audio_first)->second.ne[1];
+ } else {
+ config.audio_hidden_size = config.audio_connector_hidden_size;
+ }
+ LOG_VERBOSE("ltxav_conditioning: video_hidden_size = %" PRId64 ", audio_hidden_size = %" PRId64 ", video_layers = %" PRId64 ", audio_layers = %" PRId64,
+ config.connector_hidden_size, config.audio_connector_hidden_size,
+ config.connector_num_layers, config.audio_connector_num_layers);
+ return config;
+ }
+ };
+
+ __STATIC_INLINE__ std::vector<float> build_1d_rope_matrix(int64_t seq_len,
+ int dim,
+ int num_heads = 1,
+ float theta = 10000.f,
+ float positional_scale = 4096.f,
+ bool double_precision = false) {
+ GGML_ASSERT(dim % num_heads == 0);
+ const std::vector<float> indices = double_precision ? std::vector<float>() : generate_freq_grid(theta, 1, dim);
+ const std::vector<double> indices_d =
+ double_precision ? generate_freq_grid_double(static_cast<double>(theta), 1, dim) : std::vector<double>();
+ const int half_dim = dim / 2;
+ const int pad_size = half_dim - static_cast<int>(double_precision ? indices_d.size() : indices.size());
+
+ std::vector<std::vector<float>> freqs(static_cast<size_t>(seq_len), std::vector<float>(half_dim, 0.f));
+ for (int64_t pos = 0; pos < seq_len; pos++) {
+ int out_idx = 0;
+ for (int i = 0; i < pad_size; i++) {
+ freqs[static_cast<size_t>(pos)][out_idx++] = 0.f;
+ }
+
+ if (double_precision) {
+ double coord = static_cast<double>(pos) / static_cast<double>(positional_scale);
+ for (double index : indices_d) {
+ freqs[static_cast<size_t>(pos)][out_idx++] = static_cast<float>(index * (coord * 2.0 - 1.0));
+ }
+ } else {
+ float coord = static_cast<float>(pos) / positional_scale;
+ for (float index : indices) {
+ freqs[static_cast<size_t>(pos)][out_idx++] = index * (coord * 2.f - 1.f);
+ }
+ }
+ }
+
+ if (num_heads > 1) {
+ return build_rope_matrix_from_frequencies(split_frequencies_by_heads(freqs, dim, num_heads), dim / num_heads);
+ }
+ return build_rope_matrix_from_frequencies(freqs, dim);
+ }
+
+ struct PixArtAlphaTextProjection : public GGMLBlock {
+ PixArtAlphaTextProjection(int64_t in_features,
+ int64_t hidden_size,
+ int64_t out_features = -1) {
+ if (out_features < 0) {
+ out_features = hidden_size;
+ }
+ blocks["linear_1"] = std::make_shared<Linear>(in_features, hidden_size, true, true);
+ blocks["linear_2"] = std::make_shared<Linear>(hidden_size, out_features, true, true);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* caption) {
+ auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
+ auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["linear_2"]);
+
+ caption = linear_1->forward(ctx, caption);
+ caption = ggml_ext_gelu(ctx, caption, true);
+ caption = linear_2->forward(ctx, caption);
+ return caption;
+ }
+ };
+
+ struct NormSingleLinearTextProjection : public GGMLBlock {
+ int64_t in_features;
+ int64_t hidden_size;
+
+ NormSingleLinearTextProjection(int64_t in_features,
+ int64_t hidden_size)
+ : in_features(in_features), hidden_size(hidden_size) {
+ blocks["linear_1"] = std::make_shared<Linear>(in_features, hidden_size, true, true);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* caption) {
+ auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["linear_1"]);
+ caption = ggml_rms_norm(ctx->ggml_ctx, caption, 1e-6f);
+ caption = ggml_ext_scale(ctx, caption, std::sqrt(static_cast<float>(hidden_size) / static_cast<float>(in_features)));
+ return linear_1->forward(ctx, caption);
+ }
+ };
+
+ struct BasicTransformerBlock1D : public GGMLBlock {
+ BasicTransformerBlock1D(int64_t dim,
+ int64_t n_heads,
+ int64_t d_head,
+ bool rope_interleaved,
+ bool apply_gated_attention = false) {
+ blocks["attn1"] = std::make_shared<CrossAttention>(dim, dim, n_heads, d_head, apply_gated_attention, rope_interleaved);
+ blocks["ff"] = std::make_shared<FeedForward>(dim, dim, 4, FeedForward::Activation::GELU);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx,
+ ggml_tensor* x,
+ ggml_tensor* pe,
+ ggml_tensor* attention_mask = nullptr) {
+ auto attn1 = std::dynamic_pointer_cast<CrossAttention>(blocks["attn1"]);
+ auto ff = std::dynamic_pointer_cast<FeedForward>(blocks["ff"]);
+
+ auto h = rms_norm(ctx->ggml_ctx, x);
+ h = attn1->forward(ctx, h, nullptr, attention_mask, pe);
+ x = ggml_add(ctx->ggml_ctx, x, h);
+
+ h = rms_norm(ctx->ggml_ctx, x);
+ h = ff->forward(ctx, h);
+ x = ggml_add(ctx->ggml_ctx, x, h);
+ return x;
+ }
+ };
+
+ struct Embeddings1DConnector : public GGMLBlock {
+ int64_t hidden_size;
+ int64_t num_attention_heads;
+ int64_t attention_head_dim;
+ int64_t num_layers;
+ int64_t num_learnable_registers;
+ bool rope_interleaved;
+ bool apply_gated_attention;
+
+ void init_params(ggml_context* ctx,
+ const String2TensorStorage& tensor_storage_map = {},
+ const std::string prefix = "") override {
+ if (num_learnable_registers > 0) {
+ params["learnable_registers"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_learnable_registers);
+ }
+ }
+
+ Embeddings1DConnector(int64_t hidden_size,
+ int64_t num_attention_heads = 30,
+ int64_t attention_head_dim = 128,
+ int64_t num_layers = 2,
+ int64_t num_learnable_registers = 128,
+ bool rope_interleaved = false,
+ bool apply_gated_attention = false)
+ : hidden_size(hidden_size),
+ num_attention_heads(num_attention_heads),
+ attention_head_dim(attention_head_dim),
+ num_layers(num_layers),
+ num_learnable_registers(num_learnable_registers),
+ rope_interleaved(rope_interleaved),
+ apply_gated_attention(apply_gated_attention) {
+ for (int i = 0; i < num_layers; i++) {
+ blocks["transformer_1d_blocks." + std::to_string(i)] =
+ std::make_shared<BasicTransformerBlock1D>(hidden_size,
+ num_attention_heads,
+ attention_head_dim,
+ rope_interleaved,
+ apply_gated_attention);
+ }
+ }
+
+ ggml_tensor* append_registers(GGMLRunnerContext* ctx,
+ ggml_tensor* hidden_states) {
+ if (num_learnable_registers <= 0 || params.count("learnable_registers") == 0) {
+ return hidden_states;
+ }
+
+ int64_t seq_len = hidden_states->ne[1];
+ int64_t target_len = std::max<int64_t>(1024, seq_len);
+ int64_t duplications = (target_len + num_learnable_registers - 1) / num_learnable_registers;
+ int64_t total_to_keep = duplications * num_learnable_registers - seq_len;
+ if (total_to_keep <= 0) {
+ return hidden_states;
+ }
+
+ auto regs = ggml_reshape_3d(ctx->ggml_ctx, params["learnable_registers"], hidden_size, num_learnable_registers, 1);
+ auto temp = ggml_new_tensor_3d(ctx->ggml_ctx, regs->type, regs->ne[0], regs->ne[1], hidden_states->ne[2]);
+ regs = ggml_repeat(ctx->ggml_ctx, regs, temp);
+
+ auto regs_full = regs;
+ for (int64_t i = 1; i < duplications; i++) {
+ regs_full = ggml_concat(ctx->ggml_ctx, regs_full, regs, 1);
+ }
+ regs_full = ggml_ext_slice(ctx, regs_full, 1, seq_len, seq_len + total_to_keep);
+ return ggml_concat(ctx->ggml_ctx, hidden_states, regs_full, 1);
+ }
+
+ ggml_tensor* forward(GGMLRunnerContext* ctx,
+ ggml_tensor* hidden_states,
+ ggml_tensor* pe,
+ ggml_tensor* attention_mask = nullptr) {
+ hidden_states = append_registers(ctx, hidden_states);
+
+ for (int i = 0; i < num_layers; i++) {
+ auto block = std::dynamic_pointer_cast<BasicTransformerBlock1D>(blocks["transformer_1d_blocks." + std::to_string(i)]);
+ hidden_states = block->forward(ctx, hidden_states, pe, attention_mask);
+ }
+
+ return ggml_rms_norm(ctx->ggml_ctx, hidden_states, 1e-6f);
+ }
+ };
+
+ struct LTXAVConditioningBlock : public GGMLBlock {
+ LTXConditioningConfig config;
+
+ LTXAVConditioningBlock(const LTXConditioningConfig& config)
+ : config(config) {
+ if (config.use_caption_projection) {
+ if (config.caption_proj_before_connector) {
+ if (config.caption_projection_first_linear) {
+ blocks["caption_projection"] = std::make_shared<NormSingleLinearTextProjection>(config.caption_channels, config.hidden_size);
+ }
+ } else {
+ blocks["caption_projection"] = std::make_shared<PixArtAlphaTextProjection>(config.caption_channels, config.hidden_size, config.hidden_size);
+ }
+ }
+ if (config.use_audio_caption_projection) {
+ if (config.caption_proj_before_connector) {
+ if (config.caption_projection_first_linear) {
+ blocks["audio_caption_projection"] = std::make_shared<NormSingleLinearTextProjection>(config.caption_channels, config.audio_hidden_size);
+ }
+ } else {
+ blocks["audio_caption_projection"] = std::make_shared<PixArtAlphaTextProjection>(config.caption_channels, config.audio_hidden_size, config.audio_hidden_size);
+ }
+ }
+
+ if (config.use_connector) {
+ blocks["video_embeddings_connector"] = std::make_shared<Embeddings1DConnector>(config.connector_hidden_size,
+ config.connector_num_heads,
+ config.connector_head_dim,
+ config.connector_num_layers,
+ config.connector_num_registers,
+ config.connector_rope_interleaved,
+ config.connector_apply_gated_attention);
+ }
+ if (config.use_audio_connector) {
+ blocks["audio_embeddings_connector"] = std::make_shared<Embeddings1DConnector>(config.audio_connector_hidden_size,
+ config.audio_connector_num_heads,
+ config.audio_connector_head_dim,
+ config.audio_connector_num_layers,
+ config.audio_connector_num_registers,
+ config.audio_connector_rope_interleaved,
+ config.audio_connector_apply_gated_attention);
+ }
+ }
+
+ std::pair<ggml_tensor*, ggml_tensor*> process_contexts(GGMLRunnerContext* ctx,
+ ggml_tensor* v_context,
+ ggml_tensor* a_context,
+ ggml_tensor* video_connector_pe,
+ ggml_tensor* audio_connector_pe) {
+ const bool process_audio_context = a_context != nullptr;
+ if (config.caption_proj_before_connector) {
+ if (config.use_caption_projection &&
+ blocks.count("caption_projection") > 0 &&
+ v_context != nullptr &&
+ v_context->ne[0] == config.caption_channels) {
+ auto caption_projection = std::dynamic_pointer_cast<NormSingleLinearTextProjection>(blocks["caption_projection"]);
+ if (caption_projection != nullptr) {
+ v_context = caption_projection->forward(ctx, v_context);
+ }
+ }
+ if (process_audio_context &&
+ config.use_audio_caption_projection &&
+ blocks.count("audio_caption_projection") > 0 &&
+ a_context != nullptr &&
+ a_context->ne[0] == config.caption_channels) {
+ auto caption_projection = std::dynamic_pointer_cast<NormSingleLinearTextProjection>(blocks["audio_caption_projection"]);
+ if (caption_projection != nullptr) {
+ a_context = caption_projection->forward(ctx, a_context);
+ }
+ }
+ }
+
+ if (config.use_connector && v_context != nullptr && v_context->ne[0] == config.connector_hidden_size) {
+ auto connector = std::dynamic_pointer_cast<Embeddings1DConnector>(blocks["video_embeddings_connector"]);
+ v_context = connector->forward(ctx, v_context, video_connector_pe);
+ }
+ if (process_audio_context &&
+ config.use_audio_connector &&
+ a_context != nullptr &&
+ a_context->ne[0] == config.audio_connector_hidden_size) {
+ auto connector = std::dynamic_pointer_cast<Embeddings1DConnector>(blocks["audio_embeddings_connector"]);
+ a_context = connector->forward(ctx, a_context, audio_connector_pe);
+ }
+
+ if (!config.caption_proj_before_connector &&
+ config.use_caption_projection &&
+ blocks.count("caption_projection") > 0 &&
+ v_context != nullptr &&
+ v_context->ne[0] == config.caption_channels) {
+ auto caption_projection = std::dynamic_pointer_cast<PixArtAlphaTextProjection>(blocks["caption_projection"]);
+ if (caption_projection != nullptr) {
+ v_context = caption_projection->forward(ctx, v_context);
+ }
+ }
+ if (process_audio_context &&
+ !config.caption_proj_before_connector &&
+ config.use_audio_caption_projection &&
+ blocks.count("audio_caption_projection") > 0 &&
+ a_context != nullptr &&
+ a_context->ne[0] == config.caption_channels) {
+ auto caption_projection = std::dynamic_pointer_cast<PixArtAlphaTextProjection>(blocks["audio_caption_projection"]);
+ if (caption_projection != nullptr) {
+ a_context = caption_projection->forward(ctx, a_context);
+ }
+ }
+
+ return {v_context, a_context};
+ }
+ };
+
+ struct LTXConditioningRunner : public GGMLRunner {
+ static constexpr const char* prefix = "text_encoders.connector";
+ LTXAVConditioningBlock model;
+ sd::PECache pe_cache;
+ int64_t video_projection_size = 0;
+ int64_t audio_projection_size = 0;
+
+ LTXConditioningRunner(ggml_backend_t backend, const String2TensorStorage& tensors, std::shared_ptr<RunnerWeightManager> manager, const std::string& projector_prefix = "text_encoders.connector.text_embedding_projection")
+ : GGMLRunner(backend, std::move(manager)),
+ model(LTXConditioningConfig::detect_from_weights(tensors, prefix)) {
+ if (!model.config.use_connector || !model.config.use_audio_connector)
+ throw std::runtime_error("LTX text encoding requires video and audio connector weights in the model sources");
+ auto video = tensors.find(projector_prefix + ".video_aggregate_embed.weight");
+ if (video != tensors.end()) {
+ auto audio = tensors.find(projector_prefix + ".audio_aggregate_embed.weight");
+ if (audio == tensors.end())
+ throw std::runtime_error("LTX dual text projection requires both video and audio weights");
+ video_projection_size = video->second.ne[1];
+ audio_projection_size = audio->second.ne[1];
+ }
+ model.init(params_ctx, tensors, prefix);
+ }
+
+ std::string get_desc() override { return "ltxav_conditioning"; }
+
+ ggml_tensor* connector_pe(int64_t length, bool audio) {
+ const auto& config = model.config;
+ if (!(audio ? config.use_audio_connector : config.use_connector))
+ return nullptr;
+ const auto registers = audio ? config.audio_connector_num_registers : config.connector_num_registers;
+ const auto hidden = audio ? config.audio_connector_hidden_size : config.connector_hidden_size;
+ const auto heads = audio ? config.audio_connector_num_heads : config.connector_num_heads;
+ const auto head_dim = audio ? config.audio_connector_head_dim : config.connector_head_dim;
+ const auto full_length = registers > 0 ? ((std::max<int64_t>(1024, length) + registers - 1) / registers) * registers : length;
+ sd::PECacheKey key(audio ? "ltxav_audio_connector_pe" : "ltxav_video_connector_pe");
+ key.add(full_length).add(hidden).add(heads).add(head_dim);
+ return make_input(pe_cache.get_or_create(key, [&] {
+ return sd::Tensor<float>({2, 2, head_dim / 2, full_length * heads},
+ build_1d_rope_matrix(full_length, static_cast<int>(hidden), static_cast<int>(heads), 10000.f, 4096.f, true));
+ }));
+ }
+
+ std::pair<sd::Tensor<float>, sd::Tensor<float>> compute(int n_threads, const sd::Tensor<float>& projected) {
+ if (projected.empty() || projected.dim() > 3)
+ throw std::runtime_error("LTX text projection must produce a sequence of embeddings");
+ std::pair<sd::Tensor<float>, sd::Tensor<float>> result;
+ std::pair<ggml_tensor*, ggml_tensor*> outputs;
+ const auto get_graph = [&] {
+ auto graph = new_graph_custom(LTXAV_GRAPH_SIZE);
+ auto input = make_input(projected);
+ auto video = input;
+ auto audio = input;
+ auto context = get_context();
+ if (video_projection_size) {
+ if (input->ne[0] != video_projection_size + audio_projection_size)
+ throw std::runtime_error("LTX dual text projection dimensions do not match its weights");
+ video = ggml_ext_slice(&context, input, 0, 0, video_projection_size);
+ audio = ggml_ext_slice(&context, input, 0, video_projection_size, video_projection_size + audio_projection_size);
+ }
+ outputs = model.process_contexts(&context, video, audio, connector_pe(input->ne[1], false), connector_pe(input->ne[1], true));
+ if (outputs.first->ne[0] != model.config.hidden_size || outputs.second->ne[0] != model.config.audio_hidden_size)
+ throw std::runtime_error("LTX conditioning output dimensions do not match its projections");
+ for (auto output : {outputs.first, outputs.second}) {
+ ggml_set_output(output);
+ ggml_build_forward_expand(graph, output);
+ }
+ return graph;
+ };
+ const auto read_outputs = [&] {
+ result.first = sd::make_sd_tensor_from_ggml<float>(outputs.first);
+ result.second = sd::make_sd_tensor_from_ggml<float>(outputs.second);
+ return !result.first.empty() && !result.second.empty();
+ };
+ if (!GGMLRunner::compute(get_graph, n_threads, false, true, read_outputs))
+ return {};
+ return result;
+ }
+ };
+
+} // namespace LTXV
+
+#endif // __SD_MODEL_TE_LTX_CONNECTOR_HPP__
diff --git a/src/model/te/ming_image_te.hpp b/src/model/te/ming_image_te.hpp
index c8f2047..d53430c 100644
--- a/src/model/te/ming_image_te.hpp
+++ b/src/model/te/ming_image_te.hpp
@@ -14,17 +14,17 @@ namespace MingImageTE {
static MingImageTEConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
MingImageTEConfig config;
for (const auto& entry : tensors) {
- if (starts_with(entry.first, prefix + ".backbone.") &&
+ if (starts_with(entry.first, prefix + ".llm.backbone.") &&
contains(entry.first, ".mlp.experts.") && entry.second.type == GGML_TYPE_I8) {
throw std::runtime_error("Ming-Image INT8/W4A8 text encoder experts are not supported; use the BF16 text encoder");
}
}
bool vision = false;
- config.backbone = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".backbone.", LLM::LLMArch::BAILING_MOE, vision);
+ config.backbone = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".llm.backbone.", LLM::LLMArch::BAILING_MOE, vision);
config.connector = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".connector.", LLM::LLMArch::QWEN2, vision);
- const auto query = tensors.find(prefix + ".query_tokens_dict.16x16");
- const auto projection = tensors.find(prefix + ".proj_out.weight");
- const auto direct = tensors.find(prefix + ".proj_directvlm.1.weight");
+ const auto query = tensors.find(prefix + ".connector.query_tokens_dict.16x16");
+ const auto projection = tensors.find(prefix + ".connector.proj_out.weight");
+ const auto direct = tensors.find(prefix + ".connector.proj_directvlm.1.weight");
if (query == tensors.end() || projection == tensors.end() || direct == tensors.end()) {
throw std::runtime_error("Ming-Image requires the learned queries, connector and both condition projections");
}
@@ -53,39 +53,39 @@ namespace MingImageTE {
MingImageTEConfig config;
void init_params(ggml_context* ctx, const String2TensorStorage& tensors = {}, const std::string prefix = "") override {
- params["query_tokens_dict.16x16"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, config.backbone.hidden_size, config.num_queries);
+ params["connector.query_tokens_dict.16x16"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, config.backbone.hidden_size, config.num_queries);
}
public:
explicit MingImageTextModel(const MingImageTEConfig& config)
: config(config) {
- blocks["backbone"] = std::make_shared<LLM::TextModel>(config.backbone, "ming_image.backbone");
- blocks["connector"] = std::make_shared<ConnectorModel>(config.connector);
- blocks["proj_in"] = std::make_shared<Linear>(config.backbone.hidden_size, config.connector.hidden_size);
- blocks["proj_out"] = std::make_shared<Linear>(config.connector.hidden_size, config.caption_dim);
- blocks["proj_directvlm.0"] = std::make_shared<RMSNorm>(config.backbone.hidden_size * 3, 1e-5f);
- blocks["proj_directvlm.1"] = std::make_shared<Linear>(config.backbone.hidden_size * 3, config.diffusion_dim);
+ blocks["llm.backbone"] = std::make_shared<LLM::TextModel>(config.backbone, "ming_image.backbone");
+ blocks["connector"] = std::make_shared<ConnectorModel>(config.connector);
+ blocks["connector.proj_in"] = std::make_shared<Linear>(config.backbone.hidden_size, config.connector.hidden_size);
+ blocks["connector.proj_out"] = std::make_shared<Linear>(config.connector.hidden_size, config.caption_dim);
+ blocks["connector.proj_directvlm.0"] = std::make_shared<RMSNorm>(config.backbone.hidden_size * 3, 1e-5f);
+ blocks["connector.proj_directvlm.1"] = std::make_shared<Linear>(config.backbone.hidden_size * 3, config.diffusion_dim);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* ids, int64_t prompt_length, ggml_tensor* mask, ggml_tensor* image_mask, ggml_tensor* cos, ggml_tensor* sin, ggml_tensor* connector_positions) {
auto gctx = ctx->ggml_ctx;
- auto backbone = std::dynamic_pointer_cast<LLM::TextModel>(blocks["backbone"]);
+ auto backbone = std::dynamic_pointer_cast<LLM::TextModel>(blocks["llm.backbone"]);
auto connector = std::dynamic_pointer_cast<ConnectorModel>(blocks["connector"]);
auto x = backbone->embed(ctx, ids);
const int64_t query_start = prompt_length + 1;
auto before = ggml_ext_slice(ctx, x, 1, 0, query_start);
auto after = ggml_ext_slice(ctx, x, 1, query_start + config.num_queries, x->ne[1]);
- x = ggml_concat(gctx, ggml_concat(gctx, before, params["query_tokens_dict.16x16"], 1), after, 1);
+ x = ggml_concat(gctx, ggml_concat(gctx, before, params["connector.query_tokens_dict.16x16"], 1), after, 1);
// HF hidden_states[20] includes final RMSNorm; sd.cpp selects it as num_layers + 1.
x = backbone->forward_embeds(ctx, x, nullptr, mask, {5, 12, 21}, {}, nullptr, false, image_mask, cos, sin);
auto direct = ggml_ext_slice(ctx, x, 1, 0, prompt_length);
- direct = std::dynamic_pointer_cast<RMSNorm>(blocks["proj_directvlm.0"])->forward(ctx, direct);
- direct = std::dynamic_pointer_cast<Linear>(blocks["proj_directvlm.1"])->forward(ctx, direct);
+ direct = std::dynamic_pointer_cast<RMSNorm>(blocks["connector.proj_directvlm.0"])->forward(ctx, direct);
+ direct = std::dynamic_pointer_cast<Linear>(blocks["connector.proj_directvlm.1"])->forward(ctx, direct);
auto queries = ggml_ext_slice(ctx, x, 0, config.backbone.hidden_size * 2, config.backbone.hidden_size * 3);
queries = ggml_ext_slice(ctx, queries, 1, query_start, query_start + config.num_queries);
- queries = std::dynamic_pointer_cast<Linear>(blocks["proj_in"])->forward(ctx, queries);
+ queries = std::dynamic_pointer_cast<Linear>(blocks["connector.proj_in"])->forward(ctx, queries);
queries = connector->forward_embeds(ctx, queries, connector_positions, nullptr, {});
- queries = std::dynamic_pointer_cast<Linear>(blocks["proj_out"])->forward(ctx, queries);
+ queries = std::dynamic_pointer_cast<Linear>(blocks["connector.proj_out"])->forward(ctx, queries);
queries = ggml_pad(gctx, queries, static_cast<int>(config.diffusion_dim - config.caption_dim), 0, 0, 0);
return ggml_concat(gctx, queries, direct, 1);
}
diff --git a/src/model/te/ph3_image_connector.hpp b/src/model/te/ph3_image_connector.hpp
index a0e82ec..ca2a04d 100644
--- a/src/model/te/ph3_image_connector.hpp
+++ b/src/model/te/ph3_image_connector.hpp
@@ -38,7 +38,7 @@ struct PH3ImageTextConnector : public UnaryBlock {
struct PH3ImageTextConnectorRunner : public GGMLRunner {
PH3ImageTextConnector model;
- static constexpr const char* prefix = "text_encoders.ph3_image_connector";
+ static constexpr const char* prefix = "text_encoders.connector";
PH3ImageTextConnectorRunner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map,
diff --git a/src/model/upscaler/flash_vsr.hpp b/src/model/upscaler/flash_vsr.hpp
index addfc53..95b12c3 100644
--- a/src/model/upscaler/flash_vsr.hpp
+++ b/src/model/upscaler/flash_vsr.hpp
@@ -2,6 +2,7 @@
#define __SD_MODEL_UPSCALER_FLASH_VSR_HPP__
#include "model/common/vsr.hpp"
+#include "model/vae/vae.hpp"
namespace RelayVSR {
@@ -194,7 +195,7 @@ namespace RelayVSR {
}
};
- struct FlashVSRRunner : GGMLRunner {
+ struct FlashVSRRunner : FakeVAE {
FlashVSRConfig config;
FlashVSRConditionNet model;
VSR::GraphInputs inputs;
@@ -202,11 +203,14 @@ namespace RelayVSR {
bool history = false;
FlashVSRRunner(ggml_backend_t backend, const String2TensorStorage& weights, int scale, std::shared_ptr<DeviceResidencyManager> manager)
- : GGMLRunner(backend, manager), config(FlashVSRConfig::detect_from_weights(weights, "flash", scale)), model(config) {
+ : FakeVAE(VERSION_RELAY_VSR, backend, manager), config(FlashVSRConfig::detect_from_weights(weights, "flash", scale)), model(config) {
model.init(params_ctx, weights, "flash");
}
std::string get_desc() override { return "RelayVSR Flash"; }
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model.get_param_tensors(tensors, "flash"); }
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override { model.get_param_tensors(tensors, "flash"); }
+ sd::Tensor<float> _compute(int, const sd::Tensor<float>&, bool) override {
+ throw std::runtime_error("RelayVSR Flash requires relayvsr::decode with the low-quality frames");
+ }
void reset() {
runner_end();
free_cache_ctx_and_buffer();
diff --git a/src/model/upscaler/scale.cpp b/src/model/upscaler/scale.cpp
new file mode 100644
index 0000000..9626d53
--- /dev/null
+++ b/src/model/upscaler/scale.cpp
@@ -0,0 +1,27 @@
+#include "model/upscaler/scale.hpp"
+
+#include "model/upscaler/esrgan.hpp"
+#include "model/upscaler/mosr.hpp"
+#include "model_loader.h"
+
+namespace sd::upscaling {
+ Description describe(const String2TensorStorage& tensors) {
+ if (tensors.count("gblocks.0.weight")) {
+ const auto config = MoSRConfig::detect_from_weights(tensors);
+ return config.is_valid() ? Description{Architecture::MoSR, config.scale} : Description{};
+ }
+ const auto first = tensors.find("conv_first.weight");
+ const auto last = tensors.find("conv_last.weight");
+ if (first == tensors.end() || last == tensors.end() || !tensors.count("body.0.rdb1.conv1.weight") ||
+ first->second.n_dims != 4 || last->second.n_dims != 4 || first->second.ne[2] != 3 || last->second.ne[3] != 3)
+ return {};
+ return {Architecture::ESRGAN, ESRGANConfig::detect_from_weights(tensors).scale};
+ }
+
+ int probe_scale(const std::string& path) {
+ ModelLoader loader;
+ if (path.empty() || !loader.init_from_file_and_convert_name(path, "", VERSION_ESRGAN))
+ return 0;
+ return describe(loader.get_tensor_storage_map()).scale;
+ }
+}
diff --git a/src/model/upscaler/scale.hpp b/src/model/upscaler/scale.hpp
new file mode 100644
index 0000000..d4d10d3
--- /dev/null
+++ b/src/model/upscaler/scale.hpp
@@ -0,0 +1,20 @@
+#ifndef __SD_MODEL_UPSCALER_SCALE_HPP__
+#define __SD_MODEL_UPSCALER_SCALE_HPP__
+
+#include <string>
+
+#include "model.h"
+
+namespace sd::upscaling {
+ enum class Architecture { Unknown,
+ ESRGAN,
+ MoSR };
+ struct Description {
+ Architecture architecture = Architecture::Unknown;
+ int scale = 0;
+ };
+ Description describe(const String2TensorStorage& tensors);
+ int probe_scale(const std::string& path);
+}
+
+#endif // __SD_MODEL_UPSCALER_SCALE_HPP__
diff --git a/src/model/vae/audio_geometry.hpp b/src/model/vae/audio_geometry.hpp
new file mode 100644
index 0000000..17658f7
--- /dev/null
+++ b/src/model/vae/audio_geometry.hpp
@@ -0,0 +1,29 @@
+#ifndef __SD_MODEL_VAE_AUDIO_GEOMETRY_HPP__
+#define __SD_MODEL_VAE_AUDIO_GEOMETRY_HPP__
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <stdexcept>
+#include <vector>
+
+#include "model.h"
+
+namespace sd::vae {
+ inline std::vector<int64_t> audio_latent_shape(SDVersion version, int frames, int fps) {
+ if (frames <= 0 || fps <= 0)
+ throw std::invalid_argument("audio latent frame count and fps must be positive");
+ if (sd_version_is_ltxav(version)) {
+ constexpr float latents_per_second = 16000.0f / 160.0f / 4.0f;
+ const auto length = static_cast<int64_t>(std::ceil((static_cast<float>(frames) / static_cast<float>(fps)) * latents_per_second));
+ return {16, length, 8, 1};
+ }
+ if (sd_version_is_minimax_h3(version)) {
+ const auto length = std::max<int64_t>(1, std::llround(static_cast<double>(frames) * 40.0 / fps));
+ return {length, 2, 32, 1};
+ }
+ throw std::invalid_argument("audio latent shape is not defined for this model family");
+ }
+}
+
+#endif
diff --git a/src/model/vae/auto_encoder_kl.hpp b/src/model/vae/auto_encoder_kl.hpp
index 3199377..afdc2a9 100644
--- a/src/model/vae/auto_encoder_kl.hpp
+++ b/src/model/vae/auto_encoder_kl.hpp
@@ -526,7 +526,7 @@ protected:
int64_t conv_in_out_channels = conv_in_iter->second.ne[3];
if (last_ch_mult > 0 && conv_in_out_channels % last_ch_mult == 0) {
decoder_ch = static_cast<int>(conv_in_out_channels / last_ch_mult);
- LOG_INFO("vae decoder: ch = %d", decoder_ch);
+ LOG_VERBOSE("vae decoder: ch = %d", decoder_ch);
} else {
LOG_WARN("vae decoder: failed to infer ch from %s (%" PRId64 " / %d)",
get_tensor_name(prefix, "decoder.conv_in.weight").c_str(),
@@ -652,6 +652,12 @@ public:
return z;
}
+ int get_latent_channels() const {
+ if (sd_version_uses_flux2_vae(version))
+ return dd_config.z_channels * 4;
+ return use_quant ? embed_dim : dd_config.z_channels;
+ }
+
int get_encoder_output_channels() {
int factor = dd_config.double_z ? 2 : 1;
if (sd_version_uses_flux2_vae(version)) {
@@ -760,10 +766,10 @@ struct AutoEncoderKL : public VAE {
return latents;
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior posterior, bool) override {
if (sd_version_uses_flux2_vae(version)) {
return vae_output;
- } else if (version == VERSION_SD1_PIX2PIX) {
+ } else if (posterior == VAEPosterior::Mode || (posterior == VAEPosterior::ModelDefault && version == VERSION_SD1_PIX2PIX)) {
return sd::ops::chunk(vae_output, 2, 2)[0];
} else {
return gaussian_latent_sample(vae_output, rng);
@@ -818,7 +824,7 @@ struct AutoEncoderKL : public VAE {
}
sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) override {
- auto latents_ = sd_version_is_sefi_image(version) ? sd::ops::slice(latents, 2, 16, 144) : latents;
+ auto latents_ = sd_version_is_sefi_image(version) && latents.shape()[2] == 144 ? sd::ops::slice(latents, 2, 16, 144) : latents;
if (sd_version_uses_flux2_vae(version)) {
int channel_dim = 2;
auto [mean_tensor, std_tensor] = get_latents_mean_std(latents_, channel_dim);
diff --git a/src/model/vae/geometry.hpp b/src/model/vae/geometry.hpp
new file mode 100644
index 0000000..1958149
--- /dev/null
+++ b/src/model/vae/geometry.hpp
@@ -0,0 +1,52 @@
+#ifndef __SD_MODEL_VAE_GEOMETRY_HPP__
+#define __SD_MODEL_VAE_GEOMETRY_HPP__
+
+#include "core/image_geometry.h"
+#include "core/option_types.h"
+#include "model.h"
+
+namespace sd::vae {
+ inline SDVersion version_for_format(sd_vae_format_t format, SDVersion fallback) {
+ switch (format) {
+ case SD_VAE_FORMAT_FLUX:
+ return VERSION_FLUX;
+ case SD_VAE_FORMAT_SD3:
+ return VERSION_SD3;
+ case SD_VAE_FORMAT_FLUX2:
+ return VERSION_FLUX2;
+ case SD_VAE_FORMAT_WAN:
+ return VERSION_WAN2;
+ default:
+ return fallback;
+ }
+ }
+
+ inline int spatial_scale(SDVersion version) {
+ if (version == VERSION_LTXAV)
+ return 32;
+ if (version == VERSION_WAN2_2_TI2V || version == VERSION_QWEN_IMAGE_2_1 ||
+ sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) ||
+ sd_version_is_minimax_h3(version) || sd_version_uses_flux2_vae(version))
+ return 16;
+ if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1 ||
+ sd_version_is_minit2i(version) || sd_version_is_sensenova_u1(version) ||
+ sd_version_is_z_image_l2p(version) || version == VERSION_IRIS)
+ return 1;
+ return 8;
+ }
+
+ inline int aligned_frames(SDVersion version, int frames) {
+ if (frames <= 0)
+ throw std::invalid_argument("video frame count must be positive");
+ if (sd_version_is_minimax_h3(version))
+ return int(sd::align_video_frame_count(frames, 17, 5));
+ if (sd_version_is_ltxav(version))
+ return int(sd::align_video_frame_count(frames, 8));
+ if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) ||
+ sd_version_is_hunyuan_video(version) || version == VERSION_SEEDVR2)
+ return int(sd::align_video_frame_count(frames, 4));
+ return frames;
+ }
+}
+
+#endif // __SD_MODEL_VAE_GEOMETRY_HPP__
diff --git a/src/model/vae/hunyuan_vae.hpp b/src/model/vae/hunyuan_vae.hpp
index 567a704..23428ce 100644
--- a/src/model/vae/hunyuan_vae.hpp
+++ b/src/model/vae/hunyuan_vae.hpp
@@ -787,7 +787,9 @@ namespace Hunyuan {
}
sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output,
- std::shared_ptr<RNG> rng) override {
+ std::shared_ptr<RNG> rng,
+ VAEPosterior,
+ bool) override {
SD_UNUSED(rng);
return vae_output;
}
diff --git a/src/model/vae/ltx_vae.hpp b/src/model/vae/ltx_vae.hpp
index 743549b..ddd6a01 100644
--- a/src/model/vae/ltx_vae.hpp
+++ b/src/model/vae/ltx_vae.hpp
@@ -11,7 +11,7 @@
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
-#include "model/diffusion/ltxv.hpp"
+#include "model/common/ltx.hpp"
#include "model/vae/vae.hpp"
#include "model/vae/wan_vae.hpp"
#include "model_loader.h"
@@ -1054,18 +1054,22 @@ namespace LTXVAE {
int patch_size;
bool decode_only;
+ const bool use_normalized_latents;
+
VideoVAE(int version,
bool decode_only,
bool timestep_conditioning,
int patch_size,
const String2TensorStorage& tensor_storage_map,
const std::string& prefix,
+ bool use_normalized_latents,
float decode_timestep = 0.05f)
: version(version),
decode_timestep(decode_timestep),
timestep_conditioning(timestep_conditioning),
patch_size(patch_size),
- decode_only(decode_only) {
+ decode_only(decode_only),
+ use_normalized_latents(use_normalized_latents) {
if (!decode_only) {
blocks["encoder"] = std::make_shared<Encoder>(version,
tensor_storage_map,
@@ -1084,11 +1088,10 @@ namespace LTXVAE {
ggml_tensor* decode(GGMLRunnerContext* ctx,
ggml_tensor* z,
ggml_tensor* timestep) {
- auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
- auto processor = std::dynamic_pointer_cast<PerChannelStatistics>(blocks["per_channel_statistics"]);
- auto latents = processor->un_normalize(ctx, z);
- auto out = decoder->forward(ctx, latents, timestep);
- out = WAN::WanVAE::unpatchify(ctx, out, patch_size, 1);
+ auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
+ auto latents = use_normalized_latents ? un_normalize_latents(ctx, z) : z;
+ auto out = decoder->forward(ctx, latents, timestep);
+ out = WAN::WanVAE::unpatchify(ctx, out, patch_size, 1);
return out;
}
@@ -1101,9 +1104,8 @@ namespace LTXVAE {
ggml_tensor* timestep,
int temporal_window_size = 1,
int temporal_tile_overlap = 0) {
- auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
- auto processor = std::dynamic_pointer_cast<PerChannelStatistics>(blocks["per_channel_statistics"]);
- auto latents = processor->un_normalize(ctx, z);
+ auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
+ auto latents = use_normalized_latents ? un_normalize_latents(ctx, z) : z;
const int64_t T = z->ne[2];
if (T <= 1) {
@@ -1164,9 +1166,8 @@ namespace LTXVAE {
int chunk_idx,
int temporal_tile_overlap,
int& feat_idx) {
- auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
- auto processor = std::dynamic_pointer_cast<PerChannelStatistics>(blocks["per_channel_statistics"]);
- auto latents = processor->un_normalize(ctx, z);
+ auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
+ auto latents = use_normalized_latents ? un_normalize_latents(ctx, z) : z;
feat_idx = 0;
int chunk_overlap = temporal_tile_overlap; // modified by forward_tiled_frame temporal inflation
@@ -1181,14 +1182,13 @@ namespace LTXVAE {
ggml_tensor* encode(GGMLRunnerContext* ctx,
ggml_tensor* x) {
GGML_ASSERT(!decode_only);
- auto encoder = std::dynamic_pointer_cast<Encoder>(blocks["encoder"]);
- auto processor = std::dynamic_pointer_cast<PerChannelStatistics>(blocks["per_channel_statistics"]);
+ auto encoder = std::dynamic_pointer_cast<Encoder>(blocks["encoder"]);
x = patchify(ctx, x, patch_size);
auto out = encoder->forward(ctx, x);
auto mean = ggml_ext_chunk(ctx, out, 2, 3, false)[0];
mean = ggml_cont(ctx->ggml_ctx, mean);
- return processor->normalize(ctx, mean);
+ return use_normalized_latents ? normalize_latents(ctx, mean) : mean;
}
ggml_tensor* normalize_latents(GGMLRunnerContext* ctx,
@@ -1222,7 +1222,8 @@ struct LTXVideoVAE : public VAE {
const std::string& prefix,
bool decode_only = true,
SDVersion version = VERSION_LTXAV,
- std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+ std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+ bool use_normalized_latents = true)
: decode_only(decode_only),
ltx_vae_version(LTXVAE::detect_ltx_vae_version(tensor_storage_map, prefix)),
timestep_conditioning(LTXVAE::detect_ltx_vae_timestep_conditioning(tensor_storage_map, prefix)),
@@ -1233,7 +1234,8 @@ struct LTXVideoVAE : public VAE {
LTXVAE::detect_ltx_vae_timestep_conditioning(tensor_storage_map, prefix),
patch_size,
tensor_storage_map,
- prefix),
+ prefix,
+ use_normalized_latents),
VAE(version, backend, prefix, weight_manager) {
vae.init(params_ctx, tensor_storage_map, prefix);
decode_timestep_tensor.values()[0] = vae.decode_timestep;
@@ -1434,7 +1436,7 @@ struct LTXVideoVAE : public VAE {
return 256;
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior, bool) override {
SD_UNUSED(rng);
if (vae_output.dim() >= 4 && vae_output.shape()[3] > 128) {
return sd::ops::slice(vae_output, 3, 0, 128);
diff --git a/src/model/vae/mage_vae.hpp b/src/model/vae/mage_vae.hpp
index 838dfea..bcf2450 100644
--- a/src/model/vae/mage_vae.hpp
+++ b/src/model/vae/mage_vae.hpp
@@ -502,9 +502,11 @@ namespace MageVAE {
return LATENT_CHANNELS * 2;
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
- const auto chunks = sd::ops::chunk(vae_output, 2, 2);
- const auto& mean = chunks[0];
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior posterior, bool) override {
+ const auto chunks = sd::ops::chunk(vae_output, 2, 2);
+ const auto& mean = chunks[0];
+ if (posterior == VAEPosterior::Mode)
+ return mean;
const auto& logvar = chunks[1];
sd::Tensor<float> stddev = sd::ops::exp(0.5f * sd::ops::clamp(logvar, -20.0f, 10.0f));
sd::Tensor<float> noise = sd::Tensor<float>::randn_like(mean, rng);
diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp
index 2e3a6e5..25cd5e3 100644
--- a/src/model/vae/minimax_h3_vae.hpp
+++ b/src/model/vae/minimax_h3_vae.hpp
@@ -698,12 +698,16 @@ namespace MiniMaxH3VAE {
}
sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output,
- std::shared_ptr<RNG> rng) override {
- SD_UNUSED(rng);
+ std::shared_ptr<RNG> rng,
+ VAEPosterior posterior,
+ bool use_supplied_rng) override {
if (config.patch_size_t == 1) {
- auto mean = sd::ops::slice(vae_output, 3, 0, 24);
- auto logvar = sd::ops::slice(vae_output, 3, 24, 48);
- auto noise = MT19937RNG(42).randn(static_cast<uint32_t>(mean.numel()));
+ auto mean = sd::ops::slice(vae_output, 3, 0, 24);
+ if (posterior == VAEPosterior::Mode)
+ return mean;
+ auto logvar = sd::ops::slice(vae_output, 3, 24, 48);
+ const auto count = static_cast<uint32_t>(mean.numel());
+ auto noise = use_supplied_rng ? rng->randn(count) : MT19937RNG(42).randn(count);
for (int64_t i = 0; i < mean.numel(); ++i) {
mean[i] += std::exp(0.5f * std::clamp(logvar[i], -30.f, 20.f)) * noise[i];
}
diff --git a/src/model/vae/seedvr2_vae.hpp b/src/model/vae/seedvr2_vae.hpp
index 01666c3..546990a 100644
--- a/src/model/vae/seedvr2_vae.hpp
+++ b/src/model/vae/seedvr2_vae.hpp
@@ -2,6 +2,7 @@
#define __SD_MODEL_VAE_SEEDVR2_VAE_HPP__
#include "model/common/ggml_block.hpp"
+#include "model/vae/vae.hpp"
namespace SeedVR2 {
@@ -259,16 +260,22 @@ namespace SeedVR2 {
}
};
- struct SeedVR2VAERunner : GGMLRunner {
+ struct SeedVR2VAERunner : VAE {
SeedVR2VAEConfig config;
VideoAutoencoderKL model;
- SeedVR2VAERunner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<DeviceResidencyManager> manager)
- : GGMLRunner(backend, manager), config(SeedVR2VAEConfig::detect_from_weights(weights, "vae")), model(config) {
- model.init(params_ctx, weights, "vae");
+ SeedVR2VAERunner(ggml_backend_t backend, const String2TensorStorage& weights, std::shared_ptr<RunnerWeightManager> manager, const std::string& prefix = "vae")
+ : VAE(VERSION_SEEDVR2, backend, prefix, manager),
+ config(SeedVR2VAEConfig::detect_from_weights(weights, prefix)),
+ model(config) {
+ model.init(params_ctx, weights, prefix);
}
std::string get_desc() override { return "SeedVR2 VAE"; }
- void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) { model.get_param_tensors(tensors, "vae"); }
+ void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override { model.get_param_tensors(tensors, weight_prefix); }
+ int get_encoder_output_channels(int) override { return 16; }
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& output, std::shared_ptr<RNG>, VAEPosterior, bool) override { return output; }
+ sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latent) override { return latent * 0.9152f; }
+ sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latent) override { return latent / 0.9152f; }
sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& input, bool decode) {
auto graph = [&]() {
auto gf = new_graph_custom(MAX_GRAPH_SIZE * 4);
@@ -278,7 +285,24 @@ namespace SeedVR2 {
return gf;
};
auto output = GGMLRunner::compute(graph, n_threads);
- return restore_trailing_singleton_dims(output, 4);
+ return restore_trailing_singleton_dims(std::move(output), 4);
+ }
+
+ protected:
+ sd::Tensor<float> _compute(int n_threads, const sd::Tensor<float>& input, bool decode) override {
+ if (input.empty() || (input.dim() != 4 && input.dim() != 5) || input.shape().back() != 1 ||
+ input.shape()[input.dim() - 2] != (decode ? 16 : 3)) {
+ throw std::runtime_error("SeedVR2 VAE requires a single RGB tensor or 16-channel latent");
+ }
+ auto volume = input.dim() == 5 ? input.squeeze(4) : input.squeeze(3).unsqueeze(2);
+ if (!decode && (volume.shape()[2] - 1) % 4)
+ throw std::runtime_error("SeedVR2 VAE requires 4n+1 pixel frames");
+ auto result = compute(n_threads, volume, decode);
+ if (result.empty())
+ return {};
+ if (decode && !std::all_of(result.values().begin(), result.values().end(), [](float value) { return std::isfinite(value); }))
+ throw std::runtime_error("SeedVR2 produced non-finite pixels");
+ return input.dim() == 5 ? result.unsqueeze(4) : result.squeeze(2).unsqueeze(3);
}
};
}
diff --git a/src/model/vae/tae.hpp b/src/model/vae/tae.hpp
index 09f0a1c..d2999cc 100644
--- a/src/model/vae/tae.hpp
+++ b/src/model/vae/tae.hpp
@@ -321,8 +321,7 @@ public:
}
};
-ggml_tensor*
-patchify(GGMLRunnerContext* runner_ctx, ggml_tensor* x, int64_t patch_size, int64_t b = 1) {
+inline ggml_tensor* patchify(GGMLRunnerContext* runner_ctx, ggml_tensor* x, int64_t patch_size, int64_t b = 1) {
auto ctx = runner_ctx->ggml_ctx;
// x: [f, b*c, h*q, w*r]
// return: [f, b*c*r*q, h, w]
@@ -349,7 +348,7 @@ patchify(GGMLRunnerContext* runner_ctx, ggml_tensor* x, int64_t patch_size, int6
return x;
}
-ggml_tensor* unpatchify(GGMLRunnerContext* runner_ctx, ggml_tensor* x, int64_t patch_size, int64_t b = 1) {
+inline ggml_tensor* unpatchify(GGMLRunnerContext* runner_ctx, ggml_tensor* x, int64_t patch_size, int64_t b = 1) {
auto ctx = runner_ctx->ggml_ctx;
// x: [f, b*c*r*q, h, w]
// return: [f, b*c, h*q, w*r]
@@ -779,7 +778,7 @@ struct TinyImageAutoEncoder : public VAE {
taesd.get_param_tensors(tensors, weight_prefix);
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior, bool) override {
SD_UNUSED(rng);
return vae_output;
}
@@ -863,7 +862,7 @@ struct TinyVideoAutoEncoder : public VAE {
taehv.get_param_tensors(tensors, weight_prefix);
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior, bool) override {
SD_UNUSED(rng);
return vae_output;
}
diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp
index f0a51c6..6c72d9b 100644
--- a/src/model/vae/vae.hpp
+++ b/src/model/vae/vae.hpp
@@ -2,6 +2,7 @@
#define __SD_MODEL_VAE_VAE_HPP__
#include "config.h"
+#include "model/vae/geometry.hpp"
#include <cmath>
#include <limits>
@@ -12,6 +13,10 @@
#include "model_manager.h"
#include "runtime/tiling.h"
+enum class VAEPosterior { ModelDefault,
+ Sample,
+ Mode };
+
struct VAE : public GGMLRunner {
protected:
SDVersion version;
@@ -175,26 +180,7 @@ public:
: version(version), weight_prefix(weight_prefix), GGMLRunner(backend, weight_manager) {}
int get_scale_factor() {
- int scale_factor = 8;
- if (version == VERSION_LTXAV) {
- scale_factor = 32;
- } else if (version == VERSION_WAN2_2_TI2V ||
- version == VERSION_QWEN_IMAGE_2_1 ||
- sd_version_is_hunyuan_video(version) ||
- sd_version_is_mage_flow(version) ||
- sd_version_is_minimax_h3(version)) {
- scale_factor = 16;
- } else if (sd_version_uses_flux2_vae(version)) {
- scale_factor = 16;
- } else if (version == VERSION_CHROMA_RADIANCE ||
- version == VERSION_HIDREAM_O1 ||
- sd_version_is_minit2i(version) ||
- sd_version_is_sensenova_u1(version) ||
- sd_version_is_z_image_l2p(version) ||
- version == VERSION_IRIS) {
- scale_factor = 1;
- }
- return scale_factor;
+ return sd::vae::spatial_scale(version);
}
virtual int get_encoder_output_channels(int input_channels) = 0;
@@ -383,14 +369,17 @@ public:
scale_tensor_to_0_1(&output);
}
int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("computing vae decode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
+ LOG_INFO("computing vae decode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
return std::move(output);
}
- virtual sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) = 0;
- virtual sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) = 0;
- virtual sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) = 0;
- virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) = 0;
+ virtual sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output,
+ std::shared_ptr<RNG> rng,
+ VAEPosterior posterior = VAEPosterior::ModelDefault,
+ bool use_supplied_rng = false) = 0;
+ virtual sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) = 0;
+ virtual sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) = 0;
+ virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) = 0;
virtual void set_conv2d_scale(float scale) { SD_UNUSED(scale); };
};
@@ -412,7 +401,7 @@ struct FakeVAE : public VAE {
return z;
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior, bool) override {
SD_UNUSED(rng);
return vae_output;
}
diff --git a/src/model/vae/wan_vae.hpp b/src/model/vae/wan_vae.hpp
index d44fe92..8a61ffe 100644
--- a/src/model/vae/wan_vae.hpp
+++ b/src/model/vae/wan_vae.hpp
@@ -1347,7 +1347,7 @@ namespace WAN {
ae.get_param_tensors(tensors, weight_prefix);
}
- sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
+ sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng, VAEPosterior, bool) override {
SD_UNUSED(rng);
return vae_output;
}
diff --git a/src/model_build_context.cpp b/src/model_build_context.cpp
new file mode 100644
index 0000000..d030c0a
--- /dev/null
+++ b/src/model_build_context.cpp
@@ -0,0 +1,17 @@
+#include "model_build_context.h"
+
+#include "core/util.h"
+
+namespace sd::model_builders {
+ bool ensure_model_backend_pair(SDBackendManager& backends, SDBackendModule module) {
+ if (backends.runtime_backend(module) == nullptr) {
+ LOG_ERROR("failed to initialize %s backend", sd_backend_module_name(module));
+ return false;
+ }
+ if (backends.params_backend(module) == nullptr) {
+ LOG_ERROR("failed to initialize %s params backend", sd_backend_module_name(module));
+ return false;
+ }
+ return true;
+ }
+} // namespace sd::model_builders
diff --git a/src/pipeline/model_builders.h b/src/model_build_context.h
similarity index 55%
rename from src/pipeline/model_builders.h
rename to src/model_build_context.h
index 1f56e4a..df9b992 100644
--- a/src/pipeline/model_builders.h
+++ b/src/model_build_context.h
@@ -1,11 +1,13 @@
-#ifndef __SD_PIPELINE_MODEL_BUILDERS_H__
-#define __SD_PIPELINE_MODEL_BUILDERS_H__
+#ifndef __SD_MODEL_BUILD_CONTEXT_H__
+#define __SD_MODEL_BUILD_CONTEXT_H__
+#include <map>
#include <memory>
+#include <string>
#include <vector>
-#include "config.h"
#include "core/ggml_extend_backend.h"
+#include "core/option_types.h"
#include "model.h"
#include "stable-diffusion.h"
@@ -20,20 +22,32 @@ struct ControlNet;
namespace Wav2Vec2 {
class Wav2Vec2ModelRunner;
}
-struct GenerationExtension;
-struct GenerationExtensionInitContext;
namespace IPAdapter {
struct IPAdapterRunner;
}
namespace sd::model_builders {
+ struct ModelBuildParams {
+ std::string model_args;
+ std::string tokenizer;
+ std::map<std::string, std::string> embeddings;
+ float linear_scale = 0.f;
+ float attn_scale = 0.f;
+ sd_vae_format_t vae_format = SD_VAE_FORMAT_AUTO;
+ bool build_high_noise = false;
+ bool build_ip_adapter = false;
+ };
+
struct Context {
- const sd::ContextConfig& params;
+ ModelBuildParams params;
SDVersion version;
const String2TensorStorage& tensor_storage_map;
SDBackendManager& backends;
std::shared_ptr<DeviceResidencyManager> weight_manager;
+ bool build_conditioner = true;
+ bool build_diffusion = true;
+ bool build_auxiliary = true;
};
struct CoreRunners {
@@ -46,10 +60,11 @@ namespace sd::model_builders {
};
struct VAEOptions {
- bool use_tae = false;
- bool tae_preview_only = false;
- bool use_audio_vae = false;
- bool external_vae_is_invalid = false;
+ bool use_tae = false;
+ bool tae_preview_only = false;
+ bool use_audio_vae = false;
+ bool external_vae_is_invalid = false;
+ bool apply_default_conv_scale = false;
};
struct VAERunners {
@@ -58,14 +73,8 @@ namespace sd::model_builders {
std::shared_ptr<AudioVAERunner> audio;
};
- bool ensure_backend_pair(SDBackendManager& backends, SDBackendModule module);
-
- bool build_core_runners(const Context& ctx, CoreRunners& runners);
- bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners);
- bool build_control_net_runner(const Context& ctx, std::shared_ptr<ControlNet>& runner);
- bool build_extension_runners(const GenerationExtensionInitContext& ctx,
- std::vector<std::shared_ptr<GenerationExtension>>& extensions);
+ bool ensure_model_backend_pair(SDBackendManager& backends, SDBackendModule module);
} // namespace sd::model_builders
-#endif // __SD_PIPELINE_MODEL_BUILDERS_H__
+#endif // __SD_MODEL_BUILD_CONTEXT_H__
diff --git a/src/model_component.h b/src/model_component.h
index b3e2a55..7a6ffb6 100644
--- a/src/model_component.h
+++ b/src/model_component.h
@@ -18,6 +18,7 @@ enum class ModelComponent {
Upscaler,
Detector,
LatentUpsampler,
+ AnimateDiff,
Count,
};
@@ -54,7 +55,9 @@ inline const char* model_component_name(ModelComponent component) {
case ModelComponent::Detector:
return "YOLOv8";
case ModelComponent::LatentUpsampler:
- return "LTX latent upsampler";
+ return "Latent upscaler";
+ case ModelComponent::AnimateDiff:
+ return "AnimateDiff";
case ModelComponent::Count:
break;
}
diff --git a/src/model_definition.h b/src/model_definition.h
index 1a0f2df..5477d41 100644
--- a/src/model_definition.h
+++ b/src/model_definition.h
@@ -4,6 +4,7 @@
#include "core/option_types.h"
#include <limits>
+#include <string_view>
#include "model.h"
#include "stable-diffusion.h"
@@ -36,14 +37,15 @@ struct ModelGeometry {
struct ModelDefinition {
SDVersion version = VERSION_COUNT;
const char* name = nullptr;
+ const char* id = nullptr;
ModelTraits traits;
ModelSamplingPolicy sampling;
ModelGeometry geometry;
- const char* ref_image_preset = "default";
void (*configure_vae)(const sd::model_builders::Context&, const sd::model_builders::VAEOptions&, VAE&) = nullptr;
bool (*build)(const sd::model_builders::Context&, const TokenizerConfig&, sd::model_builders::CoreRunners&) = nullptr;
};
const ModelDefinition* find_model_definition(SDVersion version);
+const ModelDefinition* find_model_definition(std::string_view id);
#endif // __SD_MODEL_DEFINITION_H__
diff --git a/src/model_io/npy_io.cpp b/src/model_io/npy_io.cpp
new file mode 100644
index 0000000..c1220e1
--- /dev/null
+++ b/src/model_io/npy_io.cpp
@@ -0,0 +1,271 @@
+#include "npy_io.h"
+
+#include <algorithm>
+#include <array>
+#include <charconv>
+#include <cstring>
+#include <limits>
+#include <optional>
+#include <string_view>
+
+namespace sd::npy {
+
+ static constexpr uint8_t npy_magic[] = {0x93, 'N', 'U', 'M', 'P', 'Y'};
+ static constexpr struct {
+ std::string_view code;
+ sd_tensor_type_t type;
+ size_t width;
+ } npy_types[] = {
+ {"u1", SD_TENSOR_U8, 1},
+ {"i4", SD_TENSOR_I32, 4},
+ {"i8", SD_TENSOR_I64, 8},
+ {"f4", SD_TENSOR_F32, 4},
+ {"f8", SD_TENSOR_F64, 8},
+ };
+
+ static char native_byte_order() {
+ const uint16_t one = 1;
+ return *reinterpret_cast<const uint8_t*>(&one) ? '<' : '>';
+ }
+
+ struct NpyHeader {
+ std::string_view descriptor;
+ std::vector<int64_t> shape;
+ bool fortran_order;
+ };
+
+ class NpyHeaderParser {
+ std::string_view input_;
+
+ void skip_whitespace() {
+ const auto first = input_.find_first_not_of(" \t\r\n");
+ input_.remove_prefix(first == std::string_view::npos ? input_.size() : first);
+ }
+
+ bool consume(char token) {
+ skip_whitespace();
+ if (input_.empty() || input_.front() != token)
+ return false;
+ input_.remove_prefix(1);
+ return true;
+ }
+
+ void require(char token) {
+ if (!consume(token))
+ throw std::runtime_error("invalid NumPy tensor header");
+ }
+
+ std::string_view read_string() {
+ skip_whitespace();
+ if (input_.empty() || (input_.front() != '\'' && input_.front() != '"'))
+ throw std::runtime_error("expected a string in NumPy tensor header");
+ const auto closing_quote = input_.find(input_.front(), 1);
+ if (closing_quote == std::string_view::npos)
+ throw std::runtime_error("unterminated string in NumPy tensor header");
+ const auto value = input_.substr(1, closing_quote - 1);
+ input_.remove_prefix(closing_quote + 1);
+ return value;
+ }
+
+ bool read_bool() {
+ skip_whitespace();
+ if (input_.substr(0, 4) == "True") {
+ input_.remove_prefix(4);
+ return true;
+ }
+ if (input_.substr(0, 5) == "False") {
+ input_.remove_prefix(5);
+ return false;
+ }
+ throw std::runtime_error("invalid fortran_order in NumPy tensor header");
+ }
+
+ std::vector<int64_t> read_shape() {
+ require('(');
+ std::vector<int64_t> shape;
+ if (consume(')'))
+ return shape;
+ do {
+ skip_whitespace();
+ int64_t dimension = 0;
+ const auto result = std::from_chars(input_.data(), input_.data() + input_.size(), dimension);
+ if (result.ec != std::errc{} || dimension < 0 || shape.size() == 8)
+ throw std::runtime_error("invalid NumPy tensor shape (maximum rank 8)");
+ input_.remove_prefix(result.ptr - input_.data());
+ shape.push_back(dimension);
+ if (!consume(',')) {
+ if (shape.size() == 1)
+ throw std::runtime_error("expected a shape tuple in NumPy tensor header");
+ require(')');
+ break;
+ }
+ } while (!consume(')'));
+ return shape;
+ }
+
+ public:
+ explicit NpyHeaderParser(std::string_view input)
+ : input_(input) {}
+
+ NpyHeader parse() {
+ std::optional<std::string_view> descriptor;
+ std::optional<std::vector<int64_t>> shape;
+ std::optional<bool> fortran_order;
+ require('{');
+ for (int i = 0; i < 3; ++i) {
+ if (i != 0)
+ require(',');
+ const auto key = read_string();
+ require(':');
+ if (key == "descr" && !descriptor)
+ descriptor = read_string();
+ else if (key == "shape" && !shape)
+ shape = read_shape();
+ else if (key == "fortran_order" && !fortran_order)
+ fortran_order = read_bool();
+ else
+ throw std::runtime_error("unexpected or duplicate key in NumPy tensor header");
+ }
+ consume(',');
+ require('}');
+ skip_whitespace();
+ if (!input_.empty())
+ throw std::runtime_error("unexpected data in NumPy tensor header");
+ std::reverse(shape->begin(), shape->end());
+ return {*descriptor, std::move(*shape), *fortran_order};
+ }
+ };
+
+ static void copy_fortran_order(uint8_t* target, const uint8_t* source, const std::vector<int64_t>& shape, size_t bytes, size_t width) {
+ std::array<size_t, 8> strides{};
+ std::array<size_t, 8> indices{};
+ size_t stride = width;
+ for (size_t i = shape.size(); i > 0; --i) {
+ strides[i - 1] = stride;
+ stride *= size_t(shape[i - 1]);
+ }
+
+ // DGL stores axis zero contiguously. Fortran data makes the opposite axis contiguous.
+ size_t offset = 0;
+ for (size_t pos = 0; pos < bytes; pos += width) {
+ std::memcpy(target + pos, source + offset, width);
+ for (size_t axis = 0; axis < shape.size(); ++axis) {
+ offset += strides[axis];
+ if (++indices[axis] < size_t(shape[axis]))
+ break;
+ offset -= indices[axis] * strides[axis];
+ indices[axis] = 0;
+ }
+ }
+ }
+
+ static size_t checked_payload_size(const std::vector<int64_t>& shape, size_t width) {
+ size_t count = 1;
+ for (int64_t dim : shape) {
+ if (count && uint64_t(dim) > std::numeric_limits<size_t>::max() / count)
+ throw std::runtime_error("invalid or overflowing tensor shape");
+ count *= size_t(dim);
+ }
+ if (count > uint64_t(INT64_MAX) || count > (std::numeric_limits<size_t>::max() - 7) / width)
+ throw std::runtime_error("tensor byte size overflow");
+ return count * width;
+ }
+
+ Reader::Reader(const std::string& path, size_t max_bytes)
+ : file_(std::filesystem::u8path(path), std::ios::binary | std::ios::ate) {
+ if (!file_)
+ throw std::runtime_error("cannot open tensor input '" + path + "'");
+ const std::streamoff file_size = file_.tellg();
+ file_.seekg(0);
+ uint8_t prefix[8];
+ if (!file_.read(reinterpret_cast<char*>(prefix), sizeof(prefix)) ||
+ !std::equal(std::begin(npy_magic), std::end(npy_magic), prefix))
+ throw std::runtime_error("tensor input is not a NumPy .npy file");
+ if (prefix[6] < 1 || prefix[6] > 3 || prefix[7] != 0)
+ throw std::runtime_error("unsupported NumPy tensor format version");
+ const size_t length_size = prefix[6] == 1 ? 2 : 4;
+ uint8_t length[4]{};
+ if (!file_.read(reinterpret_cast<char*>(length), length_size))
+ throw std::runtime_error("cannot read NumPy tensor header length");
+ const uint32_t header_size = uint32_t(length[0]) | (uint32_t(length[1]) << 8) |
+ (uint32_t(length[2]) << 16) | (uint32_t(length[3]) << 24);
+ if (header_size == 0 || header_size > 64 * 1024)
+ throw std::runtime_error("invalid NumPy tensor header length");
+ std::string header_text(header_size, '\0');
+ if (!file_.read(header_text.data(), header_text.size()) || header_text.back() != '\n')
+ throw std::runtime_error("cannot read NumPy tensor header");
+ auto header = NpyHeaderParser(header_text).parse();
+
+ const char order = header.descriptor.empty() ? '\0' : header.descriptor.front();
+ const auto code = header.descriptor.empty() ? header.descriptor : header.descriptor.substr(1);
+ const auto dtype = std::find_if(std::begin(npy_types), std::end(npy_types), [&](const auto& entry) {
+ return entry.code == code;
+ });
+ if (dtype == std::end(npy_types) || (order != '<' && order != '>' && order != '=' && !(order == '|' && dtype->width == 1)))
+ throw std::runtime_error("unsupported NumPy tensor dtype '" + std::string(header.descriptor) + "'");
+
+ const auto bytes = checked_payload_size(header.shape, dtype->width);
+ if (bytes > max_bytes)
+ throw std::runtime_error("NumPy tensor exceeds the byte limit (" + std::to_string(max_bytes) + " bytes)");
+ const auto payload = std::streamoff(sizeof(prefix) + length_size + header_size);
+ if (file_size < payload || uint64_t(file_size - payload) != bytes)
+ throw std::runtime_error("NumPy tensor payload size does not match its shape");
+
+ shape_ = std::move(header.shape);
+ type_ = dtype->type;
+ bytes_ = bytes;
+ width_ = dtype->width;
+ fortran_order_ = header.fortran_order;
+ swap_bytes_ = width_ > 1 && order != '=' && order != native_byte_order();
+ }
+
+ void Reader::read(void* buffer, size_t bytes) {
+ if (bytes != bytes_ || (bytes && !buffer))
+ throw std::runtime_error("NumPy tensor destination size does not match its shape");
+ if (!bytes_)
+ return;
+ auto* target = static_cast<uint8_t*>(buffer);
+ if (fortran_order_ && shape_.size() > 1) {
+ std::vector<uint8_t> source(bytes_);
+ if (!file_.read(reinterpret_cast<char*>(source.data()), bytes_))
+ throw std::runtime_error("cannot read NumPy tensor payload");
+ copy_fortran_order(target, source.data(), shape_, bytes_, width_);
+ } else if (!file_.read(reinterpret_cast<char*>(target), bytes_)) {
+ throw std::runtime_error("cannot read NumPy tensor payload");
+ }
+ if (swap_bytes_) {
+ for (size_t offset = 0; offset < bytes_; offset += width_)
+ std::reverse(target + offset, target + offset + width_);
+ }
+ }
+
+ void write(const std::filesystem::path& path, sd_tensor_type_t type, const std::vector<int64_t>& shape, const void* data, size_t bytes) {
+ const auto dtype = std::find_if(std::begin(npy_types), std::end(npy_types), [&](const auto& entry) {
+ return entry.type == type;
+ });
+ if (dtype == std::end(npy_types))
+ throw std::runtime_error("invalid tensor type");
+ std::string descriptor(1, dtype->width == 1 ? '|' : native_byte_order());
+ descriptor += dtype->code;
+ std::string header = "{'descr': '" + descriptor + "', 'fortran_order': False, 'shape': (";
+ for (auto it = shape.rbegin(); it != shape.rend(); ++it)
+ header += std::to_string(*it) + ", ";
+ header += "), }";
+ header.append((64 - (10 + header.size() + 1) % 64) % 64, ' ');
+ header += '\n';
+
+ std::ofstream file(path, std::ios::binary);
+ if (!file)
+ throw std::runtime_error("cannot open tensor output '" + path.u8string() + "'");
+ const uint8_t format[] = {1, 0, uint8_t(header.size()), uint8_t(header.size() >> 8)};
+ file.write(reinterpret_cast<const char*>(npy_magic), sizeof(npy_magic));
+ file.write(reinterpret_cast<const char*>(format), sizeof(format));
+ file.write(header.data(), header.size());
+ if (bytes)
+ file.write(static_cast<const char*>(data), bytes);
+ file.close();
+ if (!file)
+ throw std::runtime_error("cannot write tensor output '" + path.u8string() + "'");
+ }
+
+} // namespace sd::npy
diff --git a/src/model_io/npy_io.h b/src/model_io/npy_io.h
new file mode 100644
index 0000000..d32a3bf
--- /dev/null
+++ b/src/model_io/npy_io.h
@@ -0,0 +1,34 @@
+#ifndef __SD_MODEL_IO_NPY_IO_H__
+#define __SD_MODEL_IO_NPY_IO_H__
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "stable-diffusion.h"
+
+namespace sd::npy {
+
+ class Reader {
+ std::ifstream file_;
+ std::vector<int64_t> shape_;
+ sd_tensor_type_t type_;
+ size_t bytes_;
+ size_t width_;
+ bool fortran_order_;
+ bool swap_bytes_;
+
+ public:
+ Reader(const std::string& path, size_t max_bytes);
+
+ const std::vector<int64_t>& shape() const { return shape_; }
+ sd_tensor_type_t type() const { return type_; }
+ void read(void* buffer, size_t bytes);
+ };
+
+ void write(const std::filesystem::path& path, sd_tensor_type_t type, const std::vector<int64_t>& shape, const void* data, size_t bytes);
+
+} // namespace sd::npy
+
+#endif // __SD_MODEL_IO_NPY_IO_H__
diff --git a/src/model_loader.cpp b/src/model_loader.cpp
index 9e88711..3dbdf5f 100644
--- a/src/model_loader.cpp
+++ b/src/model_loader.cpp
@@ -16,6 +16,7 @@
#include <unordered_map>
#include <vector>
+#include "core/file_access.h"
#include "core/util.h"
#include "model_io/gguf_io.h"
#include "model_io/safetensors_io.h"
@@ -191,7 +192,7 @@ void ModelLoader::add_tensor_storage(const TensorStorage& tensor_storage) {
void ModelLoader::set_n_threads(int n_threads) {
n_threads_ = n_threads > 0 ? n_threads : sd_get_num_physical_cores();
- LOG_VERBOSE("using %d threads for model loading", n_threads_);
+ LOG_DEBUG("using %d threads for model loading", n_threads_);
}
bool ModelLoader::init_from_file(const std::string& file_path, const std::string& prefix) {
@@ -205,31 +206,34 @@ bool ModelLoader::parse_file(const std::string& file_path, const std::string& pr
}
parsed_dependencies_.push_back(stamp);
if (is_directory(file_path)) {
- const std::string diffusers_index_path = path_join(file_path, "model_index.json");
- const std::string diffusers_unet_path = path_join(file_path, "unet/diffusion_pytorch_model.safetensors");
- const bool has_diffusers_layout = file_exists(diffusers_index_path) || file_exists(diffusers_unet_path);
-
+ const std::string diffusers_index_path = path_join(file_path, "model_index.json");
+ const std::string diffusers_unet_path = path_join(file_path, "unet/diffusion_pytorch_model.safetensors");
const std::string safetensors_index_path = path_join(file_path, "model.safetensors.index.json");
- if (!has_diffusers_layout && file_exists(safetensors_index_path)) {
- LOG_INFO("load %s using root safetensors index", file_path.c_str());
+ bool has_index, has_unet, has_shard_index;
+ if (!inspect_optional_file(diffusers_index_path, has_index) ||
+ !inspect_optional_file(diffusers_unet_path, has_unet) ||
+ !inspect_optional_file(safetensors_index_path, has_shard_index))
+ return false;
+ if (!has_index && !has_unet && has_shard_index) {
+ LOG_VERBOSE("parse '%s' using root safetensors index, prefix = '%s'", file_path.c_str(), prefix.c_str());
return parse_file(safetensors_index_path, prefix);
}
- LOG_INFO("load %s using diffusers format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using diffusers format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return init_from_diffusers_file(file_path, prefix);
} else if (is_gguf_file(file_path)) {
- LOG_INFO("load %s using gguf format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using gguf format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return init_from_gguf_file(file_path, prefix);
} else if (ends_with(file_path, ".json")) {
- LOG_INFO("load %s using safetensors index format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using safetensors index format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return init_from_safetensors_index_file(file_path, prefix);
} else if (is_safetensors_file(file_path)) {
- LOG_INFO("load %s using safetensors format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using safetensors format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return init_from_safetensors_file(file_path, prefix);
} else if (is_torch_zip_file(file_path)) {
- LOG_INFO("load %s using torch zip format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using torch zip format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return init_from_torch_zip_file(file_path, prefix);
} else if (init_from_torch_legacy_file(file_path, prefix)) {
- LOG_INFO("load %s using torch legacy format", file_path.c_str());
+ LOG_VERBOSE("parse '%s' using torch legacy format, prefix = '%s'", file_path.c_str(), prefix.c_str());
return true;
} else {
if (file_exists(file_path)) {
@@ -241,7 +245,9 @@ bool ModelLoader::parse_file(const std::string& file_path, const std::string& pr
}
}
-void ModelLoader::convert_tensors_name() {
+void ModelLoader::convert_tensors_name(SDVersion version) {
+ if (version_ == VERSION_COUNT && version != VERSION_COUNT)
+ version_ = version;
if (names_converted_) {
return;
}
@@ -263,8 +269,6 @@ bool ModelLoader::init_from_file_and_convert_name(const std::string& file_path,
/*================================================= GGUFModelLoader ==================================================*/
bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::string& prefix) {
- LOG_VERBOSE("init from '%s'", file_path.c_str());
-
std::vector<TensorStorage> tensor_storages;
std::string error;
if (!read_gguf_file(file_path, tensor_storages, &error)) {
@@ -291,8 +295,6 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s
/*================================================= SafeTensorsModelLoader ==================================================*/
bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const std::string& prefix) {
- LOG_VERBOSE("init from '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
-
std::vector<TensorStorage> tensor_storages;
std::string error;
if (!read_safetensors_file(file_path, tensor_storages, &error, &metadata_, &parsed_tensor_names_[file_path])) {
@@ -317,8 +319,6 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
}
bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path, const std::string& prefix) {
- LOG_VERBOSE("init from safetensors index '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
-
std::vector<std::string> shard_paths;
std::string error;
if (!read_safetensors_index_file(file_path, shard_paths, &error)) {
@@ -332,7 +332,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
return false;
}
parsed_dependencies_.push_back(stamp);
- LOG_INFO("load %s using safetensors format", shard_path.c_str());
+ LOG_VERBOSE("parse '%s' using safetensors format, prefix = '%s'", shard_path.c_str(), prefix.c_str());
if (!init_from_safetensors_file(shard_path, prefix)) {
return false;
}
@@ -363,8 +363,6 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
/*================================================= TorchLegacyModelLoader ==================================================*/
bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, const std::string& prefix) {
- LOG_VERBOSE("init from torch legacy '%s'", file_path.c_str());
-
std::vector<TensorStorage> tensor_storages;
std::string error;
if (!read_torch_legacy_file(file_path, tensor_storages, &error)) {
@@ -391,8 +389,6 @@ bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, cons
/*================================================= TorchZipModelLoader ==================================================*/
bool ModelLoader::init_from_torch_zip_file(const std::string& file_path, const std::string& prefix) {
- LOG_VERBOSE("init from '%s'", file_path.c_str());
-
std::vector<TensorStorage> tensor_storages;
std::string error;
if (!read_torch_zip_file(file_path, tensor_storages, &error)) {
@@ -428,14 +424,12 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
return false;
}
- if (file_exists(vae_path) && !parse_file(vae_path, prefix + "vae.")) {
- return false;
- }
- if (file_exists(clip_path) && !parse_file(clip_path, prefix + "te.")) {
- return false;
- }
- if (file_exists(clip_g_path) && !parse_file(clip_g_path, prefix + "te.1.")) {
- return false;
+ for (const auto& [path, component_prefix] : {std::pair{vae_path, "vae."},
+ {clip_path, "te."},
+ {clip_g_path, "te.1."}}) {
+ bool exists;
+ if (!inspect_optional_file(path, exists) || (exists && !parse_file(path, prefix + component_prefix)))
+ return false;
}
return true;
}
@@ -443,6 +437,27 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
SDVersion ModelLoader::get_sd_version() const {
TensorStorage token_embedding_weight, input_block_weight, context_ebedding_weight;
+ for (const std::string prefix : {"", "model.diffusion_model.", "diffusion_model.", "transformer.", "unet.", "diffusion.", "wan."}) {
+ if ((tensor_storage_map.count(prefix + "positive_conditioning") || tensor_storage_map.count(prefix + "prompt_embeds")) &&
+ (tensor_storage_map.count(prefix + "lq_proj_in.conv1.weight") || tensor_storage_map.count(prefix + "lq_proj.conv1.weight")))
+ return VERSION_RELAY_VSR;
+ if (prefix != "wan." && tensor_storage_map.count(prefix + "vid_in.proj.weight") &&
+ tensor_storage_map.count(prefix + "blocks.0.attn.rope.rope.freqs") &&
+ tensor_storage_map.count(prefix + "positive_conditioning"))
+ return VERSION_SEEDVR2;
+ if (tensor_storage_map.count(prefix + "model.x_embedder.proj1.weight") &&
+ tensor_storage_map.count(prefix + "model.language_model.layers.0.self_attn.q_proj.weight"))
+ return VERSION_HIDREAM_O1;
+ }
+ for (const std::string prefix : {"", "first_stage_model.", "vae.", "flash."}) {
+ if (tensor_storage_map.count(prefix + "condition_encoder.projection.weight") &&
+ tensor_storage_map.count(prefix + "residual_projection.weight"))
+ return VERSION_RELAY_VSR;
+ if (prefix != "flash." && tensor_storage_map.count(prefix + "encoder.conv_out.bias") &&
+ tensor_storage_map.count(prefix + "decoder.up_blocks.0.upsamplers.0.upscale_conv.weight"))
+ return VERSION_SEEDVR2;
+ }
+
if (tensor_storage_map.count("model.diffusion_model.perceiver.latent_queries")) {
return VERSION_FIBO2;
}
@@ -499,10 +514,6 @@ SDVersion ModelLoader::get_sd_version() const {
if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {
return VERSION_SD3;
}
- if (tensor_storage.name.find("model.x_embedder.proj1.weight") != std::string::npos &&
- tensor_storage_map.find("model.language_model.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {
- return VERSION_HIDREAM_O1;
- }
if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.attn.norm_added_q.weight") != std::string::npos &&
tensor_storage_map.find("model.diffusion_model.transformer_blocks.0.img_mlp.w1.weight") != tensor_storage_map.end()) {
return VERSION_LENS;
@@ -548,7 +559,8 @@ SDVersion ModelLoader::get_sd_version() const {
return VERSION_LLADA_IMAGE;
}
if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {
- if (tensor_storage_map.find("text_encoders.llm.connector.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {
+ if (tensor_storage_map.count("text_encoders.llm.connector.layers.0.self_attn.q_proj.weight") ||
+ tensor_storage_map.count("text_encoders.connector.layers.0.self_attn.q_proj.weight")) {
return VERSION_MING_IMAGE;
}
if (tensor_storage_map.find("model.diffusion_model.local_decoder.out_conv.weight") != tensor_storage_map.end()) {
@@ -928,6 +940,7 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
fdata.tensors = std::move(file_tensors);
if (enable_mmap && !is_zip) {
+ sd::require_file_access(file_path, SD_DGL_FILE_READ_MODEL);
LOG_VERBOSE("using mmap for I/O");
std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);
if (mmapped) {
@@ -1163,11 +1176,12 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
if (tensors_to_process.empty()) {
continue;
}
+ sd::require_file_access(file_path, SD_DGL_FILE_READ_MODEL);
if (log_progress) {
- LOG_VERBOSE("loading %zu/%zu tensors from %s",
- tensors_to_process.size(),
- file_tensors.size(),
- file_path.c_str());
+ LOG_INFO("loading %zu/%zu tensors from %s",
+ tensors_to_process.size(),
+ file_tensors.size(),
+ file_path.c_str());
}
bool is_zip = fdata.is_zip;
diff --git a/src/model_loader.h b/src/model_loader.h
index aae3590..d644977 100644
--- a/src/model_loader.h
+++ b/src/model_loader.h
@@ -45,13 +45,17 @@ public:
enum class FileScope { Catalog,
Isolated };
-private:
struct FileStamp {
std::string path;
uintmax_t size = 0;
std::filesystem::file_time_type modified;
+ bool exists = true;
};
+ using Dependencies = std::vector<FileStamp>;
+ static bool read_file_stamp(const std::string& path, FileStamp& stamp, bool allow_missing = false);
+
+private:
struct FileRecord {
FileId id = 0;
uint64_t revision = 0;
@@ -71,7 +75,7 @@ private:
std::vector<FileStamp> parsed_dependencies_;
std::map<std::string, std::set<std::string>> parsed_tensor_names_;
- static bool read_file_stamp(const std::string& path, FileStamp& stamp);
+ bool inspect_optional_file(const std::string& path, bool& exists);
static bool file_unchanged(const FileStamp& stamp);
bool parse_file(const std::string& path, const std::string& prefix);
bool add_file_impl(const std::string& path, const std::string& prefix, FileId* id, bool force, FileScope scope);
@@ -102,8 +106,12 @@ public:
ModelLoader();
bool add_file(const std::string& path, const std::string& prefix = "", FileId* id = nullptr, bool force = false, FileScope scope = FileScope::Catalog);
+ bool set_file_prefix(FileId id, const std::string& prefix);
+ Dependencies dependencies(bool include_isolated = true) const;
+ static bool dependencies_changed(const Dependencies& dependencies, bool& changed);
bool del_file(FileId id);
uint64_t file_revision(FileId id) const;
+ bool same_weight_file(FileId id, size_t index, const ModelLoader& other, FileId other_id, size_t other_index) const;
std::string file_path(FileId id) const;
std::map<std::string, std::string> file_metadata(FileId id) const;
String2TensorStorage file_tensors(FileId id) const;
@@ -115,7 +123,7 @@ public:
uint64_t revision() const { return revision_; }
FileVersions file_versions(const std::vector<std::string>& prefixes = {}) const;
bool init_from_file(const std::string& file_path, const std::string& prefix = "");
- void convert_tensors_name();
+ void convert_tensors_name(SDVersion version = VERSION_COUNT);
bool init_from_file_and_convert_name(const std::string& file_path,
const std::string& prefix = "",
SDVersion version = VERSION_COUNT);
diff --git a/src/model_loader_files.cpp b/src/model_loader_files.cpp
index 9ac5456..30bc4d4 100644
--- a/src/model_loader_files.cpp
+++ b/src/model_loader_files.cpp
@@ -4,6 +4,7 @@
#include <atomic>
#include <limits>
+#include "core/file_access.h"
#include "core/util.h"
#include "name_conversion.h"
@@ -12,12 +13,18 @@ static uint64_t next_source_revision() {
return revision.fetch_add(1, std::memory_order_relaxed) + 1;
}
-bool ModelLoader::read_file_stamp(const std::string& path, FileStamp& stamp) {
+bool ModelLoader::read_file_stamp(const std::string& path, FileStamp& stamp, bool allow_missing) {
+ sd::require_file_access(path, SD_DGL_FILE_READ_MODEL);
std::error_code error;
const auto file_path = std::filesystem::u8path(path);
stamp.path = path;
stamp.size = 0;
- stamp.modified = std::filesystem::last_write_time(file_path, error);
+ stamp.exists = std::filesystem::exists(file_path, error);
+ if (!error && !stamp.exists && allow_missing) {
+ stamp.modified = {};
+ return true;
+ }
+ stamp.modified = std::filesystem::last_write_time(file_path, error);
if (!error && std::filesystem::is_regular_file(file_path, error)) {
stamp.size = std::filesystem::file_size(file_path, error);
}
@@ -29,29 +36,52 @@ bool ModelLoader::read_file_stamp(const std::string& path, FileStamp& stamp) {
}
bool ModelLoader::file_unchanged(const FileStamp& stamp) {
- std::error_code error;
- if (!std::filesystem::exists(std::filesystem::u8path(stamp.path), error)) {
- return false;
- }
FileStamp current;
- return read_file_stamp(stamp.path, current) &&
+ return read_file_stamp(stamp.path, current, true) && current.exists == stamp.exists &&
current.size == stamp.size && current.modified == stamp.modified;
}
+bool ModelLoader::inspect_optional_file(const std::string& path, bool& exists) {
+ FileStamp stamp;
+ if (!read_file_stamp(path, stamp, true))
+ return false;
+ exists = stamp.exists;
+ parsed_dependencies_.push_back(std::move(stamp));
+ return true;
+}
+
void ModelLoader::invalidate_file_data() {
file_data.clear();
model_files_processed = false;
}
+static String2TensorStorage prefixed_tensors(const String2TensorStorage& source, const std::string& prefix) {
+ String2TensorStorage result;
+ std::map<std::string, std::string> original_names;
+ for (const auto& [name, storage] : source) {
+ auto tensor = storage;
+ if (!starts_with(tensor.name, prefix))
+ tensor.name = prefix + tensor.name;
+ auto inserted = original_names.emplace(tensor.name, name);
+ if (!inserted.second)
+ throw std::runtime_error("model tensors '" + inserted.first->second + "' and '" + name +
+ "' collide after prefix '" + prefix + "': '" + tensor.name + "'");
+ result.emplace(tensor.name, std::move(tensor));
+ }
+ return result;
+}
+
void ModelLoader::rebuild_catalog() {
tensor_storage_map.clear();
metadata_.clear();
for (const auto& file : files_) {
if (file.scope == FileScope::Isolated)
continue;
- for (const auto& entry : file.tensors) {
- tensor_storage_map[entry.first] = entry.second;
- }
+ if (!names_converted_ || version_ == VERSION_COUNT)
+ for (auto& entry : prefixed_tensors(file.tensors, file.prefix)) {
+ auto tensor = std::move(entry.second);
+ tensor_storage_map[tensor.name] = std::move(tensor);
+ }
for (const auto& entry : file.metadata) {
metadata_[entry.first] = entry.second;
}
@@ -62,8 +92,8 @@ void ModelLoader::rebuild_catalog() {
for (const auto& file : files_) {
if (file.scope == FileScope::Isolated)
continue;
- for (const auto& entry : file.tensors) {
- TensorStorage tensor = entry.second;
+ for (auto& entry : prefixed_tensors(file.tensors, file.prefix)) {
+ TensorStorage tensor = std::move(entry.second);
tensor.name = convert_tensor_name(tensor.name, version);
tensor_storage_map[tensor.name] = std::move(tensor);
}
@@ -101,9 +131,11 @@ bool ModelLoader::add_file_impl(const std::string& path, const std::string& pref
ModelLoader parsed;
try {
- if (!parsed.parse_file(root.path, prefix)) {
+ if (!parsed.parse_file(root.path, "")) {
return false;
}
+ } catch (const sd::FileAccessDenied&) {
+ throw;
} catch (const std::exception& error) {
LOG_ERROR("invalid model source '%s': %s", path.c_str(), error.what());
return false;
@@ -192,6 +224,15 @@ bool ModelLoader::add_file_impl(const std::string& path, const std::string& pref
}
bool ModelLoader::add_file(const std::string& path, const std::string& prefix, FileId* id, bool force, FileScope scope) {
+ const auto existing = std::find_if(files_.begin(), files_.end(), [&](const FileRecord& file) {
+ return file.path == path && file.prefix == prefix && file.scope == scope;
+ });
+ if (existing != files_.end() && !force &&
+ std::all_of(existing->dependencies.begin(), existing->dependencies.end(), file_unchanged)) {
+ if (id)
+ *id = existing->id;
+ return true;
+ }
ModelLoader candidate = *this;
FileId added_id = 0;
if (!candidate.add_file_impl(path, prefix, &added_id, force, scope)) {
@@ -215,28 +256,46 @@ bool ModelLoader::del_file(FileId id) {
return true;
}
-bool ModelLoader::files_changed(bool& changed, bool include_isolated) const {
- changed = false;
+bool ModelLoader::set_file_prefix(FileId id, const std::string& prefix) {
+ auto file = std::find_if(files_.begin(), files_.end(), [id](const FileRecord& file) { return file.id == id; });
+ if (file == files_.end())
+ return false;
+ if (file->prefix != prefix) {
+ prefixed_tensors(file->tensors, prefix);
+ file->prefix = prefix;
+ ++revision_;
+ invalidate_file_data();
+ rebuild_catalog();
+ }
+ return true;
+}
+
+ModelLoader::Dependencies ModelLoader::dependencies(bool include_isolated) const {
+ Dependencies result;
for (const auto& file : files_) {
if (!include_isolated && file.scope == FileScope::Isolated)
continue;
- for (const auto& stamp : file.dependencies) {
- std::error_code error;
- if (!std::filesystem::exists(std::filesystem::u8path(stamp.path), error) && !error) {
- // An updated index may no longer reference this dependency.
- changed = true;
- continue;
- }
- FileStamp current;
- if (!read_file_stamp(stamp.path, current)) {
- return false;
- }
- changed |= current.size != stamp.size || current.modified != stamp.modified;
+ result.insert(result.end(), file.dependencies.begin(), file.dependencies.end());
+ }
+ return result;
+}
+
+bool ModelLoader::dependencies_changed(const Dependencies& dependencies, bool& changed) {
+ changed = false;
+ for (const auto& stamp : dependencies) {
+ FileStamp current;
+ if (!read_file_stamp(stamp.path, current, true)) {
+ return false;
}
+ changed |= current.exists != stamp.exists || current.size != stamp.size || current.modified != stamp.modified;
}
return true;
}
+bool ModelLoader::files_changed(bool& changed, bool include_isolated) const {
+ return dependencies_changed(dependencies(include_isolated), changed);
+}
+
bool ModelLoader::refresh_files(bool include_isolated) {
bool changed;
if (!files_changed(changed, include_isolated)) {
@@ -299,6 +358,25 @@ uint64_t ModelLoader::file_revision(FileId id) const {
return 0;
}
+bool ModelLoader::same_weight_file(FileId id, size_t index, const ModelLoader& other, FileId other_id, size_t other_index) const {
+ if (index >= file_paths_.size() || other_index >= other.file_paths_.size() ||
+ file_paths_[index] != other.file_paths_[other_index])
+ return false;
+ const auto find = [](const auto& files, FileId id) {
+ return std::find_if(files.begin(), files.end(), [id](const auto& file) { return file.id == id; });
+ };
+ const auto a = find(files_, id);
+ const auto b = find(other.files_, other_id);
+ if (a == files_.end() || b == other.files_.end())
+ return false;
+ return a->revision == b->revision ||
+ (a->path == b->path && a->prefix == b->prefix &&
+ std::equal(a->dependencies.begin(), a->dependencies.end(), b->dependencies.begin(), b->dependencies.end(),
+ [](const FileStamp& x, const FileStamp& y) {
+ return x.path == y.path && x.exists == y.exists && x.size == y.size && x.modified == y.modified;
+ }));
+}
+
std::string ModelLoader::file_path(FileId id) const {
for (const auto& file : files_) {
if (file.id == id)
@@ -336,7 +414,7 @@ std::map<std::string, std::string> ModelLoader::file_metadata(FileId id) const {
String2TensorStorage ModelLoader::file_tensors(FileId id) const {
for (const auto& file : files_) {
if (file.id == id) {
- return file.tensors;
+ return prefixed_tensors(file.tensors, file.prefix);
}
}
return {};
diff --git a/src/model_manager.cpp b/src/model_manager.cpp
index bfdf262..52bc14c 100644
--- a/src/model_manager.cpp
+++ b/src/model_manager.cpp
@@ -1,5 +1,7 @@
#include "model_manager.h"
+#include "core/backend_fit.h"
+
#include <algorithm>
#include <cstdint>
#include <iterator>
@@ -106,12 +108,66 @@ ModelManager::~ModelManager() {
release_prefetch();
}
+ModelManager::ResourceId ModelManager::create_resource() {
+ const auto id = next_resource_id_++;
+ resources_.try_emplace(id).first->second.id = id;
+ return id;
+}
+
+bool ModelManager::remove_resource(ResourceId id) {
+ auto found = resources_.find(id);
+ if (found == resources_.end() || found->second.applying_loras_)
+ return false;
+ std::unordered_set<TensorState*> states;
+ for (const auto& state : tensor_states_) {
+ if (state->resource == &found->second)
+ states.insert(state.get());
+ }
+ if (!unregister_tensor_states(states, nullptr))
+ return false;
+ resources_.erase(found);
+ return true;
+}
+
void ModelManager::set_common_ignore_tensors(std::set<std::string> ignore_tensors) {
common_ignore_tensors_ = std::move(ignore_tensors);
}
bool ModelManager::prepare_lora_sources(std::vector<LoraSpec>& loras) {
- ModelLoader candidate = model_loader_;
+ return prepare_lora_sources(default_resource_, loras);
+}
+
+bool ModelManager::set_loras(std::vector<LoraSpec> loras, SDVersion version, std::set<std::string> targets) {
+ return set_loras(default_resource_, std::move(loras), version, std::move(targets));
+}
+
+std::set<std::string> ModelManager::tensor_names() const {
+ return tensor_names(default_resource_);
+}
+
+void ModelManager::set_split_buffer_type(ggml_backend_t backend, ggml_backend_buffer_type_t type, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits) {
+ set_split_buffer_type(default_resource_, backend, type, device_limits);
+}
+
+bool ModelManager::register_param_tensors(ModelComponent component, std::map<std::string, ggml_tensor*> tensors, ResidencyMode mode, ggml_backend_t compute_backend, ggml_backend_t params_backend, size_t* size, bool allow_split_buffer, bool follow_compute, const std::map<ggml_tensor*, enum ggml_op>* tensor_ops, ModelLoader::FileId source_file, SDVersion source_version) {
+ return register_param_tensors(default_resource_, component, std::move(tensors), mode, compute_backend, params_backend,
+ size, allow_split_buffer, follow_compute, tensor_ops, source_file, source_version);
+}
+
+bool ModelManager::unregister_param_tensors(ModelComponent component, size_t* size) {
+ return unregister_param_tensors(default_resource_, component, size);
+}
+
+bool ModelManager::load_all_params_eagerly() {
+ return load_all_params_eagerly(default_resource_);
+}
+
+bool ModelManager::validate_registered_tensors() {
+ return validate_registered_tensors(default_resource_);
+}
+
+bool ModelManager::prepare_lora_sources(Resource& resource, std::vector<LoraSpec>& loras) {
+ ModelLoader candidate = resource.model_loader_;
std::vector<LoraSpec> resolved;
std::set<ModelLoader::FileId> sources;
for (auto spec : loras) {
@@ -126,45 +182,50 @@ bool ModelManager::prepare_lora_sources(std::vector<LoraSpec>& loras) {
sources.insert(spec.file_id);
resolved.push_back(std::move(spec));
}
- for (auto id : lora_sources_) {
+ for (auto id : resource.lora_sources_) {
if (sources.count(id) == 0)
candidate.del_file(id);
}
- if (!set_loader(std::move(candidate)))
+ if (!set_loader(resource, std::move(candidate)))
return false;
- lora_sources_ = std::move(sources);
- loras = std::move(resolved);
+ resource.lora_sources_ = std::move(sources);
+ loras = std::move(resolved);
return true;
}
-bool ModelManager::set_loras(std::vector<LoraSpec> loras, SDVersion version) {
+bool ModelManager::set_loras(Resource& resource, std::vector<LoraSpec> loras, SDVersion version, std::set<std::string> targets) {
if (std::any_of(loras.begin(), loras.end(), [](const LoraSpec& spec) { return spec.file_id == 0; }) &&
- !prepare_lora_sources(loras))
+ !prepare_lora_sources(resource, loras))
return false;
for (auto& spec : loras) {
- spec.file_revision = model_loader_.file_revision(spec.file_id);
+ spec.file_revision = resource.model_loader_.file_revision(spec.file_id);
if (spec.file_revision == 0)
return false;
}
- if (lora_version_ == version && lora_specs_equal(loras_, loras))
+ if (resource.lora_version_ == version && lora_specs_equal(resource.loras_, loras) &&
+ resource.lora_targets_ == targets) {
+ resource.loras_ = std::move(loras);
return true;
- if (!runtime_owners_.empty() || std::any_of(tensor_states_.begin(), tensor_states_.end(), [](const auto& state) {
- return state->pin_count != 0;
+ }
+ if (!runtime_owners_.empty() || std::any_of(tensor_states_.begin(), tensor_states_.end(), [&](const auto& state) {
+ return state->resource == &resource && state->pin_count != 0;
})) {
LOG_ERROR("cannot change LoRA configuration during execution");
return false;
}
- loras_ = std::move(loras);
- lora_version_ = version;
- current_lora_epoch_++;
- reset_lora_applied_params();
+ resource.loras_ = std::move(loras);
+ resource.lora_targets_ = std::move(targets);
+ resource.lora_version_ = version;
+ resource.current_lora_epoch_++;
+ reset_lora_applied_params(resource);
return true;
}
-std::set<std::string> ModelManager::tensor_names() const {
+std::set<std::string> ModelManager::tensor_names(const Resource& resource) const {
std::set<std::string> names;
for (const auto& state : tensor_states_) {
- if (state != nullptr && state->component != ModelComponent::LoRA && model_sources_.count(state->source_file) == 0) {
+ if (state->resource == &resource && state->component != ModelComponent::LoRA &&
+ (!resource.shared_weights_key_.empty() || resource.model_sources_.count(state->source_file) == 0)) {
names.insert(state->name);
}
}
@@ -185,16 +246,17 @@ size_t estimate_tensors_size(const std::map<std::string, ggml_tensor*>& tensors)
return size;
}
-void ModelManager::set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits) {
+void ModelManager::set_split_buffer_type(Resource& resource, ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits) {
if (compute_backend == nullptr) {
return;
}
if (split_buft == nullptr) {
- split_buffer_types_.erase(compute_backend);
+ resource.split_buffer_types_.erase(compute_backend);
return;
}
- split_buffer_types_[compute_backend] = split_buft;
- split_buffer_devices_[split_buft] = device_limits;
+ resource.split_buffer_types_[compute_backend] = split_buft;
+ resource.split_buffer_limits_[split_buft] = device_limits;
+ split_buffer_devices_[split_buft] = device_limits;
}
bool ModelManager::tensor_shape_supports_split_buffer(const ggml_tensor* tensor) {
@@ -213,17 +275,7 @@ ggml_backend_buffer_type_t ModelManager::split_buffer_type_for(const TensorState
return state.split_buffer_type;
}
-bool ModelManager::register_param_tensors(ModelComponent component,
- std::map<std::string, ggml_tensor*> tensors,
- ResidencyMode residency_mode,
- ggml_backend_t compute_backend,
- ggml_backend_t params_backend,
- size_t* registered_tensor_size,
- bool allow_split_buffer,
- bool params_follow_compute_backend,
- const std::map<ggml_tensor*, enum ggml_op>* tensor_ops,
- ModelLoader::FileId source_file,
- SDVersion source_version) {
+bool ModelManager::register_param_tensors(Resource& resource, ModelComponent component, std::map<std::string, ggml_tensor*> tensors, ResidencyMode residency_mode, ggml_backend_t compute_backend, ggml_backend_t params_backend, size_t* registered_tensor_size, bool allow_split_buffer, bool params_follow_compute_backend, const std::map<ggml_tensor*, enum ggml_op>* tensor_ops, ModelLoader::FileId source_file, SDVersion source_version) {
if (component == ModelComponent::Count) {
LOG_ERROR("model manager tensor component is invalid");
return false;
@@ -232,8 +284,8 @@ bool ModelManager::register_param_tensors(ModelComponent component,
*registered_tensor_size += estimate_tensors_size(tensors);
}
- const auto scoped_sources = source_file != 0 ? source_loader(source_file).file_tensors(source_file, source_version) : String2TensorStorage{};
- const auto& sources = source_file != 0 ? scoped_sources : model_loader_.get_tensor_storage_map();
+ const auto scoped_sources = source_file != 0 ? source_loader(resource, source_file).file_tensors(source_file, source_version) : String2TensorStorage{};
+ const auto& sources = source_file != 0 ? scoped_sources : resource.model_loader_.get_tensor_storage_map();
std::unordered_set<ggml_tensor*> new_tensors;
std::vector<std::unique_ptr<TensorState>> new_states;
new_states.reserve(tensors.size());
@@ -251,6 +303,7 @@ bool ModelManager::register_param_tensors(ModelComponent component,
ggml_set_name(tensor, name.c_str());
auto state = std::make_unique<TensorState>();
+ state->resource = &resource;
state->name = name;
state->tensor = tensor;
state->component = component;
@@ -260,8 +313,8 @@ bool ModelManager::register_param_tensors(ModelComponent component,
state->residency_mode = residency_mode;
state->compute_backend = compute_backend;
state->params_backend = params_backend;
- auto split_buffer = split_buffer_types_.find(compute_backend);
- if (allow_split_buffer && split_buffer != split_buffer_types_.end()) {
+ auto split_buffer = resource.split_buffer_types_.find(compute_backend);
+ if (allow_split_buffer && split_buffer != resource.split_buffer_types_.end()) {
state->split_buffer_type = split_buffer->second;
}
state->params_follow_compute_backend = params_follow_compute_backend;
@@ -283,27 +336,37 @@ bool ModelManager::register_param_tensors(ModelComponent component,
return true;
}
-bool ModelManager::unregister_param_tensors(ModelComponent component, size_t* registered_tensor_size) {
+bool ModelManager::unregister_param_tensors(Resource& resource, ModelComponent component, size_t* registered_tensor_size) {
std::unordered_set<TensorState*> states;
for (auto& state : tensor_states_) {
- if (state->component == component)
+ if (state->resource == &resource && state->component == component)
states.insert(state.get());
}
return unregister_tensor_states(states, registered_tensor_size);
}
bool ModelManager::unregister_param_tensors(const std::vector<ggml_tensor*>& tensors) {
+ return unregister_param_tensors(tensors, nullptr);
+}
+
+bool ModelManager::unregister_param_tensors(const std::vector<ggml_tensor*>& tensors, const Resource* resource) {
std::unordered_set<TensorState*> states;
for (auto tensor : tensors) {
auto found = tensor_states_by_tensor_.find(tensor);
- if (found != tensor_states_by_tensor_.end())
+ if (found != tensor_states_by_tensor_.end()) {
+ if (resource && found->second->resource != resource) {
+ LOG_ERROR("cannot unregister a tensor from a different model resource");
+ return false;
+ }
states.insert(found->second);
+ }
}
return unregister_tensor_states(states, nullptr);
}
bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState*>& target_states,
- size_t* registered_tensor_size) {
+ size_t* registered_tensor_size,
+ bool keep_registration) {
size_t released_size = 0;
for (auto& state : tensor_states_) {
if (state == nullptr || target_states.count(state.get()) == 0) {
@@ -324,9 +387,6 @@ bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState
return true;
}
- clear_all_prefetched_params();
- release_compute_staging_blocks(false, &target_states);
-
std::vector<ParamsStorageBlock*> storage_blocks_to_release;
std::unordered_set<TensorState*> affected_storage_states;
for (const auto& block : params_storage_blocks_) {
@@ -343,14 +403,42 @@ bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState
if (!has_target_state) {
continue;
}
- storage_blocks_to_release.push_back(block.get());
+ if (!block->shared ||
+ std::all_of(block->states.begin(), block->states.end(), [&](auto* state) { return target_states.count(state); }))
+ storage_blocks_to_release.push_back(block.get());
for (TensorState* state : block->states) {
- if (state != nullptr) {
+ if (state != nullptr && (!block->shared || target_states.count(state))) {
affected_storage_states.insert(state);
}
}
}
+ for (auto it = prefetch_blocks_.begin(); it != prefetch_blocks_.end();) {
+ if (std::any_of(it->second->states.begin(), it->second->states.end(), [&](TensorState* state) {
+ return target_states.count(state) || affected_storage_states.count(state);
+ })) {
+ free_prefetch_block(*it->second);
+ it = prefetch_blocks_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ // An idle staging block must be released as a whole before unregistering any member.
+ auto staging_states = target_states;
+ for (const auto& block : compute_staging_blocks_) {
+ const bool intersects = std::any_of(block->staged_tensors.begin(), block->staged_tensors.end(), [&](const auto& entry) {
+ return entry.first && target_states.count(entry.first);
+ });
+ const bool idle = std::all_of(block->staged_tensors.begin(), block->staged_tensors.end(), [](const auto& entry) {
+ return !entry.first || entry.first->pin_count == 0;
+ });
+ if (intersects && idle)
+ for (const auto& entry : block->staged_tensors)
+ if (entry.first)
+ staging_states.insert(entry.first);
+ }
+ release_compute_staging_blocks(false, &staging_states);
+
for (TensorState* state : affected_storage_states) {
if (state == nullptr) {
continue;
@@ -369,8 +457,27 @@ bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState
erase_params_storage_block(block);
}
}
+ for (auto& block : params_storage_blocks_) {
+ if (block->shared) {
+ auto& states = block->states;
+ states.erase(std::remove_if(states.begin(), states.end(), [&](auto* state) {
+ return target_states.count(state) != 0;
+ }),
+ states.end());
+ }
+ }
resolved_tensor_states_.clear();
+ if (keep_registration) {
+ for (auto* state : target_states) {
+ state->tensor->buffer = nullptr;
+ state->tensor->data = nullptr;
+ state->tensor->extra = nullptr;
+ state->loaded_to_params_backend = false;
+ state->applied_lora_epoch = UINT64_MAX;
+ }
+ return true;
+ }
for (auto it = tensor_states_by_tensor_.begin(); it != tensor_states_by_tensor_.end();) {
if (target_states.count(it->second) > 0) {
it = tensor_states_by_tensor_.erase(it);
@@ -395,39 +502,95 @@ bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState
return true;
}
-bool ModelManager::load_all_params_eagerly() {
+ModelManager::ResourceMemory ModelManager::resource_memory(const Resource& resource) const {
+ ResourceMemory result;
+ result.split = resource.layer_split_;
+ for (const auto& state : tensor_states_) {
+ if (state->resource != &resource || !state->tensor)
+ continue;
+ const auto bytes = ggml_nbytes(state->tensor);
+ result.params_bytes += bytes;
+ result.largest_tensor = std::max(result.largest_tensor, bytes);
+ if (state->compute_backend && std::find(result.compute_backends.begin(), result.compute_backends.end(), state->compute_backend) == result.compute_backends.end())
+ result.compute_backends.push_back(state->compute_backend);
+ result.params_backend = state->params_backend;
+ result.mode = state->residency_mode;
+ result.split = result.split || state->split_buffer_type || state->tensor->view_src;
+ }
+ return result;
+}
+
+bool ModelManager::place_resource_params(Resource& resource, ggml_backend_t backend, ResidencyMode mode) {
+ if (resource.layer_split_)
+ return false;
+ std::unordered_set<TensorState*> changed;
+ for (const auto& state : tensor_states_)
+ if (state->resource == &resource && (state->params_backend != backend || state->residency_mode != mode)) {
+ if (state->split_buffer_type || state->tensor->view_src)
+ return false;
+ changed.insert(state.get());
+ }
+ if (!changed.empty() && !unregister_tensor_states(changed, nullptr, true))
+ return false;
+ for (auto* state : changed) {
+ state->params_backend = backend;
+ state->residency_mode = mode;
+ state->params_follow_compute_backend = mode == ResidencyMode::Disk;
+ }
+ return true;
+}
+
+size_t ModelManager::next_weight_use(const TensorState& state) const {
+ if (planned_next_use_.empty())
+ return 0;
+ const auto found = planned_next_use_.find(state.resource->shared_weights_key_);
+ return found == planned_next_use_.end() ? SIZE_MAX : found->second;
+}
+
+bool ModelManager::load_all_params_eagerly(const Resource& resource) {
std::vector<TensorState*> all_states;
all_states.reserve(tensor_states_.size());
for (const auto& s : tensor_states_) {
- if (s != nullptr) {
+ if (s->resource == &resource) {
all_states.push_back(s.get());
}
}
return load_tensors_to_params_backend(all_states);
}
-bool ModelManager::validate_registered_tensors() {
+bool ModelManager::validate_registered_tensors(const Resource& resource) {
bool ok = true;
+ std::vector<TensorState*> share;
for (const auto& state : tensor_states_) {
- if (state == nullptr) {
- ok = false;
+ if (state->resource != &resource)
continue;
- }
bool state_ok = validate_tensor(*state);
if (state_ok) {
state->metadata_validated = true;
+ if (!resource.shared_weights_key_.empty() && !state->loaded_to_params_backend)
+ share.push_back(state.get());
}
ok = state_ok && ok;
}
+ if (ok)
+ reuse_shared_weights(share);
return ok;
}
bool ModelManager::validate_param_tensors(const std::vector<ggml_tensor*>& tensors) {
+ return validate_param_tensors(tensors, nullptr);
+}
+
+bool ModelManager::validate_param_tensors(const std::vector<ggml_tensor*>& tensors, const Resource* resource) {
std::vector<TensorState*> states;
if (!resolve_required_tensor_states(tensors, states)) {
return false;
}
for (auto* state : states) {
+ if (resource && state->resource != resource) {
+ LOG_ERROR("cannot validate a tensor from a different model resource");
+ return false;
+ }
if (!validate_tensor(*state)) {
return false;
}
@@ -440,7 +603,7 @@ bool ModelManager::load_tensors_to_params_backend(const std::vector<TensorState*
std::vector<TensorState*> need_load;
need_load.reserve(states.size());
for (TensorState* state : states) {
- if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) {
+ if (state == nullptr || should_ignore(*state) || (state->sources.empty() && is_optional_missing_tensor(state->name))) {
continue;
}
if (!state->metadata_validated) {
@@ -453,6 +616,7 @@ bool ModelManager::load_tensors_to_params_backend(const std::vector<TensorState*
need_load.push_back(state);
}
}
+ reuse_shared_weights(need_load);
if (need_load.empty()) {
return true;
}
@@ -476,7 +640,7 @@ bool ModelManager::load_tensors_to_params_backend(const std::vector<TensorState*
}
}
- if (!alloc_params_buffers(need_alloc, created_storage_blocks) ||
+ if (!alloc_params_buffers(need_alloc, states, created_storage_blocks) ||
!load_tensors(need_load)) {
for (ParamsStorageBlock* block : created_storage_blocks) {
if (block != nullptr) {
@@ -508,13 +672,38 @@ bool ModelManager::load_tensors_to_params_backend(const std::vector<TensorState*
ggml_backend_buft_name(entry.first));
}
+ std::set<std::string_view> shared_groups;
+ bool shared_loras = false;
+ for (auto* state : need_load) {
+ if (!state->resource->shared_weights_key_.empty()) {
+ shared_groups.insert(state->resource->shared_weights_key_);
+ shared_loras |= state->component == ModelComponent::LoRA;
+ }
+ }
+ if (!shared_groups.empty()) {
+ std::vector<TensorState*> share;
+ for (const auto& state : tensor_states_) {
+ if (!state->loaded_to_params_backend && state->metadata_validated &&
+ (shared_groups.count(state->resource->shared_weights_key_) != 0 ||
+ (shared_loras && state->component == ModelComponent::LoRA)))
+ share.push_back(state.get());
+ }
+ // A sibling can outlive the runner that first read the shared weights.
+ reuse_shared_weights(share);
+ }
+
return true;
}
+bool ModelManager::needs_lora_staging(const TensorState& state) {
+ return !state.resource->shared_weights_key_.empty() && state.component != ModelComponent::LoRA &&
+ state.resource->lora_targets_.count(state.name) != 0;
+}
+
bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorState*>& states) {
- std::map<std::tuple<ggml_backend_t, ggml_backend_buffer_type_t, ModelLoader::FileId>, std::vector<TensorState*>> states_by_staging_target;
+ std::map<std::tuple<ggml_backend_t, ggml_backend_buffer_type_t, Resource*, ModelLoader::FileId>, std::vector<TensorState*>> states_by_staging_target;
for (TensorState* state : states) {
- if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) {
+ if (state == nullptr || should_ignore(*state) || (state->sources.empty() && is_optional_missing_tensor(state->name))) {
continue;
}
if (state->compute_backend == nullptr) {
@@ -526,8 +715,9 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
return false;
}
// Custom CPU operators must retain host weights even when the runner uses a GPU.
- if (state->usage_op == GGML_OP_CUSTOM ||
- state->compute_backend == state->params_backend || state->staged_to_compute_backend) {
+ if (state->staged_to_compute_backend ||
+ (!needs_lora_staging(*state) &&
+ (state->usage_op == GGML_OP_CUSTOM || state->compute_backend == state->params_backend))) {
continue;
}
if (!state->loaded_to_params_backend || state->tensor == nullptr || state->tensor->data == nullptr) {
@@ -538,7 +728,7 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
if (staging_buft == nullptr) {
staging_buft = ggml_backend_get_default_buffer_type(state->compute_backend);
}
- states_by_staging_target[{state->compute_backend, staging_buft, state->source_file}].push_back(state);
+ states_by_staging_target[{state->compute_backend, staging_buft, state->resource, state->source_file}].push_back(state);
}
for (const auto& pair : states_by_staging_target) {
@@ -559,10 +749,30 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
const int64_t t0 = ggml_time_ms();
size_t staged_bytes = 0;
size_t staged_blocks = 0;
- auto stage_chunk = [&](const std::vector<TensorState*>& chunk) -> bool {
+ auto stage_chunk = [&](const std::vector<TensorState*>& chunk, size_t bytes) -> bool {
if (chunk.empty()) {
return true;
}
+ if (retain_weights_ && ggml_backend_buft_is_host(staging_buft) && !ensure_host_capacity(bytes, states)) {
+ return false;
+ }
+#ifdef _WIN32
+ // WDDM device allocations also consume the system commit budget.
+ if (retain_weights_ && !ggml_backend_buft_is_host(staging_buft) &&
+ sd_backend_is(compute_backend, "CUDA") && !ensure_host_capacity(0, states, false, bytes)) {
+ return false;
+ }
+#endif
+ if (retain_weights_ && !ggml_backend_buft_is_host(staging_buft) &&
+ std::any_of(chunk.begin(), chunk.end(), [](const auto* state) {
+ return needs_lora_staging(*state) && state->params_backend == state->compute_backend;
+ })) {
+ std::vector<ggml_tensor*> protected_params;
+ for (auto* state : states)
+ protected_params.push_back(state->tensor);
+ if (!execution_engine()->ensure_capacity(*this, {compute_backend, 0, bytes}, {}, {}, protected_params))
+ return false;
+ }
ggml_init_params init_params;
init_params.mem_size = std::max<size_t>(1, chunk.size()) * ggml_tensor_overhead();
init_params.mem_buffer = nullptr;
@@ -617,7 +827,7 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
ggml_backend_buft_get_alloc_size(staging_buft, state->tensor), alignment);
if (!chunk.empty() && backend_limit > 0 &&
tensor_size > backend_limit - std::min(chunk_size, backend_limit)) {
- if (!stage_chunk(chunk)) {
+ if (!stage_chunk(chunk, chunk_size)) {
return false;
}
chunk.clear();
@@ -626,7 +836,7 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
chunk.push_back(state);
chunk_size = tensor_size > SIZE_MAX - chunk_size ? SIZE_MAX : chunk_size + tensor_size;
}
- if (!stage_chunk(chunk)) {
+ if (!stage_chunk(chunk, chunk_size)) {
return false;
}
LOG_VERBOSE("model manager staged compute params (%6.2f MB, %zu tensors, %zu blocks) to %s, taking %.2fs",
@@ -641,13 +851,24 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
}
bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states) {
- if (loras_.empty() || applying_loras_)
- return true;
- applying_loras_ = true;
+ std::map<Resource*, std::vector<TensorState*>> groups;
+ for (auto* state : states) {
+ if (state && !state->resource->loras_.empty() && !state->resource->applying_loras_)
+ groups[state->resource].push_back(state);
+ }
+ for (const auto& [resource, tensors] : groups) {
+ if (!apply_resource_loras(*resource, tensors))
+ return false;
+ }
+ return true;
+}
+
+bool ModelManager::apply_resource_loras(Resource& resource, const std::vector<TensorState*>& states) {
struct ApplyGuard {
- bool& active;
- ~ApplyGuard() { active = false; }
- } guard{applying_loras_};
+ Resource& resource;
+ ~ApplyGuard() { resource.applying_loras_ = false; }
+ } guard{resource};
+ resource.applying_loras_ = true;
struct LoraApplyGroup {
std::map<std::string, ggml_tensor*> model_tensors;
@@ -658,10 +879,18 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
std::map<ApplyTarget, LoraApplyGroup> groups;
for (TensorState* state : states) {
if (state == nullptr || state->tensor == nullptr || state->component == ModelComponent::LoRA ||
- state->component == ModelComponent::LatentUpsampler || model_sources_.count(state->source_file) != 0 || should_ignore(*state) || is_optional_missing_tensor(state->name)) {
+ should_ignore(*state) || (state->sources.empty() && is_optional_missing_tensor(state->name))) {
+ continue;
+ }
+ if (!resource.shared_weights_key_.empty()) {
+ if (!needs_lora_staging(*state))
+ continue;
+ if (!state->staged_to_compute_backend)
+ return false;
+ } else if (state->component == ModelComponent::LatentUpsampler || resource.model_sources_.count(state->source_file) != 0) {
continue;
}
- if (state->applied_lora_epoch == current_lora_epoch_) {
+ if (state->applied_lora_epoch == resource.current_lora_epoch_) {
continue;
}
if (state->compute_backend == nullptr) {
@@ -670,13 +899,13 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
}
if (state->tensor->buffer != nullptr &&
ggml_backend_buffer_get_type(state->tensor->buffer) == split_buffer_type_for(*state)) {
- if (!warned_split_lora_skip_) {
+ if (!resource.warned_split_lora_skip_) {
LOG_WARN(
"model manager skipping direct lora application to row-split tensors "
"(use --lora-apply-mode at_runtime with row split)");
- warned_split_lora_skip_ = true;
+ resource.warned_split_lora_skip_ = true;
}
- state->applied_lora_epoch = current_lora_epoch_;
+ state->applied_lora_epoch = resource.current_lora_epoch_;
continue;
}
if (state->tensor->data == nullptr) {
@@ -692,22 +921,29 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
return true;
}
- std::set<std::string> all_tensor_names = tensor_names();
+ std::set<std::string> all_tensor_names = tensor_names(resource);
for (auto& group_pair : groups) {
ggml_backend_t compute_backend = std::get<0>(group_pair.first);
LoraApplyGroup& group = group_pair.second;
- for (const LoraSpec& lora_spec : loras_) {
+ for (const LoraSpec& lora_spec : resource.loras_) {
if (group.model_tensors.empty()) {
continue;
}
std::string id = lora_id(lora_spec);
const auto* target = group.states.front();
- // The temporary runner is destroyed before this manager call returns.
- auto borrowed_manager = std::shared_ptr<ModelManager>(this, [](ModelManager*) {});
- auto lora = std::make_shared<LoraModel>(id, compute_backend, target->params_backend,
- borrowed_manager, lora_spec.file_id, lora_version_,
- target->residency_mode);
+ auto lora = lora_spec.prepared.lock();
+ if (lora && (lora->backend() != compute_backend || lora->params_backend != target->params_backend))
+ lora.reset();
+ const bool temporary = !lora;
+ if (temporary) {
+ // The temporary runner is destroyed before this manager call returns.
+ auto borrowed_manager = std::shared_ptr<ModelManager>(this, [](ModelManager*) {});
+ auto weights = std::make_shared<ResourceHandle>(std::move(borrowed_manager), resource.id);
+ lora = std::make_shared<LoraModel>(id, compute_backend, target->params_backend,
+ std::move(weights), lora_spec.file_id, resource.lora_version_,
+ target->residency_mode);
+ }
LoraModel::filter_t lora_tensor_filter = nullptr;
if (!lora_spec.tensor_name_prefix_filter.empty()) {
@@ -715,7 +951,7 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
return starts_with(tensor_name, lora_spec.tensor_name_prefix_filter);
};
}
- if (!lora->init_params(n_threads_, lora_tensor_filter)) {
+ if (temporary && !lora->init_params(resource.n_threads_, lora_tensor_filter)) {
LOG_WARN("load lora tensors from %s failed", lora_spec.path.c_str());
if (lora_spec.required) {
return false;
@@ -730,32 +966,48 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
continue;
}
lora->multiplier = lora_spec.multiplier;
- if (!lora->apply(group.model_tensors, all_tensor_names, lora_version_, n_threads_, false))
+ if (!lora->apply(group.model_tensors, all_tensor_names, resource.lora_version_, resource.n_threads_, false))
return false;
- lora->release_loaded_tensors();
+ if (temporary)
+ lora->release_loaded_tensors();
+ else
+ lora->runner_end();
}
for (TensorState* state : group.states) {
if (state != nullptr) {
- state->applied_lora_epoch = current_lora_epoch_;
+ state->applied_lora_epoch = resource.current_lora_epoch_;
}
}
}
return true;
}
-void ModelManager::reset_lora_applied_params() {
+void ModelManager::reset_lora_applied_params(Resource& resource) {
std::unordered_set<TensorState*> affected;
for (auto& state : tensor_states_) {
- if (state->component != ModelComponent::LoRA && state->applied_lora_epoch != UINT64_MAX) {
+ if (state->resource == &resource && state->component != ModelComponent::LoRA && state->applied_lora_epoch != UINT64_MAX) {
affected.insert(state.get());
}
}
- invalidate_sources(affected);
+ if (!resource.shared_weights_key_.empty()) {
+ for (auto it = compute_staging_blocks_.begin(); it != compute_staging_blocks_.end();) {
+ if (std::any_of((*it)->staged_tensors.begin(), (*it)->staged_tensors.end(), [&](const auto& item) {
+ return affected.count(item.first) != 0;
+ })) {
+ free_compute_staging_block(**it);
+ it = compute_staging_blocks_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ } else {
+ invalidate_sources(affected);
+ }
}
bool ModelManager::should_ignore(const TensorState& state) const {
- if (model_sources_.count(state.source_file) != 0) {
+ if (state.resource != &default_resource_ || state.resource->model_sources_.count(state.source_file) != 0) {
return false;
}
for (const auto& ignore_prefix : common_ignore_tensors_) {
@@ -767,12 +1019,12 @@ bool ModelManager::should_ignore(const TensorState& state) const {
}
bool ModelManager::is_optional_missing_tensor(const std::string& name) const {
- return name.find("cond_stage_model.transformer.text_model.encoder.layers.23") != std::string::npos ||
+ return name.find("cond_stage_model.transformer.text_model.encoder.layers.23.") != std::string::npos ||
name.find("alphas_cumprod") != std::string::npos;
}
bool ModelManager::validate_tensor(const TensorState& state) const {
- if (state.tensor == nullptr || should_ignore(state) || is_optional_missing_tensor(state.name)) {
+ if (state.tensor == nullptr || should_ignore(state) || (state.sources.empty() && is_optional_missing_tensor(state.name))) {
return true;
}
@@ -826,7 +1078,7 @@ bool ModelManager::mmap_params(const std::vector<TensorState*>& states,
std::map<std::string, ggml_tensor*> candidates;
std::map<std::string, TensorState*> states;
};
- std::map<ggml_backend_dev_t, MmapGroup> groups;
+ std::map<std::pair<Resource*, ggml_backend_dev_t>, MmapGroup> groups;
for (TensorState* state : states) {
if (state == nullptr || !can_mmap_storage(*state) || state->tensor == nullptr ||
state->tensor->data != nullptr || state->tensor->view_src != nullptr) {
@@ -836,18 +1088,19 @@ bool ModelManager::mmap_params(const std::vector<TensorState*>& states,
if (!sd_backend_is_cpu(state->compute_backend) && !sd_backend_is_cpu(state->params_backend)) {
device = ggml_backend_get_device(state->compute_backend);
}
- MmapGroup& group = groups[device];
+ MmapGroup& group = groups[{state->resource, device}];
group.candidates[state->name] = state->tensor;
group.states[state->name] = state;
}
- for (auto& [device, group] : groups) {
+ for (auto& [target, group] : groups) {
+ auto [resource, device] = target;
// Device buffers wrap read-only mappings only; params that LoRAs are merged into in place
// are loaded instead.
- if (device != nullptr && writable_mmap_) {
+ if (device != nullptr && resource->writable_mmap_) {
continue;
}
- auto mmap_store = model_loader_.mmap_tensors(group.candidates, {}, writable_mmap_, device);
+ auto mmap_store = resource->model_loader_.mmap_tensors(group.candidates, {}, resource->writable_mmap_, device);
if (mmap_store.empty()) {
continue;
}
@@ -871,20 +1124,210 @@ bool ModelManager::mmap_params(const std::vector<TensorState*>& states,
}
bool ModelManager::can_mmap_storage(const TensorState& state) const {
- if (state.sources.size() > 1 || state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
+ if (state.sources.size() > 1 || state.source_file != 0 || !state.resource->enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
return false;
}
if (state.compute_backend == nullptr || state.params_backend == nullptr) {
return false;
}
+ if (max_ram_bytes_ != 0 && state.resource->writable_mmap_) {
+ return false;
+ }
return sd_backend_is_cpu(state.compute_backend) ||
sd_backend_is_cpu(state.params_backend) ||
backend_supports_host_buffer(state.compute_backend);
}
+size_t ModelManager::host_weight_bytes() const {
+ size_t bytes = 0;
+ auto add = [&](size_t size) { bytes += std::min(size, SIZE_MAX - bytes); };
+ auto add_buffer = [&](ggml_backend_buffer_t buffer) {
+ if (buffer && ggml_backend_buffer_is_host(buffer))
+ add(ggml_backend_buffer_get_size(buffer));
+ };
+ for (const auto& block : params_storage_blocks_) {
+ add_buffer(block->buffer);
+ // Private writable mappings can hold anonymous pages after merging a LoRA.
+ if (!block->mmap_tensor_stores.empty() && block->states.front()->resource->writable_mmap_) {
+ for (auto* state : block->states)
+ add(ggml_nbytes(state->tensor));
+ }
+ }
+ for (const auto& block : compute_staging_blocks_)
+ add_buffer(block->buffer);
+ for (const auto& entry : prefetch_blocks_) {
+ for (const auto& block : entry.second->staging_blocks)
+ add_buffer(block->buffer);
+ }
+ return bytes;
+}
+
+bool ModelManager::evict_host_weight_block(const std::unordered_set<TensorState*>& protected_states) {
+ auto idle = [&](TensorState* state) {
+ return state->pin_count == 0 && !protected_states.count(state);
+ };
+ std::map<TensorState*, ComputeStagingBlock*> staging_for_tensor;
+ auto staging_idle = [&](const ComputeStagingBlock& block) {
+ return std::all_of(block.staged_tensors.begin(), block.staged_tensors.end(),
+ [&](const auto& entry) { return idle(entry.first); });
+ };
+ size_t farthest = 0;
+ uint64_t oldest = UINT64_MAX;
+ ParamsStorageBlock* storage = nullptr;
+ ComputeStagingBlock* staging = nullptr;
+ bool prefer_staged = false;
+ auto better = [&](size_t next, uint64_t last, bool shared) {
+ return next > farthest || (next == farthest &&
+ ((shared && !prefer_staged) || (shared == prefer_staged && last < oldest)));
+ };
+ for (const auto& block : compute_staging_blocks_) {
+ uint64_t last_use = 0;
+ size_t next_use = planned_next_use_.empty() ? 0 : SIZE_MAX;
+ for (const auto& entry : block->staged_tensors) {
+ auto* state = entry.first;
+ staging_for_tensor.emplace(state, block.get());
+ last_use = std::max(last_use, state->last_use_epoch);
+ next_use = std::min(next_use, next_weight_use(*state));
+ }
+ const bool shared = std::any_of(block->staged_tensors.begin(), block->staged_tensors.end(), [](const auto& entry) {
+ return !entry.first->resource->shared_weights_key_.empty();
+ });
+ if (better(next_use, last_use, shared) &&
+ ggml_backend_buffer_is_host(block->buffer) && staging_idle(*block)) {
+ farthest = next_use;
+ oldest = last_use;
+ staging = block.get();
+ prefer_staged = shared;
+ }
+ }
+
+ std::unordered_set<ComputeStagingBlock*> dependent_staging;
+ for (const auto& block : params_storage_blocks_) {
+ if (!block->buffer || !ggml_backend_buffer_is_host(block->buffer))
+ continue;
+ uint64_t last_use = 0;
+ size_t next_use = planned_next_use_.empty() ? 0 : SIZE_MAX;
+ std::unordered_set<ComputeStagingBlock*> dependencies;
+ const bool reclaimable = std::all_of(block->states.begin(), block->states.end(), [&](TensorState* state) {
+ if (!idle(state) || state->sources.empty() || state->applied_lora_epoch != UINT64_MAX ||
+ state->tensor->view_src != nullptr)
+ return false;
+ last_use = std::max(last_use, state->last_use_epoch);
+ next_use = std::min(next_use, next_weight_use(*state));
+ if (state->staged_to_compute_backend) {
+ auto it = staging_for_tensor.find(state);
+ if (it == staging_for_tensor.end())
+ return false;
+ dependencies.insert(it->second);
+ }
+ return true;
+ });
+ if (reclaimable && better(next_use, last_use, false) &&
+ std::all_of(dependencies.begin(), dependencies.end(), [&](const auto* block) { return staging_idle(*block); })) {
+ farthest = next_use;
+ oldest = last_use;
+ storage = block.get();
+ prefer_staged = false;
+ dependent_staging = std::move(dependencies);
+ }
+ }
+ if (!storage && !staging)
+ return false;
+ if (!storage)
+ dependent_staging.insert(staging);
+ for (auto it = compute_staging_blocks_.begin(); it != compute_staging_blocks_.end();) {
+ if (dependent_staging.count(it->get())) {
+ LOG_VERBOSE("model manager evicted %.2f MiB of staged weights on %s for RAM",
+ ggml_backend_buffer_get_size((*it)->buffer) / (1024.0 * 1024.0),
+ ggml_backend_name((*it)->compute_backend));
+ free_compute_staging_block(**it);
+ it = compute_staging_blocks_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ if (storage) {
+ LOG_VERBOSE("model manager evicted %.2f MiB of weight storage from RAM",
+ ggml_backend_buffer_get_size(storage->buffer) / (1024.0 * 1024.0));
+ free_params_storage_block(*storage);
+ erase_params_storage_block(storage);
+ }
+ return true;
+}
+
+bool ModelManager::ensure_host_capacity(size_t pending, const std::vector<TensorState*>& required_states, bool weight_allocation, size_t pending_device_commit) {
+ constexpr size_t headroom = 1024ULL * 1024 * 1024;
+ size_t available = 0;
+ auto fits = [&] {
+ const size_t resident = max_ram_bytes_ ? host_weight_bytes() : 0;
+ available = weight_allocation && max_ram_bytes_ ? max_ram_bytes_ - std::min(max_ram_bytes_, resident) : SIZE_MAX;
+ const int64_t free_ram = sd::backend_fit::available_ram_bytes();
+ if (free_ram >= 0) {
+ const auto free_bytes = static_cast<uint64_t>(free_ram);
+ available = static_cast<size_t>(std::min<uint64_t>(
+ available, free_bytes > headroom ? free_bytes - headroom : 0));
+ if (free_bytes < headroom)
+ return false;
+ }
+ if (pending_device_commit > 0) {
+ const int64_t free_commit = sd::backend_fit::available_commit_bytes();
+ if (free_commit >= 0) {
+ const auto free_bytes = static_cast<uint64_t>(free_commit);
+ const auto reserve = pending_device_commit + std::min(headroom, SIZE_MAX - pending_device_commit);
+ if (free_bytes < reserve) {
+ available = 0;
+ return false;
+ }
+ available = std::min(available, static_cast<size_t>(free_bytes - reserve));
+ }
+ }
+ return pending <= available && (!max_ram_bytes_ || resident <= max_ram_bytes_);
+ };
+ host_memory_error_.clear();
+ if (fits())
+ return true;
+
+ if (!weight_allocation || !max_ram_bytes_ || pending <= max_ram_bytes_) {
+ std::unordered_set<TensorState*> protected_states(required_states.begin(), required_states.end());
+ for (const auto& entry : prefetch_blocks_)
+ protected_states.insert(entry.second->states.begin(), entry.second->states.end());
+ while (evict_host_weight_block(protected_states) || (evict_idle_resource_ && evict_idle_resource_())) {
+ if (fits())
+ return true;
+ }
+ }
+ const size_t resident = host_weight_bytes();
+ host_memory_error_ = std::string(weight_allocation ? "RAM weight capacity exceeded. Requested " : "RAM graph capacity exceeded. Requested ") + std::to_string(pending) +
+ " bytes, " + std::to_string(available) + " bytes available, " +
+ std::to_string(resident) + " bytes resident";
+ if (max_ram_bytes_)
+ host_memory_error_ += ", max_ram is " + std::to_string(max_ram_bytes_) + " bytes";
+ if (pending_device_commit)
+ host_memory_error_ += ", pending device commit is " + std::to_string(pending_device_commit) + " bytes";
+ LOG_ERROR("%s", host_memory_error_.c_str());
+ return false;
+}
+
+bool ModelManager::set_max_ram(size_t bytes) {
+ const auto previous = max_ram_bytes_;
+ max_ram_bytes_ = bytes;
+ if (ensure_host_capacity(0, {}))
+ return true;
+ max_ram_bytes_ = previous;
+ return false;
+}
+
+bool ModelManager::ensure_host_memory(size_t pending, const std::vector<ggml_tensor*>& protected_params, size_t pending_device_commit) {
+ std::vector<TensorState*> states;
+ return resolve_required_tensor_states(protected_params, states) && ensure_host_capacity(pending, states, false, pending_device_commit);
+}
+
bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
+ const std::vector<TensorState*>& required_states,
std::vector<ParamsStorageBlock*>& created_storage_blocks) {
- std::map<std::tuple<ggml_backend_buffer_type_t, int, ModelLoader::FileId>, std::vector<TensorState*>> states_by_buffer_type;
+ std::map<std::tuple<ggml_backend_buffer_type_t, int, Resource*, ModelLoader::FileId>, std::vector<TensorState*>> states_by_buffer_type;
+ std::vector<ggml_tensor*> protected_params;
+ size_t host_bytes = 0;
for (TensorState* state : states) {
if (state == nullptr || state->tensor == nullptr) {
continue;
@@ -893,7 +1336,20 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
if (params_buft == nullptr) {
return false;
}
- states_by_buffer_type[{params_buft, static_cast<int>(state->residency_mode), state->source_file}].push_back(state);
+ if (retain_weights_ && ggml_backend_buft_is_host(params_buft)) {
+ const size_t bytes = GGML_PAD(ggml_backend_buft_get_alloc_size(params_buft, state->tensor),
+ ggml_backend_buft_get_alignment(params_buft));
+ host_bytes += std::min(bytes, SIZE_MAX - host_bytes);
+ }
+ states_by_buffer_type[{params_buft, static_cast<int>(state->residency_mode), state->resource, state->source_file}].push_back(state);
+ }
+ if (host_bytes > 0 && !ensure_host_capacity(host_bytes, required_states))
+ return false;
+ if (retain_weights_) {
+ for (auto* state : required_states) {
+ if (state)
+ protected_params.push_back(state->tensor);
+ }
}
for (const auto& pair : states_by_buffer_type) {
@@ -911,6 +1367,26 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
return true;
}
+ if (retain_weights_ && !ggml_backend_buft_is_host(params_buft)) {
+ const auto* state = chunk.front();
+ auto backend = state->residency_mode == ResidencyMode::Disk ? state->compute_backend : state->params_backend;
+ const auto& placements = state->resource->split_buffer_limits_;
+ auto placement = placements.find(params_buft);
+ const auto devices = placement == placements.end()
+ ? std::vector<std::pair<ggml_backend_t, size_t>>{{backend, 0}}
+ : placement->second;
+ for (const auto& [device, limit] : devices) {
+ if (device && !execution_engine()->ensure_capacity(
+ *this, {device, 0, chunk_size, 0, limit}, {}, {}, protected_params)) {
+ LOG_ERROR("model manager cannot fit params buffer on %s", ggml_backend_name(device));
+ return false;
+ }
+ }
+#ifdef _WIN32
+ if (sd_backend_is(backend, "CUDA") && !ensure_host_capacity(0, required_states, false, chunk_size))
+ return false;
+#endif
+ }
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(params_buft, chunk_size);
if (buffer == nullptr) {
LOG_ERROR("model manager alloc params backend buffer failed, size = %.2fMB",
@@ -1023,9 +1499,10 @@ bool ModelManager::load_tensor_parts(TensorState& state) {
}
return true;
};
+ auto& resource = *state.resource;
const bool success = state.source_file == 0
- ? model_loader_.load_tensors(callback, enable_mmap_, &names, false)
- : source_loader(state.source_file).load_file_tensors(state.source_file, state.source_version, callback, names, enable_mmap_);
+ ? resource.model_loader_.load_tensors(callback, resource.enable_mmap_, &names, false)
+ : source_loader(resource, state.source_file).load_file_tensors(state.source_file, state.source_version, callback, names, resource.enable_mmap_);
if (!success || loaded != names) {
return false;
}
@@ -1038,7 +1515,7 @@ bool ModelManager::load_tensor_parts(TensorState& state) {
}
bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
- using ReadGroup = std::pair<ModelLoader::FileId, SDVersion>;
+ using ReadGroup = std::tuple<Resource*, ModelLoader::FileId, SDVersion>;
using ReadBatch = std::map<std::string, std::vector<ggml_tensor*>>;
std::map<ReadGroup, std::vector<ReadBatch>> groups;
size_t part_count = 0;
@@ -1053,7 +1530,7 @@ bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
return false;
}
auto add_destination = [&](const TensorState& state, const std::string& name, ggml_tensor* tensor) {
- auto& batches = groups[{state.source_file, state.source_version}];
+ auto& batches = groups[{state.resource, state.source_file, state.source_version}];
// The loader supplies one destination per name; incompatible destinations need another batch.
auto batch = std::find_if(batches.begin(), batches.end(), [&](const ReadBatch& candidate) {
auto found = candidate.find(name);
@@ -1111,9 +1588,9 @@ bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
loaded.insert(source.name);
return true;
};
- const auto file = group.first.first;
- bool success = file == 0 ? model_loader_.load_tensors(callback, enable_mmap_, &names)
- : source_loader(file).load_file_tensors(file, group.first.second, callback, names, enable_mmap_);
+ const auto [resource, file, version] = group.first;
+ bool success = file == 0 ? resource->model_loader_.load_tensors(callback, resource->enable_mmap_, &names)
+ : source_loader(*resource, file).load_file_tensors(file, version, callback, names, resource->enable_mmap_);
if (!success || loaded != names)
return false;
for (auto& entry : batch) {
@@ -1138,7 +1615,8 @@ ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorStat
return nullptr;
}
ggml_backend_buffer_type_t params_buft = nullptr;
- if (state.compute_backend != nullptr && state.params_backend != state.compute_backend) {
+ if (state.compute_backend != nullptr && state.params_backend != state.compute_backend &&
+ sd_backend_is_cpu(state.params_backend)) {
ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend);
if (compute_dev != nullptr) {
params_buft = ggml_backend_dev_host_buffer_type(compute_dev);
@@ -1248,6 +1726,10 @@ void ModelManager::release_compute_staging_blocks(bool force,
void ModelManager::free_params_storage_block(ParamsStorageBlock& block) {
if (block.buffer != nullptr) {
+ LOG_DEBUG("free params buffer: %.2f MB(%s), %zu tensors on %s",
+ ggml_backend_buffer_get_size(block.buffer) / (1024.0 * 1024.0),
+ ggml_backend_buffer_is_host(block.buffer) ? "RAM" : "VRAM",
+ block.states.size(), ggml_backend_buffer_name(block.buffer));
ggml_backend_buffer_free(block.buffer);
block.buffer = nullptr;
}
@@ -1269,12 +1751,6 @@ void ModelManager::free_params_storage_block(ParamsStorageBlock& block) {
void ModelManager::release_params_storage_blocks(bool force,
const std::unordered_set<TensorState*>* target_states) {
- struct ReleaseStats {
- size_t bytes = 0;
- size_t tensors = 0;
- size_t blocks = 0;
- };
- std::map<ggml_backend_buffer_type_t, ReleaseStats> released;
for (auto it = params_storage_blocks_.begin(); it != params_storage_blocks_.end();) {
ParamsStorageBlock* block = it->get();
bool can_release = force;
@@ -1296,25 +1772,12 @@ void ModelManager::release_params_storage_blocks(bool force,
}
if (can_release) {
- if (block->buffer != nullptr) {
- auto& stats = released[ggml_backend_buffer_get_type(block->buffer)];
- stats.bytes += ggml_backend_buffer_get_size(block->buffer);
- stats.tensors += block->states.size();
- ++stats.blocks;
- }
free_params_storage_block(*block);
it = params_storage_blocks_.erase(it);
} else {
++it;
}
}
- for (const auto& entry : released) {
- LOG_VERBOSE("model manager released params backend buffers (%6.2f MB, %zu tensors, %zu blocks, %s) from %s",
- entry.second.bytes / (1024.f * 1024.f),
- entry.second.tensors, entry.second.blocks,
- ggml_backend_buft_is_host(entry.first) ? "RAM" : "VRAM",
- ggml_backend_buft_name(entry.first));
- }
}
void ModelManager::erase_params_storage_block(ParamsStorageBlock* block) {
@@ -1432,6 +1895,19 @@ bool ModelManager::assign_compute_backend(const std::vector<ggml_tensor*>& tenso
continue;
}
+ if (params_changes)
+ detach_shared_weight(*state);
+ if (retain_weights_ && state->pin_count == 0 &&
+ (state->staged_to_compute_backend || (params_changes && state->loaded_to_params_backend))) {
+ const auto candidates = evictable_compute_backend_blocks(state->compute_backend, {});
+ const auto found = std::find_if(candidates.begin(), candidates.end(), [&](const auto& block) {
+ return std::find(block.tensors.begin(), block.tensors.end(), state->tensor) != block.tensors.end();
+ });
+ if (found != candidates.end())
+ evict_compute_backend_block(*found);
+ }
+ if (params_changes)
+ detach_shared_weight(*state);
if (state->pin_count > 0 || state->staged_to_compute_backend) {
LOG_ERROR("model manager cannot move active tensor '%s' to another compute backend",
state->name.c_str());
@@ -1464,18 +1940,18 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>&
continue;
}
const bool compute_resident =
- state->compute_backend == state->params_backend
+ state->compute_backend == state->params_backend && !needs_lora_staging(*state)
? state->loaded_to_params_backend
: state->staged_to_compute_backend;
if (missing_only && compute_resident) {
continue;
}
- if (!seen.insert(state).second || should_ignore(*state) || is_optional_missing_tensor(state->name)) {
+ if (!seen.insert(state).second || should_ignore(*state) || (state->sources.empty() && is_optional_missing_tensor(state->name))) {
continue;
}
ggml_backend_buffer_type_t buffer_type = nullptr;
- if (state->compute_backend == state->params_backend) {
+ if (state->compute_backend == state->params_backend && !needs_lora_staging(*state)) {
buffer_type = params_buffer_type_for(*state);
} else {
buffer_type = split_buffer_type_for(*state);
@@ -1527,8 +2003,11 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
};
// The loader may retain device mappings after their parameter blocks are released.
- for (ggml_backend_buffer_t buffer : model_loader_.get_device_mmap_buffers()) {
+ for (auto buffer : default_resource_.model_loader_.get_device_mmap_buffers())
add_buffer(buffer);
+ for (const auto& entry : resources_) {
+ for (auto buffer : entry.second.model_loader_.get_device_mmap_buffers())
+ add_buffer(buffer);
}
for (const auto& block : params_storage_blocks_) {
if (block != nullptr) {
@@ -1555,6 +2034,56 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
return total_size;
}
+ModelManager::RAMBudget ModelManager::available_ram_for_fit() const {
+ size_t reclaimable = 0;
+ auto add = [&](ggml_backend_buffer_t buffer) {
+ if (buffer && ggml_backend_buffer_is_host(buffer))
+ reclaimable += std::min(ggml_backend_buffer_get_size(buffer), SIZE_MAX - reclaimable);
+ };
+ std::map<const TensorState*, bool> staged_idle;
+ for (const auto& block : compute_staging_blocks_) {
+ const bool idle = std::all_of(block->staged_tensors.begin(), block->staged_tensors.end(), [](const auto& entry) {
+ return entry.first->pin_count == 0;
+ });
+ for (const auto& [state, _] : block->staged_tensors)
+ staged_idle[state] = idle;
+ if (idle)
+ add(block->buffer);
+ }
+ for (const auto& block : params_storage_blocks_)
+ if (block->mmap_tensor_stores.empty() && std::all_of(block->states.begin(), block->states.end(), [&](const auto* state) {
+ return state->pin_count == 0 && !state->sources.empty() && state->applied_lora_epoch == UINT64_MAX &&
+ !state->tensor->view_src && (!state->staged_to_compute_backend || (staged_idle.count(state) && staged_idle.at(state)));
+ }))
+ add(block->buffer);
+ const auto resident = host_weight_bytes();
+ const auto fixed = resident - std::min(resident, reclaimable);
+ int64_t free = sd::backend_fit::available_ram_bytes();
+ const auto commit = sd::backend_fit::available_commit_bytes();
+ if (commit >= 0)
+ free = free < 0 ? commit : std::min(free, commit);
+ constexpr size_t reserve = 1024ULL * 1024 * 1024;
+ size_t available = free < 0 ? 0 : size_t(std::max<int64_t>(free - reserve, 0));
+ available += std::min(reclaimable, SIZE_MAX - available);
+ const auto weights = max_ram_bytes_ ? std::min(available, max_ram_bytes_ > fixed ? max_ram_bytes_ - fixed : 0) : available;
+ return {weights, available};
+}
+
+size_t ModelManager::available_vram_for_fit(ggml_backend_t backend, size_t free_bytes, size_t total_bytes) const {
+ if (backend == nullptr || total_bytes == 0 || free_bytes > total_bytes)
+ return free_bytes;
+ size_t reclaimable = 0;
+ for (const auto& block : evictable_compute_backend_blocks(backend, {}))
+ reclaimable += std::min(block.bytes, total_bytes - reclaimable);
+ const auto resident = compute_backend_resident_bytes(backend);
+ const auto limit = max_vram_.bytes_for_backend(backend);
+ const auto remaining = resident - std::min(resident, reclaimable) + other_runtime_resident_bytes(0, backend);
+ size_t available = free_bytes + std::min(reclaimable, total_bytes - free_bytes);
+ if (limit > 0)
+ available = std::min(available, limit > remaining ? limit - remaining : 0);
+ return available;
+}
+
void ModelManager::update_runtime_residency(uintptr_t owner_id,
ggml_backend_t compute_backend,
size_t resident_bytes) {
@@ -1600,13 +2129,22 @@ bool ModelManager::prepare_params(const std::vector<ggml_tensor*>& tensors) {
return false;
}
+ for (const auto* state : required_states) {
+ if (state != nullptr && state->sources.empty()) {
+ LOG_ERROR("required tensor '%s' is missing from model metadata", state->name.c_str());
+ return false;
+ }
+ }
+
if (!load_tensors_to_params_backend(required_states)) {
return false;
}
- if (!stage_tensors_to_compute_backend(required_states)) {
- release_compute_staging_blocks(false);
- release_params_storage_blocks(false);
+ if (!stage_tensors_to_compute_backend(required_states) ||
+ (retain_weights_ && !ensure_host_capacity(0, required_states))) {
+ const std::unordered_set<TensorState*> failed(required_states.begin(), required_states.end());
+ release_compute_staging_blocks(false, &failed);
+ release_params_storage_blocks(false, &failed);
return false;
}
@@ -1662,7 +2200,8 @@ void ModelManager::evict_compute_backend_params(const std::vector<ggml_tensor*>&
// Session cleanup may follow unregistration, after tensor metadata has been freed.
for (auto* tensor : tensors) {
const auto found = tensor_states_by_tensor_.find(tensor);
- if (found != tensor_states_by_tensor_.end()) {
+ if (found != tensor_states_by_tensor_.end() &&
+ (!retain_weights_ || found->second->residency_mode == ResidencyMode::Disk)) {
target_states.insert(found->second);
}
}
@@ -1727,11 +2266,11 @@ WeightResidencyInfo ModelManager::inspect_compute_backend_params(
bool prefetch_candidate = true;
for (TensorState* state : states) {
if (state == nullptr || should_ignore(*state) ||
- is_optional_missing_tensor(state->name)) {
+ (state->sources.empty() && is_optional_missing_tensor(state->name))) {
continue;
}
const bool compute_resident =
- state->compute_backend == state->params_backend
+ state->compute_backend == state->params_backend && !needs_lora_staging(*state)
? state->loaded_to_params_backend
: state->staged_to_compute_backend;
if (compute_resident) {
@@ -1779,10 +2318,27 @@ void ModelManager::remove_runtime_owner(uintptr_t owner_id) {
}
}
+std::map<ggml_backend_t, size_t> ModelManager::split_devices_for(const std::vector<TensorState*>& states) const {
+ std::map<ggml_backend_t, size_t> devices;
+ for (const auto* state : states) {
+ const auto& placements = state->resource->split_buffer_limits_;
+ auto placement = placements.find(split_buffer_type_for(*state));
+ if (placement == placements.end())
+ continue;
+ for (const auto& [backend, limit] : placement->second) {
+ auto [it, inserted] = devices.emplace(backend, limit);
+ if (!inserted && limit > 0)
+ it->second = it->second == 0 ? limit : std::min(it->second, limit);
+ }
+ }
+ return devices;
+}
+
ModelManager::CapacityCheck ModelManager::check_capacity(
const DeviceMemoryRequest& request,
const std::vector<TensorState*>& states,
- bool log_details) const {
+ bool log_details,
+ bool allow_reclamation) const {
CapacityCheck result;
if (request.compute_backend == nullptr || sd_backend_is_cpu(request.compute_backend)) {
return result;
@@ -1799,16 +2355,37 @@ ModelManager::CapacityCheck ModelManager::check_capacity(
#endif
result.required_device_bytes = add(add(request.pending_allocation_bytes, missing), safety_margin);
result.required_budget_bytes = add(request.runtime_peak_bytes(), missing);
- auto available_device_bytes = [&](ggml_backend_t backend) {
+
+ std::vector<ggml_tensor*> protected_params;
+ if (allow_reclamation) {
+ protected_params.reserve(states.size());
+ for (const auto* state : states)
+ protected_params.push_back(state->tensor);
+ }
+ auto account_device = [&](ggml_backend_t backend, size_t limit) {
auto device = ggml_backend_get_device(backend);
- if (device == nullptr) {
- return SIZE_MAX;
- }
+ if (device == nullptr)
+ return;
size_t free_bytes = 0, total_bytes = 0;
ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
const size_t weights_resident = compute_backend_resident_bytes(backend);
const size_t other_runtime = other_runtime_resident_bytes(request.owner_id, backend);
- const size_t resident = add(weights_resident, add(other_runtime, request.runtime_resident_bytes));
+ size_t reclaimable = 0;
+ if (allow_reclamation) {
+ for (const auto& block : evictable_compute_backend_blocks(backend, protected_params))
+ reclaimable += std::min(block.bytes, weights_resident - reclaimable);
+ }
+ const size_t resident = add(weights_resident - reclaimable, other_runtime);
+
+ if (retain_weights_) {
+ const auto shared_limit = max_vram_.bytes_for_backend(backend);
+ if (shared_limit > 0)
+ limit = limit == 0 ? shared_limit : std::min(limit, shared_limit);
+ }
+ if (limit > 0) {
+ result.requires_reclamation |= resident > limit;
+ result.available_budget_bytes = std::min(result.available_budget_bytes, resident < limit ? limit - resident : 0);
+ }
if (log_details) {
LOG_WARN("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB",
ggml_backend_name(backend),
@@ -1816,59 +2393,30 @@ ModelManager::CapacityCheck ModelManager::check_capacity(
weights_resident / (1024.0 * 1024.0), other_runtime / (1024.0 * 1024.0),
request.runtime_resident_bytes / (1024.0 * 1024.0));
}
- if (free_bytes == 0 && total_bytes == 0) {
- return SIZE_MAX;
- }
+ if (free_bytes == 0 && total_bytes == 0)
+ return;
// Vulkan's heap budget subtraction can underflow when usage exceeds the budget.
if (total_bytes > 0 && free_bytes > total_bytes && sd_backend_is(backend, "Vulkan")) {
if (ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_IGPU) {
- return size_t{0};
+ result.available_device_bytes = 0;
+ return;
}
- // A wrapped report may reflect real pressure; reclaim before accepting the tracked estimate.
LOG_VERBOSE("ignoring impossible free memory report on %s (%.2f MB free / %.2f MB total), using tracked residency",
ggml_backend_name(backend), free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0));
free_bytes = total_bytes;
result.wrapped_device_report = true;
}
+ free_bytes = add(free_bytes, reclaimable);
if (total_bytes > 0) {
- free_bytes = std::min(free_bytes, resident < total_bytes ? total_bytes - resident : 0);
+ const size_t used = add(resident, request.runtime_resident_bytes);
+ free_bytes = std::min(free_bytes, used < total_bytes ? total_bytes - used : 0);
}
- return free_bytes;
+ result.available_device_bytes = std::min(result.available_device_bytes, free_bytes);
};
- result.available_device_bytes = available_device_bytes(request.compute_backend);
- if (request.max_backend_bytes > 0) {
- const size_t resident = add(compute_backend_resident_bytes(request.compute_backend),
- other_runtime_resident_bytes(request.owner_id, request.compute_backend));
- result.requires_reclamation = result.requires_reclamation || resident > request.max_backend_bytes;
- result.available_budget_bytes = resident < request.max_backend_bytes
- ? request.max_backend_bytes - resident
- : 0;
- }
- std::map<ggml_backend_t, size_t> split_devices;
- for (auto state : states) {
- auto placement = split_buffer_devices_.find(split_buffer_type_for(*state));
- if (placement != split_buffer_devices_.end()) {
- for (const auto& entry : placement->second) {
- auto inserted = split_devices.emplace(entry);
- if (!inserted.second && entry.second > 0) {
- auto& limit = inserted.first->second;
- limit = limit == 0 ? entry.second : std::min(limit, entry.second);
- }
- }
- }
- }
- // GGML exposes only a split buffer's total size, not per-device allocations.
- // Charge that upper bound on every participant instead of undercounting a shard.
- for (const auto& entry : split_devices) {
- result.available_device_bytes = std::min(result.available_device_bytes, available_device_bytes(entry.first));
- if (entry.second > 0) {
- const size_t resident = add(compute_backend_resident_bytes(entry.first),
- other_runtime_resident_bytes(request.owner_id, entry.first));
- result.requires_reclamation = result.requires_reclamation || resident > entry.second;
- const size_t available = resident < entry.second ? entry.second - resident : 0;
- result.available_budget_bytes = std::min(result.available_budget_bytes, available);
- }
- }
+ account_device(request.compute_backend, request.max_backend_bytes);
+ // GGML exposes only the split buffer's total size, so charge it on each participant.
+ for (const auto& [backend, limit] : split_devices_for(states))
+ account_device(backend, limit);
return result;
}
@@ -1926,9 +2474,10 @@ std::vector<WeightResidencyBlock> ModelManager::evictable_compute_backend_blocks
return;
}
WeightResidencyBlock candidate;
- candidate.buffer = buffer;
- candidate.backend = backend;
- candidate.bytes = ggml_backend_buffer_get_size(buffer);
+ candidate.buffer = buffer;
+ candidate.backend = backend;
+ candidate.bytes = ggml_backend_buffer_get_size(buffer);
+ candidate.next_use = planned_next_use_.empty() ? 0 : SIZE_MAX;
for (auto* state : states) {
if (state == nullptr) {
continue;
@@ -1941,8 +2490,10 @@ std::vector<WeightResidencyBlock> ModelManager::evictable_compute_backend_blocks
return;
}
if (state->residency_mode != ResidencyMode::Disk) {
- if (cpu || state->params_backend == nullptr || sd_backend_is_cpu(state->params_backend) ||
- state->compute_backend != state->params_backend ||
+ const bool direct_gpu_storage = !cpu && state->params_backend &&
+ !sd_backend_is_cpu(state->params_backend) &&
+ state->compute_backend == state->params_backend;
+ if ((!retain_weights_ && !direct_gpu_storage) ||
state->sources.empty() || state->applied_lora_epoch != UINT64_MAX ||
state->split_buffer_type != nullptr || state->tensor->view_src != nullptr) {
return;
@@ -1951,6 +2502,7 @@ std::vector<WeightResidencyBlock> ModelManager::evictable_compute_backend_blocks
}
}
candidate.tensors.push_back(state->tensor);
+ candidate.next_use = std::min(candidate.next_use, next_weight_use(*state));
candidate.last_use_epoch = std::max(candidate.last_use_epoch, state->last_use_epoch);
}
result.push_back(std::move(candidate));
@@ -1983,6 +2535,8 @@ bool ModelManager::evict_compute_backend_block(const WeightResidencyBlock& block
}
for (auto it = compute_staging_blocks_.begin(); it != compute_staging_blocks_.end(); ++it) {
if ((*it)->buffer == block.buffer) {
+ LOG_DEBUG("model manager evicted %.2f MiB of staged weights on %s",
+ block.bytes / (1024.0 * 1024.0), ggml_backend_name(block.backend));
free_compute_staging_block(**it);
compute_staging_blocks_.erase(it);
return true;
@@ -1990,7 +2544,9 @@ bool ModelManager::evict_compute_backend_block(const WeightResidencyBlock& block
}
for (auto it = params_storage_blocks_.begin(); it != params_storage_blocks_.end(); ++it) {
if ((*it)->buffer == block.buffer) {
- if (block.requires_demotion) {
+ LOG_DEBUG("model manager evicted %.2f MiB of weight storage on %s",
+ block.bytes / (1024.0 * 1024.0), ggml_backend_name(block.backend));
+ if (block.requires_demotion && !retain_weights_) {
for (auto* state : (*it)->states) {
if (state != nullptr) {
state->residency_mode = ResidencyMode::Disk;
@@ -2007,13 +2563,14 @@ bool ModelManager::evict_compute_backend_block(const WeightResidencyBlock& block
bool ModelManager::fits_compute_backend_capacity(
const DeviceMemoryRequest& request,
- const std::vector<ggml_tensor*>& required_params) const {
+ const std::vector<ggml_tensor*>& required_params,
+ bool allow_reclamation) const {
std::vector<TensorState*> states;
if (!resolve_required_tensor_states(required_params, states, request.compute_backend)) {
return false;
}
- const auto capacity = check_capacity(request, states);
- return !capacity.requires_reclamation && capacity.fits();
+ return check_capacity(request, states).fits() ||
+ (retain_weights_ && allow_reclamation && check_capacity(request, states, false, true).fits());
}
bool ModelManager::ensure_compute_backend_capacity(
@@ -2021,5 +2578,26 @@ bool ModelManager::ensure_compute_backend_capacity(
const std::vector<ggml_tensor*>& required_params,
const std::vector<std::vector<ggml_tensor*>>& preferred_eviction_order,
const std::vector<ggml_tensor*>& protected_params) {
+ if (retain_weights_) {
+ std::vector<TensorState*> states;
+ if (!resolve_required_tensor_states(required_params, states, request.compute_backend))
+ return false;
+ const auto devices = split_devices_for(states);
+ if (devices.size() > 1) {
+ auto protected_weights = protected_params;
+ protected_weights.insert(protected_weights.end(), required_params.begin(), required_params.end());
+ const auto missing = compute_backend_alloc_size(states, true);
+ for (const auto& [backend, limit] : devices) {
+ if (backend == request.compute_backend)
+ continue;
+ const auto pending = request.pending_allocation_bytes;
+ const DeviceMemoryRequest peer{backend, request.owner_id,
+ pending > SIZE_MAX - missing ? SIZE_MAX : pending + missing,
+ request.runtime_resident_bytes, limit};
+ if (!execution_engine()->ensure_capacity(*this, peer, {}, preferred_eviction_order, protected_weights))
+ return false;
+ }
+ }
+ }
return execution_engine()->ensure_capacity(*this, request, required_params, preferred_eviction_order, protected_params);
}
diff --git a/src/model_manager.h b/src/model_manager.h
index f592a63..3007f40 100644
--- a/src/model_manager.h
+++ b/src/model_manager.h
@@ -2,6 +2,7 @@
#define __MODEL_MANAGER_H__
#include <cstdint>
+#include <functional>
#include <list>
#include <map>
#include <memory>
@@ -10,10 +11,13 @@
#include <unordered_set>
#include <vector>
+#include "core/ggml_graph_cut.h"
#include "device_residency_manager.h"
#include "model_component.h"
#include "model_loader.h"
+struct LoraModel;
+
class ModelManager : public DeviceResidencyManager {
public:
enum class ResidencyMode {
@@ -21,6 +25,22 @@ public:
ParamBackend,
};
+ using ResourceId = uint64_t;
+
+ struct ResourceMemory {
+ size_t params_bytes = 0;
+ size_t largest_tensor = 0;
+ std::vector<ggml_backend_t> compute_backends;
+ ggml_backend_t params_backend = nullptr;
+ ResidencyMode mode = ResidencyMode::ParamBackend;
+ bool split = false;
+ };
+
+ struct RAMBudget {
+ size_t weights = 0;
+ size_t total = 0;
+ };
+
struct LoraSpec {
std::string path;
float multiplier = 1.0f;
@@ -29,12 +49,47 @@ public:
bool required = false;
ModelLoader::FileId file_id = 0;
uint64_t file_revision = 0;
+ std::weak_ptr<LoraModel> prepared;
};
private:
static constexpr size_t MAX_RESIDENCY_BLOCK_BYTES = 1024ULL * 1024ULL * 1024ULL;
+ struct Resource {
+ ResourceId id = 0;
+ ModelLoader model_loader_;
+ std::map<ModelLoader::FileId, ModelLoader> model_sources_;
+ std::map<ggml_backend_t, ggml_backend_buffer_type_t> split_buffer_types_;
+ std::map<ggml_backend_buffer_type_t, std::vector<std::pair<ggml_backend_t, size_t>>> split_buffer_limits_;
+ bool warned_split_lora_skip_ = false;
+ std::vector<LoraSpec> loras_;
+ std::set<std::string> lora_targets_;
+ std::string shared_weights_key_;
+ std::set<ModelLoader::FileId> lora_sources_;
+ bool applying_loras_ = false;
+ SDVersion lora_version_ = VERSION_COUNT;
+ uint64_t current_lora_epoch_ = 0;
+ int n_threads_ = 0;
+ bool enable_mmap_ = false;
+ bool writable_mmap_ = false;
+ bool segmented_compute_disabled_ = false;
+ bool prefetch_disabled_ = false;
+ bool layer_split_ = false;
+ };
+
+ Resource default_resource_;
+ std::set<std::string> common_ignore_tensors_;
+ std::map<ResourceId, Resource> resources_;
+ ResourceId next_resource_id_ = 1;
+ std::map<std::string, size_t> planned_next_use_;
+ bool retain_weights_ = false;
+ mutable sd::ggml_graph_cut::MaxVramAssignment max_vram_;
+ size_t max_ram_bytes_ = 0;
+ std::string host_memory_error_;
+ std::function<bool()> evict_idle_resource_;
+
struct TensorState {
+ Resource* resource = nullptr;
std::string name;
ggml_tensor* tensor = nullptr;
ModelComponent component = ModelComponent::Count;
@@ -62,6 +117,7 @@ private:
ggml_backend_buffer_t buffer = nullptr;
std::vector<MmapTensorStore> mmap_tensor_stores;
std::vector<TensorState*> states;
+ bool shared = false;
};
struct ComputeStagingBlock {
@@ -90,33 +146,18 @@ private:
std::vector<TensorState*> states;
};
- ModelLoader model_loader_;
// Auxiliary models retain their own catalogs and conversion settings.
- std::map<ModelLoader::FileId, ModelLoader> model_sources_;
std::vector<std::unique_ptr<TensorState>> tensor_states_;
std::map<const ggml_tensor*, TensorState*> tensor_states_by_tensor_;
mutable std::list<ResolvedTensorStates> resolved_tensor_states_;
std::vector<std::unique_ptr<ParamsStorageBlock>> params_storage_blocks_;
std::vector<std::unique_ptr<ComputeStagingBlock>> compute_staging_blocks_;
- std::map<ggml_backend_t, ggml_backend_buffer_type_t> split_buffer_types_;
std::map<ggml_backend_buffer_type_t, std::vector<std::pair<ggml_backend_t, size_t>>> split_buffer_devices_;
std::map<uintptr_t, std::unique_ptr<PrefetchBlock>> prefetch_blocks_;
std::map<ggml_backend_t, ggml_backend_t> prefetch_backends_;
std::map<std::pair<uintptr_t, ggml_backend_t>, RuntimeResidency> runtime_residencies_;
std::unordered_set<uintptr_t> runtime_owners_;
- bool warned_split_lora_skip_ = false;
- std::set<std::string> common_ignore_tensors_;
- std::vector<LoraSpec> loras_;
- std::set<ModelLoader::FileId> lora_sources_;
- bool applying_loras_ = false;
- SDVersion lora_version_ = VERSION_COUNT;
- uint64_t current_lora_epoch_ = 0;
- uint64_t residency_epoch_ = 0;
- int n_threads_ = 0;
- bool enable_mmap_ = false;
- bool writable_mmap_ = false;
- bool segmented_compute_disabled_ = false;
- bool prefetch_disabled_ = false;
+ uint64_t residency_epoch_ = 0;
void finish_compute_backend_usage(const std::vector<TensorState*>& states);
void release_all();
@@ -133,28 +174,37 @@ private:
std::vector<TensorState*>& required_states,
ggml_backend_t compute_backend = nullptr) const;
bool should_ignore(const TensorState& state) const;
- const ModelLoader& source_loader(ModelLoader::FileId file) const;
+ static const ModelLoader& source_loader(const Resource& resource, ModelLoader::FileId file);
bool is_optional_missing_tensor(const std::string& name) const;
bool validate_tensor(const TensorState& state) const;
bool load_tensors_to_params_backend(const std::vector<TensorState*>& states);
+ void reuse_shared_weights(std::vector<TensorState*>& need_load);
+ void detach_shared_weight(TensorState& state);
+ static bool needs_lora_staging(const TensorState& state);
bool apply_loras_to_params(const std::vector<TensorState*>& states);
bool mmap_params(const std::vector<TensorState*>& states,
std::vector<ParamsStorageBlock*>& created_storage_blocks);
static std::vector<TensorStorage> find_tensor_sources(const TensorState& state, const String2TensorStorage& sources);
bool load_tensor_parts(TensorState& state);
bool can_mmap_storage(const TensorState& state) const;
+ size_t host_weight_bytes() const;
+ bool evict_host_weight_block(const std::unordered_set<TensorState*>& protected_states);
+ bool ensure_host_capacity(size_t pending, const std::vector<TensorState*>& required_states, bool weight_allocation = true, size_t pending_device_commit = 0);
bool alloc_params_buffers(const std::vector<TensorState*>& states,
+ const std::vector<TensorState*>& required_states,
std::vector<ParamsStorageBlock*>& created_storage_blocks);
bool load_tensors(const std::vector<TensorState*>& states);
bool stage_tensors_to_compute_backend(const std::vector<TensorState*>& states);
size_t compute_backend_alloc_size(const std::vector<TensorState*>& states,
bool missing_only) const;
size_t compute_backend_resident_bytes(ggml_backend_t compute_backend) const;
+ std::map<ggml_backend_t, size_t> split_devices_for(const std::vector<TensorState*>& states) const;
using CapacityCheck = DeviceMemoryCapacity;
CapacityCheck check_capacity(const DeviceMemoryRequest& request,
const std::vector<TensorState*>& states,
- bool log_details = false) const;
+ bool log_details = false,
+ bool allow_reclamation = false) const;
ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const;
ggml_backend_buffer_type_t split_buffer_type_for(const TensorState& state) const;
@@ -165,15 +215,54 @@ private:
void free_compute_staging_block(ComputeStagingBlock& block);
void free_params_storage_block(ParamsStorageBlock& block);
void erase_params_storage_block(ParamsStorageBlock* block);
- void reset_lora_applied_params();
- bool unregister_tensor_states(const std::unordered_set<TensorState*>& states, size_t* size);
+ bool apply_resource_loras(Resource& resource, const std::vector<TensorState*>& states);
+ void reset_lora_applied_params(Resource& resource);
+ bool unregister_tensor_states(const std::unordered_set<TensorState*>& states, size_t* size, bool keep_registration = false);
size_t other_runtime_resident_bytes(uintptr_t owner_id,
ggml_backend_t compute_backend) const;
+ Resource& resource_at(ResourceId id) { return id == 0 ? default_resource_ : resources_.at(id); }
+ bool set_loader(Resource& resource, ModelLoader loader);
+ bool place_resource_params(Resource& resource, ggml_backend_t backend, ResidencyMode mode);
+ ResourceMemory resource_memory(const Resource& resource) const;
+ size_t next_weight_use(const TensorState& state) const;
+ bool del_file(Resource& resource, ModelLoader::FileId file);
+ ModelLoader::FileId register_model_source(Resource& resource, ModelLoader loader);
+ bool unregister_model_source(Resource& resource, ModelLoader::FileId file);
+ size_t registered_params_size(const Resource& resource, const std::set<ModelComponent>& components) const;
+ void set_split_buffer_type(Resource& resource, ggml_backend_t backend, ggml_backend_buffer_type_t type, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits);
+ bool prepare_lora_sources(Resource& resource, std::vector<LoraSpec>& loras);
+ bool set_loras(Resource& resource, std::vector<LoraSpec> loras, SDVersion version, std::set<std::string> targets);
+ std::set<std::string> tensor_names(const Resource& resource) const;
+ bool register_param_tensors(Resource& resource, ModelComponent component, std::map<std::string, ggml_tensor*> tensors, ResidencyMode residency_mode, ggml_backend_t compute_backend, ggml_backend_t params_backend, size_t* size, bool allow_split_buffer, bool follow_compute, const std::map<ggml_tensor*, enum ggml_op>* tensor_ops, ModelLoader::FileId source_file, SDVersion source_version);
+ bool unregister_param_tensors(Resource& resource, ModelComponent component, size_t* size);
+ bool unregister_param_tensors(const std::vector<ggml_tensor*>& tensors, const Resource* resource);
+ bool validate_param_tensors(const std::vector<ggml_tensor*>& tensors, const Resource* resource);
+ bool validate_registered_tensors(const Resource& resource);
+ bool load_all_params_eagerly(const Resource& resource);
+
public:
~ModelManager() override;
- const ModelLoader& loader() const { return model_loader_; }
+ class ResourceHandle;
+
+ ResourceId create_resource();
+ bool remove_resource(ResourceId id);
+ void set_retain_weights(bool retain) { retain_weights_ = retain; }
+ bool retains_idle_weights() const override { return retain_weights_; }
+ void set_max_vram(sd::ggml_graph_cut::MaxVramAssignment limit) { max_vram_ = std::move(limit); }
+ bool set_max_ram(size_t bytes);
+ const sd::ggml_graph_cut::MaxVramAssignment& max_vram() const { return max_vram_; }
+ size_t max_vram_bytes(ggml_backend_t backend) const { return max_vram_.bytes_for_backend(backend); }
+ size_t max_ram() const { return max_ram_bytes_; }
+ void set_idle_resource_eviction(std::function<bool()> evict) { evict_idle_resource_ = std::move(evict); }
+ const std::string& host_memory_error() const { return host_memory_error_; }
+ void clear_host_memory_error() { host_memory_error_.clear(); }
+ size_t available_vram_for_fit(ggml_backend_t backend, size_t free_bytes, size_t total_bytes) const;
+ RAMBudget available_ram_for_fit() const;
+ void set_weight_use_priorities(std::map<std::string, size_t> priorities) { planned_next_use_ = std::move(priorities); }
+
+ const ModelLoader& loader() const { return default_resource_.model_loader_; }
bool set_loader(ModelLoader loader);
bool add_file(const std::string& path, const std::string& prefix = "", ModelLoader::FileId* id = nullptr, bool force = false);
@@ -184,24 +273,24 @@ public:
ModelLoader::FileVersions source_versions(const std::set<ModelComponent>& components, const ModelLoader& loader) const;
size_t registered_params_size(const std::set<ModelComponent>& components) const;
- void prepare_file_io() { model_loader_.process_model_files(enable_mmap_, writable_mmap_); }
+ void prepare_file_io() { default_resource_.model_loader_.process_model_files(default_resource_.enable_mmap_, default_resource_.writable_mmap_); }
bool load_float_tensor(const std::string& name, std::vector<float>& data) {
- return model_loader_.load_float_tensor(name, data, n_threads_, enable_mmap_);
+ return default_resource_.model_loader_.load_float_tensor(name, data, default_resource_.n_threads_, default_resource_.enable_mmap_);
}
void set_n_threads(int n_threads) {
- n_threads_ = n_threads;
- model_loader_.set_n_threads(n_threads);
+ default_resource_.n_threads_ = n_threads;
+ default_resource_.model_loader_.set_n_threads(n_threads);
}
void set_segmented_compute_disabled(bool disabled) {
- segmented_compute_disabled_ = disabled;
+ default_resource_.segmented_compute_disabled_ = disabled;
}
- void set_prefetch_disabled(bool disabled) { prefetch_disabled_ = disabled; }
- void set_enable_mmap(bool enable_mmap) { enable_mmap_ = enable_mmap; }
- void set_writable_mmap(bool writable_mmap) { writable_mmap_ = writable_mmap; }
+ void set_prefetch_disabled(bool disabled) { default_resource_.prefetch_disabled_ = disabled; }
+ void set_enable_mmap(bool enable_mmap) { default_resource_.enable_mmap_ = enable_mmap; }
+ void set_writable_mmap(bool writable_mmap) { default_resource_.writable_mmap_ = writable_mmap; }
void set_common_ignore_tensors(std::set<std::string> ignore_tensors);
bool prepare_lora_sources(std::vector<LoraSpec>& loras);
- bool set_loras(std::vector<LoraSpec> loras, SDVersion version);
+ bool set_loras(std::vector<LoraSpec> loras, SDVersion version, std::set<std::string> targets = {});
void set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits);
static bool tensor_shape_supports_split_buffer(const ggml_tensor* tensor);
@@ -268,12 +357,14 @@ public:
bool assign_compute_backend(const std::vector<ggml_tensor*>& tensors,
ggml_backend_t compute_backend) override;
bool prepare_params(const std::vector<ggml_tensor*>& tensors) override;
+ bool ensure_host_memory(size_t pending, const std::vector<ggml_tensor*>& protected_params, size_t pending_device_commit = 0) override;
void register_runtime_owner(uintptr_t owner_id) override;
void remove_runtime_owner(uintptr_t owner_id) override;
bool fits_compute_backend_capacity(const DeviceMemoryRequest& request,
- const std::vector<ggml_tensor*>& required_params) const override;
- bool segmented_compute_enabled() const override { return !segmented_compute_disabled_; }
- bool prefetch_enabled() const override { return !prefetch_disabled_; }
+ const std::vector<ggml_tensor*>& required_params,
+ bool allow_reclamation) const override;
+ bool segmented_compute_enabled() const override { return !default_resource_.segmented_compute_disabled_; }
+ bool prefetch_enabled() const override { return !default_resource_.prefetch_disabled_; }
void release_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override;
void evict_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override;
WeightResidencyInfo inspect_compute_backend_params(
@@ -306,4 +397,135 @@ public:
void clear_prefetched_params(uintptr_t owner_id) override;
};
+// Runner policy is resource-local. All handles share the manager's engine,
+// weight residency and memory budgets.
+class ModelManager::ResourceHandle final : public DeviceResidencyManager {
+ const std::shared_ptr<ModelManager> manager_;
+ const ResourceId id_;
+
+ Resource& resource() const { return manager_->resource_at(id_); }
+
+public:
+ explicit ResourceHandle(std::shared_ptr<ModelManager> manager)
+ : manager_(std::move(manager)), id_(manager_->create_resource()) {}
+ ResourceHandle(std::shared_ptr<ModelManager> manager, ResourceId id)
+ : manager_(std::move(manager)), id_(id) {}
+
+ bool remove() { return manager_->remove_resource(id_); }
+ ResourceMemory memory() const { return manager_->resource_memory(resource()); }
+ void set_layer_split(bool enabled) { resource().layer_split_ = enabled; }
+ bool place_params(ggml_backend_t backend, ResidencyMode mode) { return manager_->place_resource_params(resource(), backend, mode); }
+ const ModelLoader& loader() const { return resource().model_loader_; }
+ ModelLoader base_loader() const;
+ ModelLoader::Dependencies dependencies() const;
+ bool set_loader(ModelLoader loader) { return manager_->set_loader(resource(), std::move(loader)); }
+ bool del_file(ModelLoader::FileId file) { return manager_->del_file(resource(), file); }
+ ModelLoader::FileId register_model_source(ModelLoader loader) {
+ return manager_->register_model_source(resource(), std::move(loader));
+ }
+ const ModelLoader& model_source(ModelLoader::FileId file) const { return resource().model_sources_.at(file); }
+ bool unregister_model_source(ModelLoader::FileId file) { return manager_->unregister_model_source(resource(), file); }
+ void prepare_file_io() {
+ auto& r = resource();
+ r.model_loader_.process_model_files(r.enable_mmap_, r.writable_mmap_);
+ }
+ bool load_float_tensor(const std::string& name, std::vector<float>& data) {
+ auto& r = resource();
+ return r.model_loader_.load_float_tensor(name, data, r.n_threads_, r.enable_mmap_);
+ }
+ size_t registered_params_size(const std::set<ModelComponent>& components) const {
+ return manager_->registered_params_size(resource(), components);
+ }
+ void set_n_threads(int n_threads) {
+ auto& r = resource();
+ r.n_threads_ = n_threads;
+ r.model_loader_.set_n_threads(n_threads);
+ }
+ void set_enable_mmap(bool enabled) { resource().enable_mmap_ = enabled; }
+ void set_segmented_compute_disabled(bool disabled) { resource().segmented_compute_disabled_ = disabled; }
+ void set_prefetch_disabled(bool disabled) { resource().prefetch_disabled_ = disabled; }
+ void set_shared_weights(std::string key) { resource().shared_weights_key_ = std::move(key); }
+ void set_split_buffer_type(ggml_backend_t backend, ggml_backend_buffer_type_t type, const std::vector<std::pair<ggml_backend_t, size_t>>& limits) {
+ manager_->set_split_buffer_type(resource(), backend, type, limits);
+ }
+ bool prepare_lora_sources(std::vector<LoraSpec>& loras) { return manager_->prepare_lora_sources(resource(), loras); }
+ bool set_loras(std::vector<LoraSpec> loras, SDVersion version, std::set<std::string> targets) {
+ return manager_->set_loras(resource(), std::move(loras), version, std::move(targets));
+ }
+ std::set<std::string> tensor_names() const { return manager_->tensor_names(resource()); }
+ bool register_param_tensors(ModelComponent component, std::map<std::string, ggml_tensor*> tensors, ResidencyMode mode, ggml_backend_t compute_backend, ggml_backend_t params_backend, size_t* size = nullptr, bool allow_split_buffer = false, bool follow_compute = false, const std::map<ggml_tensor*, enum ggml_op>* tensor_ops = nullptr, ModelLoader::FileId source_file = 0, SDVersion source_version = VERSION_COUNT) {
+ return manager_->register_param_tensors(resource(), component, std::move(tensors), mode, compute_backend, params_backend,
+ size, allow_split_buffer, follow_compute, tensor_ops, source_file, source_version);
+ }
+ bool unregister_param_tensors(ModelComponent component) {
+ return manager_->unregister_param_tensors(resource(), component, nullptr);
+ }
+ bool unregister_param_tensors(const std::vector<ggml_tensor*>& tensors) {
+ return manager_->unregister_param_tensors(tensors, &resource());
+ }
+ bool validate_registered_tensors() { return manager_->validate_registered_tensors(resource()); }
+ bool validate_param_tensors(const std::vector<ggml_tensor*>& tensors) {
+ return manager_->validate_param_tensors(tensors, &resource());
+ }
+ bool load_all_params_eagerly() { return manager_->load_all_params_eagerly(resource()); }
+
+ std::shared_ptr<sd::ExecutionEngine> execution_engine() override { return manager_->execution_engine(); }
+ bool segmented_compute_enabled() const override { return !resource().segmented_compute_disabled_; }
+ bool prefetch_enabled() const override { return !resource().prefetch_disabled_; }
+ bool retains_idle_weights() const override { return manager_->retains_idle_weights(); }
+ size_t context_vram_limit(ggml_backend_t backend) const override { return manager_->max_vram_bytes(backend); }
+ void register_runtime_owner(uintptr_t owner) override { manager_->register_runtime_owner(owner); }
+ void remove_runtime_owner(uintptr_t owner) override { manager_->remove_runtime_owner(owner); }
+ bool fits_compute_backend_capacity(const DeviceMemoryRequest& request, const std::vector<ggml_tensor*>& params, bool allow_reclamation) const override {
+ return manager_->fits_compute_backend_capacity(request, params, allow_reclamation);
+ }
+ bool assign_compute_backend(const std::vector<ggml_tensor*>& tensors, ggml_backend_t backend) override {
+ return manager_->assign_compute_backend(tensors, backend);
+ }
+ ggml_tensor* resolve_param_tensor(ggml_tensor* tensor) const override { return manager_->resolve_param_tensor(tensor); }
+ bool prepare_params(const std::vector<ggml_tensor*>& tensors) override { return manager_->prepare_params(tensors); }
+ void release_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override {
+ manager_->release_compute_backend_params(tensors);
+ }
+ void evict_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override {
+ manager_->evict_compute_backend_params(tensors);
+ }
+ WeightResidencyInfo inspect_compute_backend_params(const std::vector<ggml_tensor*>& tensors) const override {
+ return manager_->inspect_compute_backend_params(tensors);
+ }
+ void update_runtime_residency(uintptr_t owner, ggml_backend_t backend, size_t bytes) override {
+ manager_->update_runtime_residency(owner, backend, bytes);
+ }
+ bool ensure_host_memory(size_t pending, const std::vector<ggml_tensor*>& protected_params, size_t pending_device_commit = 0) override {
+ return manager_->ensure_host_memory(pending, protected_params, pending_device_commit);
+ }
+ bool ensure_compute_backend_capacity(const DeviceMemoryRequest& request, const std::vector<ggml_tensor*>& required, const std::vector<std::vector<ggml_tensor*>>& preferred, const std::vector<ggml_tensor*>& protected_params) override {
+ return manager_->ensure_compute_backend_capacity(request, required, preferred, protected_params);
+ }
+ DeviceMemoryCapacity inspect_compute_backend_capacity(const DeviceMemoryRequest& request,
+ const std::vector<ggml_tensor*>& params,
+ bool log_details) const override {
+ return manager_->inspect_compute_backend_capacity(request, params, log_details);
+ }
+ std::vector<WeightResidencyBlock> evictable_compute_backend_blocks(
+ ggml_backend_t backend,
+ const std::vector<ggml_tensor*>& protected_params) const override {
+ return manager_->evictable_compute_backend_blocks(backend, protected_params);
+ }
+ bool evict_compute_backend_block(const WeightResidencyBlock& block) override {
+ return manager_->evict_compute_backend_block(block);
+ }
+ WeightPrefetchResult prefetch_params(uintptr_t owner, const std::vector<ggml_tensor*>& tensors) override {
+ return manager_->prefetch_params(owner, tensors);
+ }
+ WeightPrefetchResult prepare_prefetch_params(uintptr_t owner, const std::vector<ggml_tensor*>& tensors) override {
+ return manager_->prepare_prefetch_params(owner, tensors);
+ }
+ void submit_prefetched_params(uintptr_t owner) override { manager_->submit_prefetched_params(owner); }
+ bool activate_prefetched_params(uintptr_t owner, const std::vector<ggml_tensor*>& tensors) override {
+ return manager_->activate_prefetched_params(owner, tensors);
+ }
+ void clear_prefetched_params(uintptr_t owner) override { manager_->clear_prefetched_params(owner); }
+};
+
#endif // __MODEL_MANAGER_H__
diff --git a/src/model_manager_files.cpp b/src/model_manager_files.cpp
index d6453a2..1846f47 100644
--- a/src/model_manager_files.cpp
+++ b/src/model_manager_files.cpp
@@ -1,12 +1,12 @@
#include "model_manager.h"
#include <algorithm>
+#include <string_view>
#include "core/util.h"
-static bool same_tensor_source(const TensorStorage& a, const TensorStorage& b) {
- return a.file_id == b.file_id && a.file_revision == b.file_revision &&
- a.file_index == b.file_index && a.offset == b.offset && a.index_in_zip == b.index_in_zip &&
+static bool same_tensor_encoding(const TensorStorage& a, const TensorStorage& b) {
+ return a.offset == b.offset && a.index_in_zip == b.index_in_zip &&
a.storage_key == b.storage_key && a.type == b.type && a.expected_type == b.expected_type &&
a.n_dims == b.n_dims && std::equal(a.ne, a.ne + SD_MAX_DIMS, b.ne) &&
a.is_f8_e4m3 == b.is_f8_e4m3 && a.is_f8_e5m2 == b.is_f8_e5m2 &&
@@ -15,6 +15,113 @@ static bool same_tensor_source(const TensorStorage& a, const TensorStorage& b) {
a.int8_convrot_group_size == b.int8_convrot_group_size;
}
+static bool same_tensor_source(const TensorStorage& a, const TensorStorage& b) {
+ return a.file_id == b.file_id && a.file_revision == b.file_revision &&
+ a.file_index == b.file_index && same_tensor_encoding(a, b);
+}
+
+ModelLoader ModelManager::ResourceHandle::base_loader() const {
+ const auto& source = resource();
+ auto loader = source.model_loader_;
+ for (auto file : source.lora_sources_)
+ loader.del_file(file);
+ return loader;
+}
+
+ModelLoader::Dependencies ModelManager::ResourceHandle::dependencies() const {
+ const auto& source = resource();
+ auto result = source.model_loader_.dependencies();
+ for (const auto& [_, loader] : source.model_sources_) {
+ auto dependencies = loader.dependencies();
+ result.insert(result.end(), dependencies.begin(), dependencies.end());
+ }
+ return result;
+}
+
+void ModelManager::reuse_shared_weights(std::vector<TensorState*>& need_load) {
+ if (std::none_of(need_load.begin(), need_load.end(), [](const auto* state) {
+ return !state->resource->shared_weights_key_.empty();
+ }))
+ return;
+ std::map<std::pair<std::string_view, std::string_view>, std::vector<std::pair<TensorState*, ParamsStorageBlock*>>> resident;
+ auto group = [](const TensorState& state) -> std::string_view {
+ return state.component == ModelComponent::LoRA ? std::string_view{} : state.resource->shared_weights_key_;
+ };
+ std::map<TensorState*, ggml_tensor*> original_params;
+ for (const auto& block : compute_staging_blocks_)
+ for (const auto& [state, original] : block->staged_tensors)
+ original_params.emplace(state, original);
+ for (const auto& block : params_storage_blocks_) {
+ for (auto* state : block->states) {
+ if (!state->resource->shared_weights_key_.empty() && !state->sources.empty())
+ resident[{group(*state), state->name}].emplace_back(state, block.get());
+ }
+ }
+ size_t reused = 0;
+ need_load.erase(std::remove_if(need_load.begin(), need_load.end(), [&](auto* state) {
+ if (state->resource->shared_weights_key_.empty() || state->sources.empty() ||
+ state->tensor->view_src || split_buffer_type_for(*state))
+ return false;
+ const auto found = resident.find({group(*state), state->name});
+ if (found == resident.end())
+ return false;
+ for (auto [candidate, block] : found->second) {
+ if (candidate->component != state->component || candidate->residency_mode != state->residency_mode ||
+ candidate->tensor->type != state->tensor->type ||
+ !std::equal(state->tensor->ne, state->tensor->ne + GGML_MAX_DIMS, candidate->tensor->ne) ||
+ !std::equal(state->tensor->nb, state->tensor->nb + GGML_MAX_DIMS, candidate->tensor->nb))
+ continue;
+ auto* tensor = candidate->tensor;
+ if (auto original = original_params.find(candidate); original != original_params.end())
+ tensor = original->second;
+ if (!tensor->buffer || !tensor->data ||
+ ggml_backend_buffer_get_type(tensor->buffer) != params_buffer_type_for(*state))
+ continue;
+ const auto& loader = source_loader(*state->resource, state->source_file);
+ const auto& other = source_loader(*candidate->resource, candidate->source_file);
+ if (!std::equal(state->sources.begin(), state->sources.end(),
+ candidate->sources.begin(), candidate->sources.end(),
+ [&](const TensorStorage& a, const TensorStorage& b) {
+ return same_tensor_encoding(a, b) &&
+ loader.same_weight_file(a.file_id, a.file_index, other, b.file_id, b.file_index);
+ }))
+ continue;
+ block->states.push_back(state);
+ block->shared = true;
+ state->tensor->buffer = tensor->buffer;
+ state->tensor->data = tensor->data;
+ state->tensor->extra = tensor->extra;
+ state->loaded_to_params_backend = true;
+ ++reused;
+ return true;
+ }
+ return false;
+ }),
+ need_load.end());
+ if (reused)
+ LOG_VERBOSE("model manager reused %zu resident weight tensors", reused);
+}
+
+void ModelManager::detach_shared_weight(TensorState& state) {
+ if (state.resource->shared_weights_key_.empty() || !state.loaded_to_params_backend ||
+ state.staged_to_compute_backend || state.pin_count != 0)
+ return;
+ for (auto& block : params_storage_blocks_) {
+ if (!block->shared || block->states.size() < 2)
+ continue;
+ auto found = std::find(block->states.begin(), block->states.end(), &state);
+ if (found == block->states.end())
+ continue;
+ block->states.erase(found);
+ state.tensor->buffer = nullptr;
+ state.tensor->data = nullptr;
+ state.tensor->extra = nullptr;
+ state.loaded_to_params_backend = false;
+ state.applied_lora_epoch = UINT64_MAX;
+ return;
+ }
+}
+
std::vector<TensorStorage> ModelManager::find_tensor_sources(const TensorState& state, const String2TensorStorage& sources) {
auto first = sources.find(state.name);
if (first == sources.end()) {
@@ -74,8 +181,28 @@ void ModelManager::invalidate_sources(const std::unordered_set<TensorState*>& st
}
bool ModelManager::set_loader(ModelLoader loader) {
- if (!runtime_owners_.empty() || std::any_of(tensor_states_.begin(), tensor_states_.end(), [](const auto& state) {
- return state->pin_count != 0;
+ return set_loader(default_resource_, std::move(loader));
+}
+
+bool ModelManager::del_file(ModelLoader::FileId file) {
+ return del_file(default_resource_, file);
+}
+
+ModelLoader::FileId ModelManager::register_model_source(ModelLoader loader) {
+ return register_model_source(default_resource_, std::move(loader));
+}
+
+bool ModelManager::unregister_model_source(ModelLoader::FileId file) {
+ return unregister_model_source(default_resource_, file);
+}
+
+size_t ModelManager::registered_params_size(const std::set<ModelComponent>& components) const {
+ return registered_params_size(default_resource_, components);
+}
+
+bool ModelManager::set_loader(Resource& resource, ModelLoader loader) {
+ if (!runtime_owners_.empty() || std::any_of(tensor_states_.begin(), tensor_states_.end(), [&](const auto& state) {
+ return state->resource == &resource && state->pin_count != 0;
})) {
LOG_ERROR("cannot update model sources during execution");
return false;
@@ -87,18 +214,20 @@ bool ModelManager::set_loader(ModelLoader loader) {
auto key = std::make_pair(state.source_file, state.source_version);
auto found = scoped.find(key);
if (found == scoped.end())
- found = scoped.emplace(key, model_sources_.count(key.first) != 0
- ? source_loader(key.first).file_tensors(key.first, key.second)
+ found = scoped.emplace(key, resource.model_sources_.count(key.first) != 0
+ ? source_loader(resource, key.first).file_tensors(key.first, key.second)
: loader.file_tensors(key.first, key.second))
.first;
return found->second;
};
bool lora_changed = false;
- for (const auto& spec : loras_) {
+ for (const auto& spec : resource.loras_) {
lora_changed |= loader.file_revision(spec.file_id) != spec.file_revision;
}
std::unordered_set<TensorState*> changed;
for (const auto& state : tensor_states_) {
+ if (state->resource != &resource)
+ continue;
const auto sources = find_tensor_sources(*state, sources_for(*state));
if (sources.size() != state->sources.size() ||
!std::equal(sources.begin(), sources.end(), state->sources.begin(), same_tensor_source) ||
@@ -111,19 +240,20 @@ bool ModelManager::set_loader(ModelLoader loader) {
state->sources = find_tensor_sources(*state, sources_for(*state));
}
if (lora_changed) {
- ++current_lora_epoch_;
- for (auto& spec : loras_)
+ ++resource.current_lora_epoch_;
+ for (auto& spec : resource.loras_)
spec.file_revision = loader.file_revision(spec.file_id);
}
- model_loader_ = std::move(loader);
- model_loader_.set_n_threads(n_threads_);
+ resource.model_loader_ = std::move(loader);
+ resource.model_loader_.set_n_threads(resource.n_threads_);
return true;
}
bool ModelManager::add_file(const std::string& path, const std::string& prefix, ModelLoader::FileId* id, bool force) {
- ModelLoader candidate = model_loader_;
+ const auto& resource = default_resource_;
+ ModelLoader candidate = resource.model_loader_;
ModelLoader::FileId added_id;
- if (!candidate.add_file(path, prefix, &added_id, force) || !set_loader(std::move(candidate))) {
+ if (!candidate.add_file(path, prefix, &added_id, force) || !set_loader(default_resource_, std::move(candidate))) {
return false;
}
if (id != nullptr) {
@@ -132,26 +262,28 @@ bool ModelManager::add_file(const std::string& path, const std::string& prefix,
return true;
}
-bool ModelManager::del_file(ModelLoader::FileId id) {
- ModelLoader candidate = model_loader_;
- return candidate.del_file(id) && set_loader(std::move(candidate));
+bool ModelManager::del_file(Resource& resource, ModelLoader::FileId id) {
+ ModelLoader candidate = resource.model_loader_;
+ return candidate.del_file(id) && set_loader(resource, std::move(candidate));
}
bool ModelManager::refresh_files() {
- ModelLoader candidate = model_loader_;
- return candidate.refresh_files() && set_loader(std::move(candidate));
+ const auto& resource = default_resource_;
+ ModelLoader candidate = resource.model_loader_;
+ return candidate.refresh_files() && set_loader(default_resource_, std::move(candidate));
}
ModelLoader::FileVersions ModelManager::source_versions(const std::set<ModelComponent>& components, const ModelLoader& loader) const {
+ const auto& resource = default_resource_;
ModelLoader::FileVersions versions;
const auto& sources = loader.get_tensor_storage_map();
for (const auto& state : tensor_states_) {
- if (components.count(state->component) == 0) {
+ if (state->resource != &resource || components.count(state->component) == 0) {
continue;
}
if (state->source_file != 0) {
- versions[state->source_file] = model_sources_.count(state->source_file) != 0
- ? source_loader(state->source_file).file_revision(state->source_file)
+ versions[state->source_file] = resource.model_sources_.count(state->source_file) != 0
+ ? source_loader(resource, state->source_file).file_revision(state->source_file)
: loader.file_revision(state->source_file);
continue;
}
@@ -162,44 +294,44 @@ ModelLoader::FileVersions ModelManager::source_versions(const std::set<ModelComp
return versions;
}
-size_t ModelManager::registered_params_size(const std::set<ModelComponent>& components) const {
+size_t ModelManager::registered_params_size(const Resource& resource, const std::set<ModelComponent>& components) const {
size_t bytes = 0;
std::unordered_set<const ggml_tensor*> seen;
for (const auto& state : tensor_states_) {
- if (components.count(state->component) != 0 && state->tensor != nullptr && seen.insert(state->tensor).second) {
+ if (state->resource == &resource && components.count(state->component) != 0 && state->tensor != nullptr && seen.insert(state->tensor).second) {
bytes += ggml_nbytes(state->tensor);
}
}
return bytes;
}
-const ModelLoader& ModelManager::source_loader(ModelLoader::FileId file) const {
- auto found = model_sources_.find(file);
- return found == model_sources_.end() ? model_loader_ : found->second;
+const ModelLoader& ModelManager::source_loader(const Resource& resource, ModelLoader::FileId file) {
+ auto found = resource.model_sources_.find(file);
+ return found == resource.model_sources_.end() ? resource.model_loader_ : found->second;
}
-ModelLoader::FileId ModelManager::register_model_source(ModelLoader loader) {
+ModelLoader::FileId ModelManager::register_model_source(Resource& resource, ModelLoader loader) {
const auto versions = loader.file_versions();
if (versions.size() != 1) {
LOG_ERROR("a model source must contain exactly one root model file");
return 0;
}
const auto file = versions.begin()->first;
- if (model_loader_.file_revision(file) != 0 || model_sources_.count(file) != 0) {
+ if (resource.model_loader_.file_revision(file) != 0 || resource.model_sources_.count(file) != 0) {
LOG_ERROR("model source is already registered");
return 0;
}
- loader.set_n_threads(n_threads_);
- model_sources_.emplace(file, std::move(loader));
+ loader.set_n_threads(resource.n_threads_);
+ resource.model_sources_.emplace(file, std::move(loader));
return file;
}
-bool ModelManager::unregister_model_source(ModelLoader::FileId file) {
- if (std::any_of(tensor_states_.begin(), tensor_states_.end(), [file](const auto& state) {
- return state->source_file == file;
+bool ModelManager::unregister_model_source(Resource& resource, ModelLoader::FileId file) {
+ if (std::any_of(tensor_states_.begin(), tensor_states_.end(), [&](const auto& state) {
+ return state->resource == &resource && state->source_file == file;
})) {
LOG_ERROR("cannot unregister a model source while its weights are registered");
return false;
}
- return model_sources_.erase(file) != 0;
+ return resource.model_sources_.erase(file) != 0;
}
diff --git a/src/model_manager_prefetch.cpp b/src/model_manager_prefetch.cpp
index 4056b8f..019b481 100644
--- a/src/model_manager_prefetch.cpp
+++ b/src/model_manager_prefetch.cpp
@@ -83,7 +83,21 @@ bool ModelManager::populate_prefetch_block(PrefetchBlock& block) {
return false;
}
const size_t alignment = ggml_backend_buft_get_alignment(buffer_type);
- size_t backend_limit = ggml_backend_buft_get_max_size(buffer_type);
+ bool host_capacity = ggml_backend_buft_is_host(buffer_type);
+#ifdef _WIN32
+ host_capacity |= sd_backend_is(block.compute_backend, "CUDA");
+#endif
+ if (retain_weights_ && host_capacity) {
+ size_t bytes = 0;
+ for (const auto* state : block.states) {
+ const auto padded = GGML_PAD(ggml_backend_buft_get_alloc_size(buffer_type, state->tensor), alignment);
+ bytes += std::min(padded, SIZE_MAX - bytes);
+ }
+ const bool host = ggml_backend_buft_is_host(buffer_type);
+ if (!ensure_host_capacity(host ? bytes : 0, block.states, host, host ? 0 : bytes))
+ return false;
+ }
+ size_t backend_limit = ggml_backend_buft_get_max_size(buffer_type);
if (!ggml_backend_buft_is_host(buffer_type) &&
(backend_limit == 0 || backend_limit > MAX_RESIDENCY_BLOCK_BYTES)) {
backend_limit = MAX_RESIDENCY_BLOCK_BYTES;
diff --git a/src/model_registry.cpp b/src/model_registry.cpp
index 9a80bb8..231b699 100644
--- a/src/model_registry.cpp
+++ b/src/model_registry.cpp
@@ -1,5 +1,6 @@
#include <array>
#include <stdexcept>
+#include <unordered_set>
#include "model_definition.h"
const std::vector<ModelDefinition>& fibo2_definitions();
@@ -33,15 +34,36 @@ const std::vector<ModelDefinition>& lens_definitions();
const std::vector<ModelDefinition>& iris_definitions();
const std::vector<ModelDefinition>& sd_definitions();
const std::vector<ModelDefinition>& moss_image_definitions();
+const std::vector<ModelDefinition>& seedvr2_definitions();
+
+static bool valid_model_id(std::string_view id) {
+ if (id.empty() || id == "auto" || id.front() < 'a' || id.front() > 'z')
+ return false;
+ bool separator = false;
+ for (char c : id) {
+ if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
+ separator = false;
+ } else if (c == '_' && !separator) {
+ separator = true;
+ } else {
+ return false;
+ }
+ }
+ return !separator;
+}
static const std::array<const ModelDefinition*, VERSION_COUNT>& model_definitions() {
static const auto definitions = [] {
std::array<const ModelDefinition*, VERSION_COUNT> result{};
+ std::unordered_set<std::string_view> ids;
auto append = [&](const std::vector<ModelDefinition>& family) {
for (const auto& definition : family) {
if (definition.version < 0 || definition.version >= VERSION_COUNT || result[definition.version]) {
throw std::logic_error("duplicate or invalid model registration");
}
+ if (!definition.id || !valid_model_id(definition.id) || !ids.insert(definition.id).second) {
+ throw std::logic_error("duplicate or invalid model family ID for version " + std::to_string(definition.version));
+ }
result[definition.version] = &definition;
}
};
@@ -76,6 +98,7 @@ static const std::array<const ModelDefinition*, VERSION_COUNT>& model_definition
append(iris_definitions());
append(sd_definitions());
append(moss_image_definitions());
+ append(seedvr2_definitions());
for (const auto* definition : result) {
if (!definition)
throw std::logic_error("missing model registration");
@@ -91,6 +114,14 @@ const ModelDefinition* find_model_definition(SDVersion version) {
return model_definitions()[version];
}
+const ModelDefinition* find_model_definition(std::string_view id) {
+ for (const auto* definition : model_definitions()) {
+ if (id == definition->id)
+ return definition;
+ }
+ return nullptr;
+}
+
const ModelTraits* find_model_traits(SDVersion version) {
const auto* definition = find_model_definition(version);
return definition ? &definition->traits : nullptr;
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index bd23231..a4bf8f6 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -1,3 +1,4 @@
+#include <initializer_list>
#include <map>
#include <unordered_map>
#include <unordered_set>
@@ -142,6 +143,9 @@ static std::string convert_ming_image_te_name(std::string name) {
if (name == "llm.query_tokens") {
name = "llm.query_tokens_dict.16x16";
}
+ if (name == "connector.query_tokens") {
+ name = "connector.query_tokens_dict.16x16";
+ }
static const std::vector<std::pair<std::string, std::string>> name_map = {
{"thinker.", "backbone."},
{"attention.", "self_attn."},
@@ -153,7 +157,10 @@ static std::string convert_ming_image_te_name(std::string name) {
}
std::string convert_cond_stage_model_name(std::string name, std::string prefix, SDVersion version) {
- if (version == VERSION_MING_IMAGE && prefix == "text_encoders." && starts_with(name, "llm.")) {
+ const bool connector = prefix == "text_encoders." && starts_with(name, "connector.");
+ if (connector && version != VERSION_MING_IMAGE)
+ return name;
+ if (version == VERSION_MING_IMAGE && prefix == "text_encoders." && (starts_with(name, "llm.") || connector)) {
name = convert_ming_image_te_name(name);
}
@@ -225,7 +232,7 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix,
};
if (contains(name, "t5xxl")) {
replace_with_name_map(name, t5_name_map);
- } else if (contains(name, "llm")) {
+ } else if (contains(name, "llm") || connector) {
if (contains(name, "llm.visual")) {
replace_with_name_map(name, llm_vision_name_map);
} else {
@@ -1345,7 +1352,7 @@ std::string convert_diffusers_to_original_wan_vae(std::string name, bool qwen_im
}
std::string convert_first_stage_model_name(std::string name, std::string prefix, SDVersion version) {
- if (sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
+ if (sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version) || version == VERSION_SEEDVR2) {
return name;
}
if (sd_version_uses_wan_vae(version)) {
@@ -1627,6 +1634,54 @@ static std::string convert_ip_adapter_name(std::string name, SDVersion version)
return name;
}
+static std::string convert_connector_name(std::string name, SDVersion version) {
+ const std::string canonical = "text_encoders.connector.";
+ const std::string source = "embeddings_connectors.";
+ const bool connector_file = starts_with(name, source);
+ if (connector_file)
+ name.erase(0, source.size());
+ if (starts_with(name, canonical))
+ return name;
+
+ if (sd_version_is_minimax_h3(version)) {
+ for (const auto* prefix : {"text_encoders.ph3_image_connector.", "text_encoders.llm.text_connector."}) {
+ if (starts_with(name, prefix))
+ return canonical + name.substr(strlen(prefix));
+ }
+ if (connector_file && !starts_with(name, "text_encoders."))
+ return canonical + name;
+ }
+
+ const auto move_modules = [&](std::initializer_list<const char*> modules) {
+ for (const auto* prefix : {"", "model.diffusion_model.", "diffusion_model.", "transformer.", "unet.", "text_encoders.", "text_encoders.llm."}) {
+ for (const auto* module : modules) {
+ const auto from = std::string(prefix) + module + ".";
+ if (starts_with(name, from)) {
+ name = canonical + name.substr(strlen(prefix));
+ return;
+ }
+ }
+ }
+ };
+ if (sd_version_is_ltxav(version)) {
+ move_modules({"text_embedding_projection", "video_embeddings_connector", "audio_embeddings_connector",
+ "caption_projection", "audio_caption_projection"});
+ } else if (sd_version_is_llada_image(version)) {
+ move_modules({"queryformer", "text_projection", "sigvq"});
+ } else if (version == VERSION_MING_IMAGE) {
+ for (const auto* prefix : {"text_encoders.llm.connector.", "connector."}) {
+ if (starts_with(name, prefix))
+ return canonical + name.substr(strlen(prefix));
+ }
+ move_modules({"query_tokens_dict", "proj_in", "proj_out", "proj_directvlm"});
+ for (const auto* prefix : {"text_encoders.llm.", ""}) {
+ if (name == std::string(prefix) + "query_tokens")
+ return canonical + "query_tokens_dict.16x16";
+ }
+ }
+ return name;
+}
+
std::string convert_tensor_name(std::string name, SDVersion version) {
if (version == VERSION_ESRGAN) {
return convert_esrgan_tensor_name(std::move(name));
@@ -1739,19 +1794,11 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
}
}
- const std::vector<std::pair<std::string, std::string>> connector_prefix_map = {
- {"embeddings_connectors.text_encoders.", "text_encoders."},
- {"embeddings_connectors.", sd_version_is_minimax_h3(version) ? "text_encoders.ph3_image_connector." : ""},
- };
- replace_with_prefix_map(name, connector_prefix_map);
-
std::unordered_map<std::string, std::string> prefix_map = {
{"diffusion_model.", "model.diffusion_model."},
{"unet.", "model.diffusion_model."},
{"transformer.", "model.diffusion_model."}, // dit
{"vae.", "first_stage_model."},
- {"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."},
- {"text_encoders.llm.text_connector.", "text_encoders.ph3_image_connector."},
{"text_encoder.", "cond_stage_model.transformer."},
{"te.", "cond_stage_model.transformer."},
{"text_encoder.2.", "cond_stage_model.1.transformer."},
@@ -1779,6 +1826,68 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
}
replace_with_prefix_map(name, prefix_map);
+ name = convert_connector_name(std::move(name), version);
+
+ if (sd_version_is_ltxav(version)) {
+ static const std::unordered_set<std::string> blocks{
+ "patchify_proj",
+ "audio_patchify_proj",
+ "adaln_single",
+ "audio_adaln_single",
+ "prompt_adaln_single",
+ "audio_prompt_adaln_single",
+ "av_ca_video_scale_shift_adaln_single",
+ "av_ca_a2v_gate_adaln_single",
+ "av_ca_audio_scale_shift_adaln_single",
+ "av_ca_v2a_gate_adaln_single",
+ "transformer_blocks",
+ "norm_out",
+ "proj_out",
+ "audio_norm_out",
+ "audio_proj_out",
+ "scale_shift_table",
+ "audio_scale_shift_table",
+ "keyframes_abs_pos_embedding",
+ };
+ if (blocks.count(name.substr(0, name.find('.'))))
+ name = "model.diffusion_model." + name;
+ }
+
+ if (version == VERSION_HIDREAM_O1 || version == VERSION_SENSENOVA_U1_5 ||
+ version == VERSION_SEEDVR2 || version == VERSION_RELAY_VSR) {
+ static const std::vector<std::string> wrappers{
+ "model.diffusion_model.", "diffusion_model.", "diffusion.", "transformer.", "unet."};
+ bool wrapped;
+ do {
+ wrapped = false;
+ for (const auto& prefix : wrappers) {
+ if (starts_with(name, prefix)) {
+ name.erase(0, prefix.size());
+ wrapped = true;
+ break;
+ }
+ }
+ } while (wrapped);
+ if (version == VERSION_SEEDVR2) {
+ if (starts_with(name, "vae."))
+ name = "first_stage_model." + name.substr(4);
+ else if (starts_with(name, "encoder.") || starts_with(name, "decoder."))
+ name = "first_stage_model." + name;
+ else if (!starts_with(name, "first_stage_model."))
+ name = "model.diffusion_model." + name;
+ } else if (version == VERSION_RELAY_VSR) {
+ if (starts_with(name, "first_stage_model."))
+ name = "flash." + name.substr(18);
+ else if (starts_with(name, "condition_encoder.") || starts_with(name, "lr_encoder.") ||
+ starts_with(name, "backbone.") || starts_with(name, "spatial_head.") ||
+ starts_with(name, "residual_projection.") || name == "video_type_embedding" || name == "condition_type_embedding")
+ name = "flash." + name;
+ else if (!starts_with(name, "flash.") && !starts_with(name, "wan."))
+ name = "wan." + name;
+ name = convert_relayvsr_tensor_name(std::move(name));
+ return is_lora ? "lora." + name : name;
+ }
+ }
if (version == VERSION_MOSS_IMAGE) {
const std::string te_prefix = "text_encoders.llm.";
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
deleted file mode 100644
index c2ae6b8..0000000
--- a/src/pipeline/diffusion_engine.cpp
+++ /dev/null
@@ -1,3335 +0,0 @@
-#include "diffusion_engine.h"
-#include "api/tensor.h"
-#include "core/option_types.h"
-
-#include <algorithm>
-#include <cinttypes>
-#include <cmath>
-#include <cstdlib>
-#include <list>
-#include <memory>
-#include <mutex>
-#include <set>
-#include <tuple>
-#include <type_traits>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-#include "core/ggml_extend_backend.h"
-#include "core/ggml_graph_cut.h"
-#include "core/ggml_runner.h"
-#include "core/ggml_tensor_utils.h"
-#include "core/layer_split_partition.h"
-#include "core/probe/conv_direct.h"
-#include "core/probe/flash_attn.h"
-#include "model.h"
-#include "model_definition.h"
-
-#include "config.h"
-#include "core/rng.hpp"
-#include "core/rng_mt19937.hpp"
-#include "core/rng_philox.hpp"
-#include "core/util.h"
-#include "model_builders.h"
-#include "model_loader.h"
-#include "model_manager.h"
-#include "stable-diffusion.h"
-
-#include "conditioning/conditioner.hpp"
-#include "conditioning/conditioning_cache.h"
-#include "core/backend_fit.h"
-#include "extensions/generation_extension.h"
-#include "model/adapter/ip_adapter.hpp"
-#include "model/adapter/lora.hpp"
-#include "model/audio/wav2vec2.hpp"
-#include "model/diffusion/animatediff.hpp"
-#include "model/diffusion/control.hpp"
-#include "model/diffusion/minimax_h3.hpp"
-#include "model/diffusion/model.hpp"
-#include "model/diffusion/z_image.hpp"
-#include "model/vae/audio_vae.hpp"
-#include "model/vae/ltx_vae.hpp"
-#include "model/vae/vae.hpp"
-#include "runtime/audio_processing.h"
-#include "runtime/denoiser.hpp"
-#include "runtime/guidance.h"
-#include "runtime/preview_interval.h"
-#include "runtime/sample-cache.h"
-
-#include "name_conversion.h"
-#include "runtime/latent-preview.h"
-
-#include <atomic>
-
-void calculate_alphas_cumprod(float* alphas_cumprod,
- float linear_start = 0.00085f,
- float linear_end = 0.0120f,
- int timesteps = TIMESTEPS) {
- float ls_sqrt = sqrtf(linear_start);
- float le_sqrt = sqrtf(linear_end);
- float amount = le_sqrt - ls_sqrt;
- float product = 1.0f;
- for (int i = 0; i < timesteps; i++) {
- float beta = ls_sqrt + amount * ((float)i / (timesteps - 1));
- product *= 1.0f - powf(beta, 2.0f);
- alphas_cumprod[i] = product;
- }
-}
-
-void calculate_alphas_cumprod_linear_beta(float* alphas_cumprod,
- float beta_start,
- float beta_end,
- int timesteps = TIMESTEPS) {
- float product = 1.0f;
- for (int i = 0; i < timesteps; i++) {
- float beta = beta_start + (beta_end - beta_start) * ((float)i / (timesteps - 1));
- product *= 1.0f - beta;
- alphas_cumprod[i] = product;
- }
-}
-
-template <typename T, typename = void>
-struct has_set_runtime_backends : std::false_type {};
-template <typename T>
-struct has_set_runtime_backends<T,
- std::void_t<decltype(std::declval<T&>().set_runtime_backends(
- std::declval<const std::vector<ggml_backend_t>&>()))>> : std::true_type {};
-
-static_assert(std::atomic<sd_cancel_mode_t>::is_always_lock_free,
- "sd_cancel_mode_t must be lock-free");
-
-StableDiffusionGGML::StableDiffusionGGML()
- : rng(std::make_shared<PhiloxRNG>()),
- conditioning_cache_(std::make_unique<ConditioningCache>()),
- denoiser(std::make_shared<CompVisDenoiser>()) {}
-
-StableDiffusionGGML::~StableDiffusionGGML() = default;
-
-const std::map<StableDiffusionGGML::RunnerGroup, std::set<ModelComponent>>& StableDiffusionGGML::runner_components() {
- static const std::map<RunnerGroup, std::set<ModelComponent>> components{
- {RunnerGroup::Core, {ModelComponent::Conditioner, ModelComponent::Diffusion, ModelComponent::HighNoiseDiffusion, ModelComponent::CLIPVision, ModelComponent::IPAdapter, ModelComponent::AudioEncoder}},
- {RunnerGroup::VAE, {ModelComponent::VAE, ModelComponent::PreviewVAE, ModelComponent::AudioVAE}},
- {RunnerGroup::ControlNet, {ModelComponent::ControlNet}},
- {RunnerGroup::Extensions, {ModelComponent::PhotoMaker, ModelComponent::PuLID}},
- };
- return components;
-}
-
-StableDiffusionGGML::RunnerGroups StableDiffusionGGML::all_runner_groups() {
- RunnerGroups groups;
- for (const auto& entry : runner_components()) {
- groups.insert(entry.first);
- }
- return groups;
-}
-
-ModelLoader::FileVersions StableDiffusionGGML::runner_source_versions(RunnerGroup group, const ModelLoader& loader) const {
- auto sources = model_manager->source_versions(runner_components().at(group), loader);
- if (group == RunnerGroup::Core) {
- // PhotoMaker's LoRA may already be merged into resident core weights.
- auto extra = loader.file_versions({"alphas_cumprod", "v_pred", "edm_vpred.", "pmid."});
- sources.insert(extra.begin(), extra.end());
- }
- return sources;
-}
-
-void StableDiffusionGGML::capture_runner_sources() {
- RunnerState state;
- state.catalog_revision = model_manager->loader().revision();
- for (const auto& entry : runner_components()) {
- state.sources[entry.first] = runner_source_versions(entry.first, model_manager->loader());
- }
- state.ready = true;
- runner_state_ = std::move(state);
-}
-
-void StableDiffusionGGML::end_runners() {
- if (cond_stage_model)
- cond_stage_model->runner_end();
- if (diffusion_model)
- diffusion_model->runner_end();
- if (high_noise_diffusion_model)
- high_noise_diffusion_model->runner_end();
- if (clip_vision)
- clip_vision->runner_end();
- if (ip_adapter)
- ip_adapter->runner_end();
- if (first_stage_model)
- first_stage_model->runner_end();
- if (preview_vae)
- preview_vae->runner_end();
- if (audio_vae_model)
- audio_vae_model->runner_end();
- if (control_net)
- control_net->runner_end();
- for (auto& extension : generation_extensions)
- extension->runner_end();
- for (auto& lora : runtime_lora_models)
- if (lora.model)
- lora.model->runner_end();
-}
-
-bool StableDiffusionGGML::reset_runners(const RunnerGroups& groups) {
- conditioning_cache_->clear();
- conditioning_loras_.clear();
- end_runners();
- clear_lora_adapters();
- runtime_lora_models.clear();
- for (auto group : groups) {
- for (auto component : runner_components().at(group)) {
- if (!model_manager->unregister_param_tensors(component)) {
- return false;
- }
- }
- }
- for (auto group : groups) {
- switch (group) {
- case RunnerGroup::Core:
- cond_stage_model.reset();
- diffusion_model.reset();
- high_noise_diffusion_model.reset();
- clip_vision.reset();
- ip_adapter.reset();
- ip_adapter_tokens = {};
- ip_adapter_uncond_tokens = {};
- runtime_lora_models.clear();
- break;
- case RunnerGroup::VAE:
- first_stage_model.reset();
- preview_vae.reset();
- audio_vae_model.reset();
- break;
- case RunnerGroup::ControlNet:
- control_net.reset();
- break;
- case RunnerGroup::Extensions:
- generation_extensions.clear();
- break;
- }
- }
- return true;
-}
-
-bool StableDiffusionGGML::refresh_model_sources() {
- bool changed;
- if (!model_manager->loader().files_changed(changed, false)) {
- return false;
- }
- if (!changed && runner_state_.ready && runner_state_.catalog_revision == model_manager->loader().revision()) {
- return true;
- }
- ModelLoader candidate = model_manager->loader();
- return candidate.refresh_files(false) && apply_model_update(std::move(candidate));
-}
-
-bool StableDiffusionGGML::apply_model_update(ModelLoader candidate,
- std::unique_ptr<ModelConfig> next_config,
- RunnerGroups groups) {
- const SDVersion next_version = candidate.get_sd_version();
- if (next_version == VERSION_COUNT) {
- LOG_ERROR("cannot identify updated diffusion model");
- return false;
- }
- if (!runner_state_.ready || next_version != version) {
- groups = all_runner_groups();
- } else {
- for (const auto& entry : runner_components()) {
- if (runner_state_.sources.at(entry.first) != runner_source_versions(entry.first, candidate)) {
- groups.insert(entry.first);
- }
- }
- }
- if (sd_version_is_z_image(version) && sd_version_is_z_image(next_version)) {
- if (groups.count(RunnerGroup::ControlNet) > 0) {
- groups.insert(RunnerGroup::Core);
- }
- if (groups.count(RunnerGroup::Core) > 0 &&
- model_manager->registered_params_size({ModelComponent::ControlNet}) > 0) {
- groups.insert(RunnerGroup::ControlNet);
- }
- }
- runner_state_.ready = false;
- if (!reset_runners(groups)) {
- return false;
- }
- if (!model_manager->set_loader(std::move(candidate))) {
- reset_runners(all_runner_groups());
- return false;
- }
- if (next_config) {
- config_ = std::move(next_config);
- }
- version = next_version;
- if (!build_runners(groups)) {
- reset_runners(all_runner_groups());
- return false;
- }
- capture_runner_sources();
- return true;
-}
-
-ggml_backend_t StableDiffusionGGML::backend_for(SDBackendModule module) {
- ggml_backend_t module_backend = backend_manager.runtime_backend(module);
- if (module_backend == nullptr) {
- LOG_ERROR("failed to initialize %s backend", sd_backend_module_name(module));
- }
- return module_backend;
-}
-
-ggml_backend_t StableDiffusionGGML::params_backend_for(SDBackendModule module) {
- ggml_backend_t module_backend = backend_manager.params_backend(module);
- if (module_backend == nullptr) {
- LOG_ERROR("failed to initialize %s params backend", sd_backend_module_name(module));
- }
- return module_backend;
-}
-
-void StableDiffusionGGML::set_cancel_flag(enum sd_cancel_mode_t flag) {
- cancellation_flag.store(flag, std::memory_order_release);
-}
-
-void StableDiffusionGGML::reset_cancel_flag() {
- set_cancel_flag(SD_CANCEL_RESET);
-}
-
-enum sd_cancel_mode_t StableDiffusionGGML::get_cancel_flag() {
- return cancellation_flag.load(std::memory_order_acquire);
-}
-
-size_t StableDiffusionGGML::max_graph_vram_bytes_for_module(SDBackendModule module) {
- return max_vram_assignment.bytes_for_backend(backend_for(module));
-}
-
-std::vector<size_t> StableDiffusionGGML::layer_split_vram_limits_for_backends(const std::vector<ggml_backend_t>& backends) {
- std::vector<size_t> limits;
- limits.reserve(backends.size());
- for (ggml_backend_t backend : backends) {
- limits.push_back(max_vram_assignment.bytes_for_backend(backend));
- }
- return limits;
-}
-
-bool StableDiffusionGGML::ensure_backend_pair(SDBackendModule module) {
- if (backend_for(module) == nullptr) {
- return false;
- }
- return params_backend_for(module) != nullptr;
-}
-
-template <typename T>
-bool StableDiffusionGGML::register_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- size_t* params_mem_size) {
- if (model == nullptr) {
- return true;
- }
- std::map<std::string, ggml_tensor*> group_tensors;
- std::map<ggml_tensor*, enum ggml_op> tensor_ops;
- model->get_param_tensors(group_tensors);
- if constexpr (std::is_base_of_v<Conditioner, T>) {
- model->get_param_tensor_ops(tensor_ops);
- }
- if (model_manager == nullptr) {
- return true;
- }
- ModelManager::ResidencyMode residency_mode =
- backend_manager.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend;
-
- std::vector<ggml_backend_t> module_backends = backend_manager.runtime_backends(module);
- if (module_backends.size() > 1) {
- if constexpr (has_set_runtime_backends<T>::value) {
- if (module == SDBackendModule::DIFFUSION || module == SDBackendModule::TE) {
- if (backend_manager.split_mode(module) == SDSplitMode::ROW) {
- return register_row_split_runner_params(component,
- model,
- module,
- module_backends,
- std::move(group_tensors),
- tensor_ops,
- residency_mode,
- params_mem_size);
- }
- return register_layer_split_runner_params(component,
- model,
- module,
- module_backends,
- std::move(group_tensors),
- tensor_ops,
- residency_mode,
- params_mem_size);
- }
- }
- LOG_WARN("%s module does not support multiple runtime backends; using %s",
- sd_backend_module_name(module),
- sd::layer_split_backend_device_display_name(module_backends[0]).c_str());
- }
- return model_manager->register_param_tensors(component,
- std::move(group_tensors),
- residency_mode,
- backend_for(module),
- params_backend_for(module),
- params_mem_size,
- false,
- false,
- &tensor_ops);
-}
-
-template <typename T>
-bool StableDiffusionGGML::register_row_split_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- const std::vector<ggml_backend_t>& module_backends,
- std::map<std::string, ggml_tensor*> group_tensors,
- const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
- ModelManager::ResidencyMode residency_mode,
- size_t* params_mem_size) {
- ggml_backend_t main_backend = module_backends[0];
-
- auto fall_back_to_layer_split = [&](const char* reason) {
- LOG_WARN("%s: row split unavailable (%s); falling back to layer split", model_component_name(component), reason);
- return register_layer_split_runner_params(component,
- model,
- module,
- module_backends,
- std::move(group_tensors),
- tensor_ops,
- residency_mode,
- params_mem_size);
- };
-
- ggml_backend_dev_t main_dev = ggml_backend_get_device(main_backend);
- ggml_backend_reg_t reg = main_dev != nullptr ? ggml_backend_dev_backend_reg(main_dev) : nullptr;
- if (reg == nullptr) {
- return fall_back_to_layer_split("no backend registry");
- }
- const size_t reg_dev_count = ggml_backend_reg_dev_count(reg);
- std::vector<float> tensor_split(reg_dev_count, 0.0f);
- constexpr int64_t compute_headroom_bytes = 2ll * 1024 * 1024 * 1024;
- for (ggml_backend_t backend : module_backends) {
- ggml_backend_dev_t dev = ggml_backend_get_device(backend);
- int reg_index = -1;
- for (size_t i = 0; i < reg_dev_count; i++) {
- if (ggml_backend_reg_dev_get(reg, i) == dev) {
- reg_index = (int)i;
- break;
- }
- }
- if (reg_index < 0) {
- return fall_back_to_layer_split("devices span different backend registries");
- }
- size_t free_bytes = 0, total_bytes = 0;
- ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
- int64_t usable_bytes = std::max<int64_t>((int64_t)free_bytes - compute_headroom_bytes,
- (int64_t)free_bytes / 8);
- tensor_split[reg_index] = usable_bytes > 0 ? (float)((double)usable_bytes / (1024.0 * 1024.0)) : 1.0f;
- }
-
- ggml_backend_buffer_type_t split_buft = backend_manager.split_buffer_type(main_backend, tensor_split);
- if (split_buft == nullptr) {
- return fall_back_to_layer_split("backend has no split buffer type");
- }
- std::vector<std::pair<ggml_backend_t, size_t>> split_device_limits;
- for (auto backend : module_backends) {
- split_device_limits.emplace_back(backend, max_vram_assignment.bytes_for_backend(backend));
- }
- model_manager->set_split_buffer_type(main_backend, split_buft, split_device_limits);
-
- std::map<std::string, ggml_tensor*> split_tensors;
- if constexpr (std::is_base_of_v<Conditioner, T>) {
- model->get_layer_split_param_tensors(split_tensors);
- } else {
- split_tensors = group_tensors;
- }
-
- std::map<std::string, ggml_tensor*> row_split_map;
- std::map<std::string, ggml_tensor*> regular_map;
- size_t row_split_bytes = 0;
- for (const auto& kv : group_tensors) {
- if (split_tensors.count(kv.first) != 0 &&
- sd::layer_split_tensor_block_index(kv.first) >= 0 &&
- ModelManager::tensor_shape_supports_split_buffer(kv.second)) {
- row_split_map[kv.first] = kv.second;
- row_split_bytes += ggml_nbytes(kv.second);
- } else {
- regular_map[kv.first] = kv.second;
- }
- }
- if (row_split_map.empty()) {
- return fall_back_to_layer_split("no row-splittable transformer block weights found");
- }
-
- LOG_INFO("%s row split: %zu tensors (%.1f MB) split across %zu devices (main %s)",
- model_component_name(component),
- row_split_map.size(),
- row_split_bytes / (1024.f * 1024.f),
- module_backends.size(),
- sd::layer_split_backend_device_display_name(main_backend).c_str());
-
- if (!model_manager->register_param_tensors(component,
- std::move(row_split_map),
- residency_mode,
- main_backend,
- params_backend_for(module),
- params_mem_size,
- /*allow_split_buffer=*/true,
- false,
- &tensor_ops)) {
- return false;
- }
- return model_manager->register_param_tensors(component,
- std::move(regular_map),
- residency_mode,
- main_backend,
- params_backend_for(module),
- params_mem_size,
- false,
- false,
- &tensor_ops);
-}
-
-template <typename T>
-bool StableDiffusionGGML::register_layer_split_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- const std::vector<ggml_backend_t>& module_backends,
- std::map<std::string, ggml_tensor*> group_tensors,
- const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
- ModelManager::ResidencyMode residency_mode,
- size_t* params_mem_size) {
- bool has_cpu_device = false;
- for (ggml_backend_t backend : module_backends) {
- has_cpu_device = has_cpu_device || sd_backend_is_cpu(backend);
- }
- if (has_cpu_device) {
- // The scheduler reserves the CPU slot for its fallback backend, and
- // CPU weight participation is what --params-backend <module>=cpu is
- // for; a CPU device in a split list is almost certainly a mistake.
- LOG_WARN(
- "%s: layer split across a CPU device is not supported; using %s "
- "(use --params-backend %s=cpu to keep weights in RAM)",
- model_component_name(component),
- sd::layer_split_backend_device_display_name(module_backends[0]).c_str(),
- sd_backend_module_name(module));
- return model_manager->register_param_tensors(component,
- std::move(group_tensors),
- residency_mode,
- module_backends[0],
- params_backend_for(module),
- params_mem_size,
- false,
- false,
- &tensor_ops);
- }
-
- model->set_runtime_backends(module_backends);
- model->set_graph_cut_layer_split_backend_vram_limits(layer_split_vram_limits_for_backends(module_backends));
- model->set_graph_cut_layer_split_enabled(true);
- const bool params_follow_runtime = backend_manager.params_backend_follows_runtime(module) ||
- backend_manager.params_backend_is_disk(module);
- ggml_backend_t initial_params_backend = params_follow_runtime ? module_backends[0] : params_backend_for(module);
- if (initial_params_backend == nullptr) {
- return false;
- }
-
- LOG_INFO("%s graph-cut layer split: deferring %zu tensors across %zu runtime backends until first graph",
- model_component_name(component),
- group_tensors.size(),
- module_backends.size());
-
- return model_manager->register_param_tensors(component,
- std::move(group_tensors),
- residency_mode,
- module_backends[0],
- initial_params_backend,
- params_mem_size,
- false,
- params_follow_runtime,
- &tensor_ops);
-}
-
-bool StableDiffusionGGML::unload_control_net() {
- ContextOperation operation(*this);
- if (!operation.acquired) {
- return false;
- }
- if (model_manager == nullptr || config_ == nullptr) {
- LOG_ERROR("cannot unload ControlNet: context is not initialized");
- return false;
- }
- ModelLoader candidate = model_manager->loader();
- if (config_->control_net_file != 0 && !candidate.del_file(config_->control_net_file)) {
- return false;
- }
- auto next_config = std::make_unique<ModelConfig>(*config_);
- next_config->set_control_net(0, "");
- return apply_model_update(std::move(candidate), std::move(next_config), {RunnerGroup::ControlNet});
-}
-
-bool StableDiffusionGGML::load_control_net_from_file(const std::string& path) {
- ContextOperation operation(*this);
- if (!operation.acquired) {
- return false;
- }
- if (path.empty() || model_manager == nullptr || config_ == nullptr) {
- LOG_ERROR("cannot load ControlNet: invalid path or uninitialized context");
- return false;
- }
- ModelLoader candidate = model_manager->loader();
- ModelLoader::FileId file_id;
- if (!candidate.add_file(path, "", &file_id)) {
- return false;
- }
- if (config_->control_net_file != 0 && config_->control_net_file != file_id && !candidate.del_file(config_->control_net_file)) {
- return false;
- }
- auto next_config = std::make_unique<ModelConfig>(*config_);
- next_config->set_control_net(file_id, path);
- return apply_model_update(std::move(candidate), std::move(next_config), {RunnerGroup::ControlNet});
-}
-
-bool StableDiffusionGGML::init_backend() {
- std::string error;
- if (!backend_manager.init(backend_spec.c_str(),
- params_backend_spec.c_str(),
- split_mode_spec.c_str(),
- &error)) {
- LOG_ERROR("backend config failed: %s", error.c_str());
- return false;
- }
- return ensure_backend_pair(SDBackendModule::DIFFUSION);
-}
-
-bool StableDiffusionGGML::row_split_active() {
- for (SDBackendModule module : {SDBackendModule::DIFFUSION, SDBackendModule::TE}) {
- if (backend_manager.split_mode(module) == SDSplitMode::ROW &&
- backend_manager.runtime_backends(module).size() > 1) {
- return true;
- }
- }
- return false;
-}
-
-bool StableDiffusionGGML::graph_cut_layer_split_active() {
- for (SDBackendModule module : {SDBackendModule::DIFFUSION, SDBackendModule::TE}) {
- if (backend_manager.split_mode(module) == SDSplitMode::LAYER &&
- backend_manager.runtime_backends(module).size() > 1) {
- return true;
- }
- }
- return false;
-}
-
-std::shared_ptr<RNG> StableDiffusionGGML::get_rng(rng_type_t rng_type) {
- if (rng_type == STD_DEFAULT_RNG) {
- return std::make_shared<STDDefaultRNG>();
- } else if (rng_type == CPU_RNG) {
- return std::make_shared<MT19937RNG>();
- } else { // default: CUDA_RNG
- return std::make_shared<PhiloxRNG>();
- }
-}
-
-void StableDiffusionGGML::refresh_compvis_denoiser_sigmas() {
- auto comp_vis_denoiser = std::dynamic_pointer_cast<CompVisDenoiser>(denoiser);
- if (!comp_vis_denoiser) {
- return;
- }
- std::vector<float> alphas_cumprod(TIMESTEPS);
- if (file_alphas_cumprod.size() == TIMESTEPS) {
- alphas_cumprod = file_alphas_cumprod;
- } else if (sd_version_is_pixart(version)) {
- // PixArt checkpoints train with a linear beta schedule (0.0001 -> 0.02)
- // instead of the scaled_linear schedule used by SD1.x/SDXL.
- calculate_alphas_cumprod_linear_beta(alphas_cumprod.data(), 0.0001f, 0.02f);
- } else {
- calculate_alphas_cumprod(alphas_cumprod.data());
- }
- for (int i = 0; i < TIMESTEPS; i++) {
- comp_vis_denoiser->sigmas[i] = std::sqrt((1 - alphas_cumprod[i]) / alphas_cumprod[i]);
- comp_vis_denoiser->log_sigmas[i] = std::log(comp_vis_denoiser->sigmas[i]);
- }
-}
-
-void StableDiffusionGGML::load_alphas_cumprod() {
- file_alphas_cumprod.clear();
-
- std::vector<float> loaded_alphas;
- if (!model_manager->load_float_tensor("alphas_cumprod", loaded_alphas)) {
- return;
- }
- if (loaded_alphas.size() != TIMESTEPS) {
- LOG_WARN("ignore alphas_cumprod from model file: expected %d values, got %zu",
- TIMESTEPS,
- loaded_alphas.size());
- return;
- }
- for (float alpha : loaded_alphas) {
- if (!std::isfinite(alpha) || alpha <= 0.0f || alpha > 1.0f) {
- LOG_WARN("ignore invalid alphas_cumprod from model file");
- return;
- }
- }
-
- file_alphas_cumprod = std::move(loaded_alphas);
- LOG_VERBOSE("loaded alphas_cumprod from model file");
-}
-
-bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConfig& configuration) {
- const auto* sd_ctx_params = &configuration.params;
- auto& use_tae = configuration.use_tae;
- auto& use_audio_vae = configuration.use_audio_vae;
- const auto load_model = [&](const char* name, const char* path, const std::string& prefix = "") {
- if (!model_loader.init_from_file(path, prefix))
- return false;
- configuration.record_model(name, path);
- return true;
- };
- if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
- LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
- if (!load_model("model", sd_ctx_params->model_path)) {
- LOG_ERROR("init model loader from file failed: '%s'", sd_ctx_params->model_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->diffusion_model_path)) > 0) {
- LOG_INFO("loading diffusion model from '%s'", sd_ctx_params->diffusion_model_path);
- if (!load_model("diffusion_model", sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
- LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->diffusion_model_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
- LOG_INFO("loading high noise diffusion model from '%s'", sd_ctx_params->high_noise_diffusion_model_path);
- if (!load_model("high_noise_diffusion_model", sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
- LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->high_noise_diffusion_model_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->uncond_diffusion_model_path)) > 0) {
- LOG_INFO("loading unconditional diffusion model from '%s'", sd_ctx_params->uncond_diffusion_model_path);
- if (!load_model("uncond_diffusion_model", sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
- LOG_WARN("loading unconditional diffusion model from '%s' failed", sd_ctx_params->uncond_diffusion_model_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) {
- LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path);
- if (!load_model("clip_l", sd_ctx_params->clip_l_path, "clip_l.")) {
- LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) {
- LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path);
- if (!load_model("clip_g", sd_ctx_params->clip_g_path, "clip_g.")) {
- LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) {
- LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path);
- if (!load_model("clip_vision", sd_ctx_params->clip_vision_path, "clip_vision.")) {
- LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->t5xxl_path)) > 0) {
- LOG_INFO("loading t5xxl from '%s'", sd_ctx_params->t5xxl_path);
- if (!load_model("t5xxl", sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
- LOG_WARN("loading t5xxl from '%s' failed", sd_ctx_params->t5xxl_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) {
- LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path);
- if (!load_model("pulid_weights", sd_ctx_params->pulid_weights_path,
- "model.diffusion_model.")) {
- LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) {
- LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path);
- if (!load_model("llm", sd_ctx_params->llm_path, "text_encoders.llm.")) {
- LOG_WARN("loading llm from '%s' failed", sd_ctx_params->llm_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->llm_vision_path)) > 0) {
- LOG_INFO("loading llm vision from '%s'", sd_ctx_params->llm_vision_path);
- if (!load_model("llm_vision", sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
- LOG_WARN("loading llm vision from '%s' failed", sd_ctx_params->llm_vision_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->vae_path)) > 0) {
- LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
- if (!load_model("vae", sd_ctx_params->vae_path, "vae.")) {
- LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
- external_vae_is_invalid = true;
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->taesd_path)) > 0) {
- LOG_INFO("loading tae from '%s'", sd_ctx_params->taesd_path);
- if (!load_model("taesd", sd_ctx_params->taesd_path, "tae.")) {
- LOG_WARN("loading tae from '%s' failed", sd_ctx_params->taesd_path);
- } else {
- use_tae = true;
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) {
- LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path);
- if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path, "embeddings_connectors.")) {
- LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->audio_vae_path)) > 0) {
- LOG_INFO("loading audio VAE from '%s'", sd_ctx_params->audio_vae_path);
- if (!load_model("audio_vae", sd_ctx_params->audio_vae_path)) {
- LOG_WARN("loading audio VAE weights from '%s' failed", sd_ctx_params->audio_vae_path);
- } else {
- use_audio_vae = true;
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->audio_encoder_path)) > 0) {
- LOG_INFO("loading audio encoder (wav2vec2) from '%s'", sd_ctx_params->audio_encoder_path);
- if (!load_model("audio_encoder", sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
- LOG_WARN("loading audio encoder weights from '%s' failed", sd_ctx_params->audio_encoder_path);
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->motion_module_path)) > 0) {
- LOG_INFO("loading motion module (AnimateDiff) from '%s'", sd_ctx_params->motion_module_path);
- if (!load_model("motion_module", sd_ctx_params->motion_module_path,
- "model.diffusion_model.motion_module.")) {
- LOG_WARN("loading motion module from '%s' failed", sd_ctx_params->motion_module_path);
- } else {
- configuration.animatediff_loaded = true;
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->control_net_path)) > 0) {
- if (!model_loader.add_file(sd_ctx_params->control_net_path, "", &configuration.control_net_file)) {
- LOG_ERROR("init control net model loader from file failed: '%s'", sd_ctx_params->control_net_path);
- return false;
- }
- }
-
- if (configuration.control_net_file)
- configuration.record_model("control_net", sd_ctx_params->control_net_path);
-
- if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) {
- if (!load_model("ip_adapter", sd_ctx_params->ip_adapter_path)) {
- LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path);
- return false;
- }
- }
-
- if (strlen(SAFE_STR(sd_ctx_params->photo_maker_path)) > 0) {
- configuration.photomaker_source_available = model_loader.add_file(sd_ctx_params->photo_maker_path, "pmid.");
- if (configuration.photomaker_source_available)
- configuration.record_model("photo_maker", sd_ctx_params->photo_maker_path);
- if (!configuration.photomaker_source_available) {
- LOG_WARN("loading stacked ID embedding from '%s' failed", sd_ctx_params->photo_maker_path);
- }
- }
-
- model_loader.convert_tensors_name();
-
- ggml_type wtype = sd_type_to_ggml_type(sd_ctx_params->wtype);
- std::string tensor_type_rules = SAFE_STR(sd_ctx_params->tensor_type_rules);
- if (wtype != GGML_TYPE_COUNT || tensor_type_rules.size() > 0) {
- model_loader.set_wtype_override(wtype, tensor_type_rules);
- }
-
- return true;
-}
-
-bool StableDiffusionGGML::set_sage_attention_enabled(bool enabled) {
- if (!diffusion_model) {
- return false;
- }
- if (enabled) {
-#ifndef SD_USE_UPSTREAM_GGML
- auto* ctx = ggml_init({4 * ggml_tensor_overhead(), nullptr, true});
- if (ctx == nullptr) {
- return false;
- }
- auto* q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 128, 128, 1, 1);
- auto* k = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 128, 128, 1, 1);
- auto* v = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 128, 128, 1, 1);
- GGMLRunnerContext probe_ctx;
- probe_ctx.ggml_ctx = ctx;
- auto* op = ggml_ext_sage_attn(&probe_ctx, q, k, v, 1.f / sqrtf(128.f), GGML_SAGE_ATTN_AUTO);
- bool supported = true;
- for (auto backend : backend_manager.runtime_backends(SDBackendModule::DIFFUSION)) {
- if (!ggml_backend_supports_op(backend, op)) {
- LOG_ERROR("SageAttention is unavailable on %s; it requires patched GGML, CUDA Toolkit 12.0 or newer, and SM80 or newer kernels",
- ggml_backend_name(backend));
- supported = false;
- }
- }
- ggml_free(ctx);
- if (!supported) {
- return false;
- }
-#else
- LOG_ERROR("SageAttention requires -DSD_USE_UPSTREAM_GGML=OFF and a CUDA backend");
- return false;
-#endif
- }
- diffusion_model->set_sage_attention_enabled(enabled);
- if (high_noise_diffusion_model) {
- high_noise_diffusion_model->set_sage_attention_enabled(enabled);
- }
- if (enabled) {
- LOG_INFO("Using SageAttention in the diffusion model; CUDA selects the supported kernel, unsupported layers use flash/default attention");
- }
- return true;
-}
-
-bool StableDiffusionGGML::init(const sd::ContextConfig* sd_ctx_params) {
-#ifdef SD_USE_UPSTREAM_GGML
- LOG_WARN(
- "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are "
- "converted to F16 at load time. Some operators may be unsupported and performance "
- "may be lower than with patched GGML.");
-#endif
- if (!validate_tensor_types(sd_ctx_params->wtype, sd_ctx_params->tensor_type_rules)) {
- return false;
- }
- for (float scale : {sd_ctx_params->linear_scale, sd_ctx_params->attn_scale}) {
- if (!std::isfinite(scale) || scale < 0.f || (scale > 0.f && !std::isfinite(1.f / scale))) {
- LOG_ERROR("scale overrides must be finite positive values, or 0 to keep model defaults");
- return false;
- }
- }
- if (sd_ctx_params->conditioning_cache_size < 0) {
- LOG_ERROR("conditioning_cache_size must be non-negative");
- return false;
- }
- const std::string flash_mode = SAFE_STR(sd_ctx_params->flash_attn_mode);
- std::string flash_error;
- if (!sd_parse_flash_attention_assignment(flash_mode, &flash_attn_assignment, &flash_error)) {
- LOG_ERROR("invalid flash_attn_mode '%s': %s", flash_mode.c_str(), flash_error.c_str());
- return false;
- }
- const std::string conv_mode = SAFE_STR(sd_ctx_params->conv_direct_mode);
- std::string conv_error;
- if (!sd_parse_conv_direct_assignment(conv_mode, &conv_direct_assignment, &conv_error)) {
- LOG_ERROR("invalid conv_direct_mode '%s': %s", conv_mode.c_str(), conv_error.c_str());
- return false;
- }
- flash_attn_by_backend.clear();
- conv2d_direct_by_backend.clear();
- conv3d_direct_by_backend.clear();
- conditioning_cache_->set_capacity(static_cast<size_t>(sd_ctx_params->conditioning_cache_size));
- auto configuration = std::make_unique<ModelConfig>(*sd_ctx_params);
- n_threads = sd_ctx_params->n_threads;
- tensor_executor = std::make_unique<sd::ParallelExecutor>(n_threads > 0 ? n_threads : sd_get_num_physical_cores());
- sd::ParallelScope tensor_scope(tensor_executor.get());
- enable_mmap = sd_ctx_params->enable_mmap;
- disable_prefetch = sd_ctx_params->disable_prefetch;
- disable_segmented_compute = sd_ctx_params->disable_segmented_compute;
- eager_load = sd_ctx_params->eager_load;
- backend_spec = SAFE_STR(sd_ctx_params->backend);
- params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
- split_mode_spec = SAFE_STR(sd_ctx_params->split_mode);
- auto_fit_enabled = sd_ctx_params->auto_fit && params_backend_spec.empty();
- max_vram_assignment.reset(0.f);
- {
- std::string error;
- if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) {
- LOG_ERROR("%s", error.c_str());
- return false;
- }
- }
-
- std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers);
- add_rpc_devices(rpc_servers_spec);
-
- rng = get_rng(sd_ctx_params->rng_type);
- if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) {
- sampler_rng = get_rng(sd_ctx_params->sampler_rng_type);
- } else {
- sampler_rng = rng;
- }
-
- ggml_log_set(sd_ggml_log_callback, nullptr);
-
- model_manager = std::make_shared<ModelManager>();
- model_manager->set_n_threads(n_threads);
- model_manager->set_enable_mmap(enable_mmap);
- model_manager->set_segmented_compute_disabled(disable_segmented_compute);
- model_manager->set_prefetch_disabled(disable_prefetch);
- ModelLoader model_loader;
-
- if (!init_model_loader(model_loader, *configuration)) {
- return false;
- }
-
- version = model_loader.get_sd_version();
- if (version == VERSION_COUNT) {
- LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
- return false;
- } else {
- LOG_INFO("Version: %s ", sd_version_name(version));
- }
-
- if (auto_fit_enabled) {
- if (!sd::backend_fit::derive_backend_specs(model_loader,
- sd_type_to_ggml_type(sd_ctx_params->wtype),
- max_vram_assignment,
- backend_spec,
- params_backend_spec)) {
- return false;
- }
- }
-
- if (!init_backend()) {
- return false;
- }
- {
- std::string error;
- if (!max_vram_assignment.canonicalize_backend_keys(&error)) {
- LOG_ERROR("%s", error.c_str());
- return false;
- }
- }
- if (eager_load && graph_cut_layer_split_active()) {
- LOG_WARN("--eager-load is not supported with graph-cut layer split; weights will be prepared lazily");
- eager_load = false;
- }
-
- diffusion_conv_direct = sd_ctx_params->diffusion_conv_direct;
- return apply_model_update(std::move(model_loader), std::move(configuration), all_runner_groups());
-}
-
-bool StableDiffusionGGML::uses_tae() const {
- return config_->use_tae || version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09;
-}
-
-bool StableDiffusionGGML::tae_preview_only() const {
- return config_->params.tae_preview_only && version != VERSION_SDXS_512_DS && version != VERSION_SDXS_09;
-}
-
-void StableDiffusionGGML::configure_weight_loading() {
- const auto* sd_ctx_params = &config_->params;
- const auto& model_loader = model_manager->loader();
- const auto wtype_stat = model_loader.get_wtype_stat();
- bool have_int8_tensorwise = false;
- bool have_quant_type_rule = false;
- for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) {
- if (tensor_storage.is_int8_tensorwise || tensor_storage.w4_convrot_kind) {
- have_int8_tensorwise = true;
- }
- if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) {
- have_quant_type_rule = true;
- }
- if (have_int8_tensorwise && have_quant_type_rule) {
- break;
- }
- }
-
- if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) {
- bool have_quantized_weight = have_int8_tensorwise;
- for (const auto& [type, _] : wtype_stat) {
- if (ggml_is_quantized(type)) {
- have_quantized_weight = true;
- break;
- }
- }
- // Avoid full-model LoRA merge buffers on constrained setups.
- const bool params_offloaded = params_backend_for(SDBackendModule::DIFFUSION) != backend_for(SDBackendModule::DIFFUSION);
- const bool streaming_constrained = params_offloaded ||
- backend_manager.params_backend_is_disk(SDBackendModule::DIFFUSION);
-
- if (have_quant_type_rule || have_quantized_weight || streaming_constrained || row_split_active()) {
- apply_lora_immediately = false;
- } else {
- apply_lora_immediately = true;
- }
- } else if (sd_ctx_params->lora_apply_mode == LORA_APPLY_IMMEDIATELY) {
- if (have_int8_tensorwise) {
- LOG_WARN(
- "ComfyUI INT8/packed INT4 weights do not support the immediately LoRA apply mode; "
- "using at_runtime instead");
- apply_lora_immediately = false;
- } else if (row_split_active()) {
- LOG_WARN(
- "row-split tensors do not support the immediately LoRA apply mode; "
- "LoRAs will not be applied to them (use --lora-apply-mode at_runtime)");
- apply_lora_immediately = false;
- } else {
- apply_lora_immediately = true;
- }
- } else {
- apply_lora_immediately = false;
- }
-
- bool needs_writable_mmap = enable_mmap && apply_lora_immediately;
- model_manager->set_writable_mmap(needs_writable_mmap);
- if (enable_mmap && apply_lora_immediately) {
- LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap");
- }
- model_manager->prepare_file_io();
- load_alphas_cumprod();
-}
-
-sd::model_builders::Context StableDiffusionGGML::model_build_context() {
- return {config_->params, version, model_manager->loader().get_tensor_storage_map(), backend_manager, model_manager};
-}
-
-bool StableDiffusionGGML::build_core_runners() {
- sd::model_builders::CoreRunners runners;
- if (!sd::model_builders::build_core_runners(model_build_context(), runners)) {
- return false;
- }
- cond_stage_model = std::move(runners.conditioner);
- diffusion_model = std::move(runners.diffusion);
- high_noise_diffusion_model = std::move(runners.high_noise_diffusion);
- clip_vision = std::move(runners.clip_vision);
- ip_adapter = std::move(runners.ip_adapter);
- audio_encoder = std::move(runners.audio_encoder);
-
- cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::TE));
- diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::DIFFUSION));
- if (high_noise_diffusion_model) {
- high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::DIFFUSION));
- }
- if (clip_vision) {
- clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::CLIP_VISION));
- }
- if (audio_encoder) {
- audio_encoder->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::AUDIO_ENCODER));
- }
- return register_runner_params(ModelComponent::Conditioner, cond_stage_model, SDBackendModule::TE) &&
- register_runner_params(ModelComponent::Diffusion, diffusion_model, SDBackendModule::DIFFUSION) &&
- register_runner_params(ModelComponent::HighNoiseDiffusion, high_noise_diffusion_model, SDBackendModule::DIFFUSION) &&
- register_runner_params(ModelComponent::CLIPVision, clip_vision, SDBackendModule::CLIP_VISION) &&
- register_runner_params(ModelComponent::IPAdapter, ip_adapter, SDBackendModule::DIFFUSION) &&
- register_runner_params(ModelComponent::AudioEncoder, audio_encoder, SDBackendModule::AUDIO_ENCODER);
-}
-
-bool StableDiffusionGGML::build_vae_runners() {
- sd::model_builders::VAEOptions options;
- options.use_tae = uses_tae();
- options.tae_preview_only = tae_preview_only();
- options.use_audio_vae = config_->use_audio_vae;
- options.external_vae_is_invalid = external_vae_is_invalid;
- sd::model_builders::VAERunners runners;
- if (!sd::model_builders::build_vae_runners(model_build_context(), options, runners)) {
- return false;
- }
- first_stage_model = std::move(runners.vae);
- preview_vae = std::move(runners.preview);
- audio_vae_model = std::move(runners.audio);
-
- first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::VAE));
- if (preview_vae) {
- preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::VAE));
- }
- return register_runner_params(ModelComponent::VAE, first_stage_model, SDBackendModule::VAE) &&
- register_runner_params(ModelComponent::PreviewVAE, preview_vae, SDBackendModule::VAE) &&
- register_runner_params(ModelComponent::AudioVAE, audio_vae_model, SDBackendModule::VAE);
-}
-
-bool StableDiffusionGGML::build_control_net_runner() {
- if (config_->control_net_file == 0) {
- return true;
- }
- if (sd_version_is_z_image(version)) {
- auto z_image = std::dynamic_pointer_cast<ZImage::ZImageRunner>(diffusion_model);
- if (z_image == nullptr || !z_image->control_enabled()) {
- LOG_WARN("z-image control net file present but the diffusion model has no control weights; ignoring");
- return true;
- }
- std::map<std::string, ggml_tensor*> control_tensors;
- z_image->get_control_param_tensors(control_tensors);
- if (control_tensors.empty()) {
- LOG_ERROR("z-image control net has no param tensors to register");
- return false;
- }
- ModelManager::ResidencyMode residency_mode = backend_manager.params_backend_is_disk(SDBackendModule::DIFFUSION)
- ? ModelManager::ResidencyMode::Disk
- : ModelManager::ResidencyMode::ParamBackend;
- return model_manager->register_param_tensors(ModelComponent::ControlNet,
- std::move(control_tensors),
- residency_mode,
- backend_for(SDBackendModule::DIFFUSION),
- params_backend_for(SDBackendModule::DIFFUSION));
- }
- if (!sd::model_builders::build_control_net_runner(model_build_context(), control_net)) {
- return false;
- }
- control_net->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::CONTROL_NET));
- return register_runner_params(ModelComponent::ControlNet, control_net, SDBackendModule::CONTROL_NET);
-}
-
-bool StableDiffusionGGML::build_extension_runners() {
- GenerationExtensionInitContext extension_ctx{
- &config_->params,
- version,
- model_manager->loader().get_tensor_storage_map(),
- config_->photomaker_source_available,
- model_manager,
- n_threads,
- [this](SDBackendModule module) { return ensure_backend_pair(module); },
- [this](SDBackendModule module) { return backend_for(module); },
- [this](SDBackendModule module) { return params_backend_for(module); },
- };
- if (!sd::model_builders::build_extension_runners(extension_ctx, generation_extensions)) {
- return false;
- }
- for (auto& extension : generation_extensions) {
- if (!register_runner_params(extension->component(), extension, SDBackendModule::PHOTOMAKER)) {
- return false;
- }
- }
- return true;
-}
-
-bool StableDiffusionGGML::flash_attention_for(SDBackendModule module) {
- const auto& params = config_->params;
- const std::string mode = flash_attn_assignment.get(module);
- if (mode == "on") {
- return true;
- }
- if (mode == "off") {
- return false;
- }
- if (strlen(SAFE_STR(params.flash_attn_mode)) == 0) {
- if (params.flash_attn) {
- return true;
- }
- if (params.diffusion_flash_attn) {
- return module == SDBackendModule::DIFFUSION;
- }
- }
- auto backends = backend_manager.runtime_backends(module);
- bool enabled = !backends.empty();
- bool any_enabled = false;
- for (auto backend : backends) {
- auto it = flash_attn_by_backend.find(backend);
- if (it == flash_attn_by_backend.end()) {
- it = flash_attn_by_backend.emplace(backend, sd_probe_flash_attention(backend, n_threads)).first;
- }
- enabled = enabled && it->second;
- any_enabled = any_enabled || it->second;
- }
- if (any_enabled && !enabled) {
- LOG_INFO("flash attention auto: %s -> off (compute devices disagree)", sd_backend_module_name(module));
- }
- return enabled;
-}
-
-bool StableDiffusionGGML::conv_direct_for(SDBackendModule module, bool conv3d) {
- const auto& params = config_->params;
- const std::string mode = conv_direct_assignment.get(module);
- if (mode == "on" || mode == "off") {
- return mode == "on";
- }
- if (strlen(SAFE_STR(params.conv_direct_mode)) == 0 &&
- (module == SDBackendModule::VAE ? params.vae_conv_direct : params.diffusion_conv_direct)) {
- return true;
- }
- auto& by_backend = conv3d ? conv3d_direct_by_backend : conv2d_direct_by_backend;
- auto backends = backend_manager.runtime_backends(module);
- bool enabled = !backends.empty();
- bool any_enabled = false;
- for (auto backend : backends) {
- auto it = by_backend.find(backend);
- if (it == by_backend.end()) {
- it = by_backend.emplace(backend, sd_probe_conv_direct(backend, conv3d)).first;
- }
- enabled = enabled && it->second;
- any_enabled = any_enabled || it->second;
- }
- if (any_enabled && !enabled) {
- LOG_INFO("conv%dd direct auto: %s -> off (compute devices disagree)", conv3d ? 3 : 2, sd_backend_module_name(module));
- }
- return enabled;
-}
-
-bool StableDiffusionGGML::validate_and_load_runners() {
- const auto* sd_ctx_params = &config_->params;
- const bool use_tae = uses_tae();
- const bool tae_preview_only = this->tae_preview_only();
- cond_stage_model->set_flash_attention_enabled(flash_attention_for(SDBackendModule::TE));
- if (clip_vision) {
- clip_vision->set_flash_attention_enabled(flash_attention_for(SDBackendModule::CLIP_VISION));
- }
- if (first_stage_model || preview_vae || audio_vae_model) {
- const bool enabled = flash_attention_for(SDBackendModule::VAE);
- if (first_stage_model) {
- first_stage_model->set_flash_attention_enabled(enabled);
- }
- if (preview_vae) {
- preview_vae->set_flash_attention_enabled(enabled);
- }
- if (audio_vae_model) {
- audio_vae_model->set_flash_attention_enabled(enabled);
- }
- }
- const bool diffusion_fa = flash_attention_for(SDBackendModule::DIFFUSION);
- diffusion_model->set_flash_attention_enabled(diffusion_fa);
- if (high_noise_diffusion_model) {
- high_noise_diffusion_model->set_flash_attention_enabled(diffusion_fa);
- }
- if (ip_adapter) {
- ip_adapter->set_flash_attention_enabled(diffusion_fa);
- }
- if (control_net) {
- control_net->set_flash_attention_enabled(flash_attention_for(SDBackendModule::CONTROL_NET));
- }
- if (audio_encoder) {
- audio_encoder->set_flash_attention_enabled(flash_attention_for(SDBackendModule::AUDIO_ENCODER));
- }
- if (sd_ctx_params->sage_attn && !set_sage_attention_enabled(true)) {
- return false;
- }
- const auto set_conv_direct = [this](SDBackendModule module, const std::vector<GGMLRunner*>& runners) {
- const bool conv2d = conv_direct_for(module, false);
- const bool conv3d = conv_direct_for(module, true);
- for (auto runner : runners) {
- if (runner != nullptr) {
- runner->set_conv2d_direct_enabled(conv2d);
- runner->set_conv3d_direct_enabled(conv3d);
- }
- }
- };
- set_conv_direct(SDBackendModule::DIFFUSION, {diffusion_model.get(), high_noise_diffusion_model.get()});
- if (first_stage_model || preview_vae || audio_vae_model) {
- set_conv_direct(SDBackendModule::VAE, {first_stage_model.get(), preview_vae.get(), audio_vae_model.get()});
- }
- if (control_net) {
- set_conv_direct(SDBackendModule::CONTROL_NET, {control_net.get()});
- }
- LOG_VERBOSE("validating model metadata");
-
- std::set<std::string> ignore_tensors;
- if (version == VERSION_MING_IMAGE) {
- ignore_tensors.insert("text_encoders.llm.vision.");
- ignore_tensors.insert("text_encoders.llm.linear_proj.");
- ignore_tensors.insert("text_encoders.llm.backbone.lm_head.");
- }
- if (use_tae && !tae_preview_only) {
- ignore_tensors.insert("first_stage_model.");
- }
- for (auto& extension : generation_extensions) {
- extension->add_ignore_tensors(ignore_tensors);
- }
- ignore_tensors.insert("model.diffusion_model.__x0__");
- ignore_tensors.insert("model.diffusion_model.__32x32__");
- ignore_tensors.insert("model.diffusion_model.__index_timestep_zero__");
-
- if (audio_encoder != nullptr) {
- // These wav2vec2 tensors are unused during feature extraction.
- ignore_tensors.insert("wav2vec2.lm_head.");
- ignore_tensors.insert("wav2vec2.masked_spec_embed");
- }
-
- if (audio_vae_model) {
- if (!sd_version_is_minimax_h3(version)) {
- ignore_tensors.insert("audio_vae.encoder");
- }
- }
- if (version == VERSION_OVIS_IMAGE) {
- ignore_tensors.insert("text_encoders.llm.vision_model.");
- ignore_tensors.insert("text_encoders.llm.visual_tokenizer.");
- ignore_tensors.insert("text_encoders.llm.vte.");
- }
- if (version == VERSION_SVD) {
- ignore_tensors.insert("conditioner.embedders.3");
- }
- if (sd_version_is_ernie_image(version)) {
- ignore_tensors.insert("text_encoders.llm.vision_tower.");
- ignore_tensors.insert("text_encoders.llm.multi_modal_projector.");
- }
- if (sd_version_is_lens(version)) {
- ignore_tensors.insert("text_encoders.llm.tokenizer_json");
- ignore_tensors.insert("text_encoders.llm.model.layers.0.mlp.experts.gate_up_proj.weight_scale_2");
- ignore_tensors.insert("text_encoders.llm.model.layers.0.mlp.experts.down_proj.weight_scale_2");
- }
- if (sd_version_is_ideogram4(version)) {
- ignore_tensors.insert("text_encoders.llm.lm_head.");
- ignore_tensors.insert("text_encoders.llm.visual.");
- ignore_tensors.insert("text_encoders.llm.vision_model.");
- ignore_tensors.insert("text_encoders.llm.tokenizer_json");
- }
- if (version == VERSION_HIDREAM_O1) {
- ignore_tensors.insert("lm_head.");
- ignore_tensors.insert("model.visual.deepstack_merger_list.");
- }
-
- model_manager->set_common_ignore_tensors(ignore_tensors);
- if (!model_manager->validate_registered_tensors()) {
- LOG_ERROR("model metadata validation failed");
- return false;
- }
-
- if (eager_load) {
- if (!model_manager->load_all_params_eagerly()) {
- LOG_ERROR("model params eager load failed");
- return false;
- }
- LOG_VERBOSE("model metadata validated; weights pre-loaded to params backend");
- } else {
- LOG_VERBOSE("model metadata validated; weights will be prepared lazily");
- }
-
- {
- size_t text_encoder_params_mem_size = model_manager->registered_params_size({ModelComponent::Conditioner});
- size_t unet_params_mem_size = model_manager->registered_params_size({ModelComponent::Diffusion, ModelComponent::HighNoiseDiffusion});
- size_t vae_params_mem_size = model_manager->registered_params_size(runner_components().at(RunnerGroup::VAE));
- size_t control_net_params_mem_size = model_manager->registered_params_size({ModelComponent::ControlNet});
- size_t extension_params_mem_size = model_manager->registered_params_size(runner_components().at(RunnerGroup::Extensions));
- size_t total_params_ram_size = 0;
- size_t total_params_vram_size = 0;
- auto add_params_memory = [&](size_t size, SDBackendModule module) {
- if (size == 0) {
- return true;
- }
- ggml_backend_t module_backend = params_backend_for(module);
- if (module_backend == nullptr) {
- return false;
- }
- if (sd_backend_is_cpu(module_backend)) {
- total_params_ram_size += size;
- } else {
- total_params_vram_size += size;
- }
- return true;
- };
- auto params_memory_location = [&](size_t size, SDBackendModule module) {
- if (size == 0) {
- return "N/A";
- }
- ggml_backend_t module_backend = params_backend_for(module);
- if (module_backend == nullptr) {
- return "N/A";
- }
- return sd_backend_is_cpu(module_backend) ? "RAM" : "VRAM";
- };
-
- if (!add_params_memory(text_encoder_params_mem_size, SDBackendModule::TE) ||
- !add_params_memory(extension_params_mem_size, SDBackendModule::PHOTOMAKER) ||
- !add_params_memory(unet_params_mem_size, SDBackendModule::DIFFUSION) ||
- !add_params_memory(vae_params_mem_size, SDBackendModule::VAE) ||
- !add_params_memory(control_net_params_mem_size, SDBackendModule::CONTROL_NET)) {
- return false;
- }
-
- size_t total_params_size = total_params_ram_size + total_params_vram_size;
- LOG_INFO(
- "total params memory size = %.2fMB (VRAM %.2fMB, RAM %.2fMB): "
- "text_encoders %.2fMB(%s), diffusion_model %.2fMB(%s), vae %.2fMB(%s), controlnet %.2fMB(%s), extensions %.2fMB(%s)",
- total_params_size / 1024.0 / 1024.0,
- total_params_vram_size / 1024.0 / 1024.0,
- total_params_ram_size / 1024.0 / 1024.0,
- text_encoder_params_mem_size / 1024.0 / 1024.0,
- params_memory_location(text_encoder_params_mem_size, SDBackendModule::TE),
- unet_params_mem_size / 1024.0 / 1024.0,
- params_memory_location(unet_params_mem_size, SDBackendModule::DIFFUSION),
- vae_params_mem_size / 1024.0 / 1024.0,
- params_memory_location(vae_params_mem_size, SDBackendModule::VAE),
- control_net_params_mem_size / 1024.0 / 1024.0,
- params_memory_location(control_net_params_mem_size, SDBackendModule::CONTROL_NET),
- extension_params_mem_size / 1024.0 / 1024.0,
- params_memory_location(extension_params_mem_size, SDBackendModule::PHOTOMAKER));
- }
- return true;
-}
-
-bool StableDiffusionGGML::build_denoiser() {
- const auto* sd_ctx_params = &config_->params;
- const auto& model_loader = model_manager->loader();
- const auto& tensor_storage_map = model_loader.get_tensor_storage_map();
- denoiser = std::make_shared<CompVisDenoiser>();
- default_flow_shift = INFINITY;
- prediction_t pred_type = sd_ctx_params->prediction;
-
- if (pred_type == PREDICTION_COUNT) {
- const auto* definition = find_model_definition(version);
- if (!definition)
- return false;
- const auto& policy = definition->sampling;
- pred_type = policy.default_prediction;
- default_flow_shift = policy.flow_shift_from_weights ? policy.flow_shift_from_weights(tensor_storage_map) : policy.default_flow_shift;
- if (policy.probe_v_prediction) {
- pred_type = is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version)) ? V_PRED : EPS_PRED;
- } else if (policy.prediction_from_weights) {
- pred_type = policy.prediction_from_weights(tensor_storage_map);
- }
- }
-
- switch (pred_type) {
- case EPS_PRED:
- LOG_INFO("running in eps-prediction mode");
- break;
- case V_PRED:
- LOG_INFO("running in v-prediction mode");
- denoiser = std::make_shared<CompVisVDenoiser>();
- break;
- case EDM_V_PRED:
- LOG_INFO("running in v-prediction EDM mode");
- denoiser = std::make_shared<EDMVDenoiser>();
- break;
- case FLOW_PRED: {
- LOG_INFO("running in FLOW mode");
- denoiser = std::make_shared<DiscreteFlowDenoiser>();
- break;
- }
- case FLUX_FLOW_PRED: {
- LOG_INFO("running in Flux FLOW mode");
- denoiser = std::make_shared<FluxFlowDenoiser>();
- break;
- }
- case FLUX3_FLOW_PRED: {
- LOG_INFO("running in Flux.3 FLOW mode");
- denoiser = std::make_shared<Flux3FlowDenoiser>();
- break;
- }
- case H3_AV_FLOW_PRED: {
- if (version == VERSION_MINIMAX_H3_IMAGE) {
- LOG_INFO("running in MiniMax H3 Image FLOW mode");
- denoiser = std::make_shared<MiniMaxH3ImageDenoiser>(MiniMaxH3::pdd_sigmas);
- break;
- }
- LOG_INFO("running in MiniMax H3 AV FLOW mode");
- denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, version == VERSION_P_H3_IMAGE ? 1.f : 3.f, get_latent_channel());
- break;
- }
- case MING_IMAGE_FLOW_PRED: {
- LOG_INFO("running in Ming-Image FLOW mode");
- denoiser = std::make_shared<MingImageFlowDenoiser>();
- break;
- }
- case SEFI_FLOW_PRED: {
- LOG_INFO("running in SeFi-Image dual-time FLOW mode");
- denoiser = std::make_shared<SefiFlowDenoiser>();
- break;
- }
- case MINIT2I_FLOW_PRED: {
- LOG_INFO("running in MiniT2I FLOW mode");
- denoiser = std::make_shared<MiniT2IFlowDenoiser>();
- break;
- }
- case SENSENOVA_U1_FLOW_PRED: {
- LOG_INFO("running in SenseNova U1.5 FLOW mode");
- denoiser = std::make_shared<SenseNovaU1FlowDenoiser>(default_flow_shift);
- break;
- }
- default: {
- LOG_ERROR("Unknown predition type %i", pred_type);
- return false;
- }
- }
-
- refresh_compvis_denoiser_sigmas();
- return true;
-}
-
-bool StableDiffusionGGML::build_runners(const RunnerGroups& groups) {
- const auto& model_loader = model_manager->loader();
- std::map<ggml_type, uint32_t> wtype_stat = model_loader.get_wtype_stat();
- std::map<ggml_type, uint32_t> conditioner_wtype_stat = model_loader.get_conditioner_wtype_stat();
- std::map<ggml_type, uint32_t> diffusion_model_wtype_stat = model_loader.get_diffusion_model_wtype_stat();
- std::map<ggml_type, uint32_t> vae_wtype_stat = model_loader.get_vae_wtype_stat();
-
- auto wtype_stat_to_str = [](const std::map<ggml_type, uint32_t>& m, int key_width = 8, int value_width = 5) -> std::string {
- std::ostringstream oss;
- bool first = true;
- for (const auto& [type, count] : m) {
- if (!first)
- oss << "|";
- first = false;
- oss << std::right << std::setw(key_width) << ggml_type_name(type)
- << ": "
- << std::left << std::setw(value_width) << count;
- }
- return oss.str();
- };
-
- LOG_INFO("Weight type stat: %s", wtype_stat_to_str(wtype_stat).c_str());
- LOG_INFO("Conditioner weight type stat: %s", wtype_stat_to_str(conditioner_wtype_stat).c_str());
- LOG_INFO("Diffusion model weight type stat: %s", wtype_stat_to_str(diffusion_model_wtype_stat).c_str());
- LOG_INFO("VAE weight type stat: %s", wtype_stat_to_str(vae_wtype_stat).c_str());
-
- LOG_DEBUG("ggml tensor size = %d bytes", (int)sizeof(ggml_tensor));
-
- configure_weight_loading();
- for (auto group : groups) {
- bool success = false;
- switch (group) {
- case RunnerGroup::Core:
- success = build_core_runners();
- break;
- case RunnerGroup::VAE:
- success = build_vae_runners();
- break;
- case RunnerGroup::ControlNet:
- success = build_control_net_runner();
- break;
- case RunnerGroup::Extensions:
- success = build_extension_runners();
- break;
- }
- if (!success) {
- return false;
- }
- }
- if (!validate_and_load_runners()) {
- return false;
- }
- return groups.count(RunnerGroup::Core) == 0 || build_denoiser();
-}
-
-bool StableDiffusionGGML::is_using_v_parameterization_for_sd2(bool is_inpaint) {
- struct RunnerEndOnExit {
- GGMLRunner* runner = nullptr;
- ~RunnerEndOnExit() {
- if (runner != nullptr) {
- runner->runner_end();
- }
- }
- };
- RunnerEndOnExit diffusion_runner_end{diffusion_model.get()};
-
- sd::Tensor<float> x_t = sd::full<float>({8, 8, 4, 1}, 0.5f);
- sd::Tensor<float> c = sd::full<float>({1024, 2, 1, 1}, 0.5f);
- sd::Tensor<float> steps = sd::full<float>({1}, 999.0f);
- sd::Tensor<float> concat;
- if (is_inpaint) {
- concat = sd::zeros<float>({8, 8, 5, 1});
- }
-
- int64_t t0 = ggml_time_ms();
- sd::Tensor<float> out;
- DiffusionParams diffusion_params;
- diffusion_params.x = &x_t;
- diffusion_params.timesteps = &steps;
- diffusion_params.context = &c;
- diffusion_params.extra = UNetDiffusionExtra{};
- if (!concat.empty()) {
- diffusion_params.c_concat = &concat;
- }
- auto out_opt = diffusion_model->compute(n_threads, diffusion_params);
- GGML_ASSERT(!out_opt.empty());
- out = std::move(out_opt);
-
- double result = static_cast<double>((out - x_t).mean());
- int64_t t1 = ggml_time_ms();
- LOG_VERBOSE("check is_using_v_parameterization_for_sd2, taking %.2fs", (t1 - t0) * 1.0f / 1000);
- return result < -1;
-}
-
-std::string StableDiffusionGGML::lora_log_id(const ModelManager::LoraSpec& lora) {
- return lora.is_high_noise ? "|high_noise|" + lora.path : lora.path;
-}
-
-std::shared_ptr<LoraModel> StableDiffusionGGML::load_lora_model(const ModelManager::LoraSpec& lora_spec,
- SDBackendModule module,
- LoraModel::filter_t module_filter) {
- if (!ensure_backend_pair(module)) {
- return nullptr;
- }
- if (lora_spec.is_high_noise) {
- LOG_VERBOSE("high noise lora: %s", lora_spec.path.c_str());
- }
- const auto mode = backend_manager.params_backend_is_disk(module)
- ? ModelManager::ResidencyMode::Disk
- : ModelManager::ResidencyMode::ParamBackend;
- auto lora = std::make_shared<LoraModel>(lora_log_id(lora_spec), backend_for(module), params_backend_for(module),
- model_manager, lora_spec.file_id, version, mode,
- backend_manager.params_backend_follows_runtime(module));
- LoraModel::filter_t lora_tensor_filter = module_filter;
- if (!lora_spec.tensor_name_prefix_filter.empty()) {
- lora_tensor_filter = [module_filter, prefix = lora_spec.tensor_name_prefix_filter](const std::string& tensor_name) {
- return starts_with(tensor_name, prefix) && (!module_filter || module_filter(tensor_name));
- };
- }
- if (!lora->init_params(n_threads, lora_tensor_filter)) {
- LOG_WARN("load lora tensors from %s failed", lora_spec.path.c_str());
- return nullptr;
- }
-
- lora->multiplier = lora_spec.multiplier;
- return lora;
-}
-
-void StableDiffusionGGML::clear_lora_adapters() {
- if (cond_stage_model) {
- cond_stage_model->set_weight_adapter(nullptr);
- }
- if (diffusion_model) {
- diffusion_model->set_weight_adapter(nullptr);
- }
- if (high_noise_diffusion_model) {
- high_noise_diffusion_model->set_weight_adapter(nullptr);
- }
- if (first_stage_model) {
- first_stage_model->set_weight_adapter(nullptr);
- }
-}
-
-std::vector<std::shared_ptr<LoraModel>> StableDiffusionGGML::load_runtime_loras_for_module(const std::vector<ModelManager::LoraSpec>& loras,
- const std::set<std::string>& model_tensor_names,
- SDBackendModule module,
- LoraModel::filter_t module_filter,
- bool& success,
- std::vector<RuntimeLora>& next_models) {
- std::vector<std::shared_ptr<LoraModel>> module_lora_models;
- for (const auto& lora_spec : loras) {
- auto cached = std::find_if(runtime_lora_models.begin(), runtime_lora_models.end(), [&](const RuntimeLora& entry) {
- return entry.model != nullptr && entry.module == module && entry.matches(lora_spec);
- });
- auto lora = cached == runtime_lora_models.end() ? load_lora_model(lora_spec, module, module_filter)
- : std::move(cached->model);
- if (lora == nullptr) {
- if (lora_spec.required) {
- LOG_ERROR("required lora load failed: %s", lora_spec.path.c_str());
- success = false;
- }
- continue;
- }
- if (lora->lora_tensors.empty()) {
- continue;
- }
-
- lora->preprocess_lora_tensors(model_tensor_names);
- lora->multiplier = lora_spec.multiplier;
- next_models.push_back({lora_spec, module, lora});
- module_lora_models.push_back(std::move(lora));
- }
- return module_lora_models;
-}
-
-bool StableDiffusionGGML::apply_loras_immediately(const std::vector<ModelManager::LoraSpec>& loras) {
- if (model_manager == nullptr) {
- if (!loras.empty()) {
- LOG_WARN("model manager is not available for immediate lora");
- }
- return false;
- }
-
- clear_lora_adapters();
- runtime_lora_models.clear();
-
- if (!loras.empty()) {
- LOG_INFO("apply lora immediately");
- }
- return model_manager->set_loras(loras, version);
-}
-
-bool StableDiffusionGGML::apply_loras_at_runtime(const std::vector<ModelManager::LoraSpec>& loras) {
- if (model_manager != nullptr) {
- if (!model_manager->set_loras({}, version))
- return false;
- }
- clear_lora_adapters();
- if (loras.empty()) {
- runtime_lora_models.clear();
- return true;
- }
-
- bool success = true;
- std::vector<RuntimeLora> next_models;
- std::set<std::string> model_tensor_names;
- if (model_manager != nullptr) {
- model_tensor_names = model_manager->tensor_names();
- }
-
- LOG_INFO("apply lora at runtime");
- if (cond_stage_model) {
- auto lora_tensor_filter = [&](const std::string& tensor_name) {
- if (is_cond_stage_model_name(tensor_name)) {
- return true;
- }
- return false;
- };
- auto cond_stage_lora_models =
- load_runtime_loras_for_module(loras,
- model_tensor_names,
- SDBackendModule::TE,
- lora_tensor_filter, success, next_models);
- // Only attach the adapter when there are LoRAs targeting the cond_stage model.
- // An empty MultiLoraAdapter still routes every linear/conv through
- // forward_with_lora() instead of the direct kernel path — slower for no benefit.
- if (!cond_stage_lora_models.empty()) {
- auto multi_lora_adapter = std::make_shared<MultiLoraAdapter>(cond_stage_lora_models);
- cond_stage_model->set_weight_adapter(multi_lora_adapter);
- }
- }
- if (diffusion_model) {
- auto lora_tensor_filter = [&](const std::string& tensor_name) {
- if (is_diffusion_model_name(tensor_name)) {
- return true;
- }
- return false;
- };
- auto diffusion_lora_models =
- load_runtime_loras_for_module(loras,
- model_tensor_names,
- SDBackendModule::DIFFUSION,
- lora_tensor_filter, success, next_models);
- if (!diffusion_lora_models.empty()) {
- auto multi_lora_adapter = std::make_shared<MultiLoraAdapter>(diffusion_lora_models);
- diffusion_model->set_weight_adapter(multi_lora_adapter);
- if (high_noise_diffusion_model) {
- high_noise_diffusion_model->set_weight_adapter(multi_lora_adapter);
- }
- }
- }
-
- if (first_stage_model) {
- auto lora_tensor_filter = [&](const std::string& tensor_name) {
- if (is_first_stage_model_name(tensor_name)) {
- return true;
- }
- return false;
- };
- auto first_stage_lora_models =
- load_runtime_loras_for_module(loras,
- model_tensor_names,
- SDBackendModule::VAE,
- lora_tensor_filter, success, next_models);
- if (!first_stage_lora_models.empty()) {
- auto multi_lora_adapter = std::make_shared<MultiLoraAdapter>(first_stage_lora_models);
- first_stage_model->set_weight_adapter(multi_lora_adapter);
- }
- }
- runtime_lora_models = std::move(next_models);
- return success;
-}
-
-void StableDiffusionGGML::lora_stat() {
- if (!runtime_lora_models.empty()) {
- LOG_INFO("runtime_lora_models:");
- for (auto& lora_model : runtime_lora_models) {
- lora_model.model->stat();
- }
- }
-}
-
-bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_count) {
- generation_progress.report("preparing", "loras");
- std::vector<ModelManager::LoraSpec> all_loras;
- all_loras.reserve(lora_count);
- for (uint32_t i = 0; i < lora_count; i++) {
- std::string lora_id = SAFE_STR(loras[i].path);
- ModelManager::LoraSpec lora_spec;
- lora_spec.path = lora_id;
- lora_spec.multiplier = loras[i].multiplier;
- lora_spec.is_high_noise = loras[i].is_high_noise;
- all_loras.push_back(std::move(lora_spec));
- if (loras[i].is_high_noise) {
- lora_id = "|high_noise|" + lora_id;
- }
- LOG_VERBOSE("lora %s:%.2f", lora_id.c_str(), loras[i].multiplier);
- }
-
- for (auto& extension : generation_extensions) {
- extension->collect_loras(all_loras);
- }
-
- int64_t t0 = ggml_time_ms();
- end_runners();
- clear_lora_adapters();
- if (!model_manager->prepare_lora_sources(all_loras)) {
- conditioning_cache_->clear();
- return false;
- }
- if (!std::equal(all_loras.begin(), all_loras.end(),
- conditioning_loras_.begin(), conditioning_loras_.end(),
- [](const ModelManager::LoraSpec& a, const ModelManager::LoraSpec& b) {
- return a.file_id == b.file_id && a.file_revision == b.file_revision &&
- a.multiplier == b.multiplier && a.is_high_noise == b.is_high_noise &&
- a.tensor_name_prefix_filter == b.tensor_name_prefix_filter;
- })) {
- conditioning_cache_->clear();
- }
- runtime_lora_models.erase(std::remove_if(runtime_lora_models.begin(), runtime_lora_models.end(), [&](const RuntimeLora& entry) {
- return std::none_of(all_loras.begin(), all_loras.end(), [&](const ModelManager::LoraSpec& spec) {
- return entry.matches(spec);
- });
- }),
- runtime_lora_models.end());
- const bool success = apply_lora_immediately ? apply_loras_immediately(all_loras)
- : apply_loras_at_runtime(all_loras);
- if (!success) {
- conditioning_cache_->clear();
- clear_lora_adapters();
- runtime_lora_models.clear();
- return false;
- }
- runner_state_.catalog_revision = model_manager->loader().revision();
- int64_t t1 = ggml_time_ms();
- if (!all_loras.empty()) {
- LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
- }
- conditioning_loras_ = std::move(all_loras);
- return true;
-}
-
-SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) {
- if (params.prompt_input != nullptr) {
- return conditioning_cache_->get(*cond_stage_model, n_threads, params);
- }
- auto input = sd::parse_prompt_input(params.text);
- auto resolved = params;
- if (sd::prompt_has_images(input)) {
- if (!cond_stage_model->supports_prompt_images()) {
- LOG_ERROR("Inline prompt images require a compatible LLM and vision weights");
- return {};
- }
- if (prompt_image_callback == nullptr) {
- LOG_ERROR("Inline prompt images require sd_set_prompt_image_callback()");
- return {};
- }
- if (!sd::resolve_prompt_images(input, [&](const std::string& source) -> sd::Tensor<float> {
- const sd_tensor_t* tensor = nullptr;
- if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &tensor) || !tensor) {
- LOG_ERROR("Failed to resolve prompt image: %s", source.c_str());
- return {};
- }
- auto image = sd::api::image_view(*tensor);
- auto pixels = sd_image_to_tensor(image);
- if (image.channel <= 2) {
- auto gray = sd::ops::slice(pixels, 2, 0, 1);
- return sd::ops::concat(sd::ops::concat(gray, gray, 2), gray, 2);
- }
- return image.channel == 3 ? pixels : sd::ops::slice(pixels, 2, 0, 3);
- })) {
- return {};
- }
- resolved.prompt_input = &input;
- } else {
- resolved.text = sd::prompt_input_text(input);
- }
- return conditioning_cache_->get(*cond_stage_model, n_threads, resolved);
-}
-
-void StableDiffusionGGML::reset_generation_extensions() {
- for (auto& extension : generation_extensions) {
- extension->reset_runtime_condition();
- }
-}
-
-void StableDiffusionGGML::prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params,
- const sd::PuLIDConfig& pulid_params,
- ConditionerParams& condition_params,
- int total_steps) {
- reset_generation_extensions();
- GenerationExtensionConditionContext ctx{
- cond_stage_model.get(),
- condition_params,
- pm_params,
- pulid_params,
- n_threads,
- total_steps,
- };
-
- for (auto& extension : generation_extensions) {
- extension->prepare_condition(ctx);
- }
-}
-
-sd::Tensor<float> StableDiffusionGGML::get_clip_vision_output(const sd::Tensor<float>& image,
- bool return_pooled,
- int clip_skip,
- bool zero_out_masked) {
- sd::Tensor<float> output;
- if (zero_out_masked) {
- if (return_pooled) {
- output = sd::zeros<float>({clip_vision->vision_model.projection_dim});
- } else {
- output = sd::zeros<float>({clip_vision->vision_model.hidden_size, 257});
- }
- } else {
- auto pixel_values = clip_preprocess(image, clip_vision->vision_model.image_size, clip_vision->vision_model.image_size);
- auto output_opt = clip_vision->compute(n_threads, pixel_values, return_pooled, clip_skip);
- if (output_opt.empty()) {
- LOG_ERROR("clip_vision compute failed");
- return {};
- }
- output = std::move(output_opt);
- }
- return output;
-}
-
-// Returns 50 Hz wav2vec2 states in sd::Tensor layout: [dim, frames, layers].
-sd::Tensor<float> StableDiffusionGGML::get_audio_embedding(const sd::Audio& audio) {
- if (audio_encoder == nullptr) {
- LOG_ERROR("audio encoder model is not loaded");
- return {};
- }
- if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) {
- LOG_ERROR("invalid driving audio");
- return {};
- }
- auto mono = sd::audio::downmix_to_mono(audio.data, audio.sample_count, audio.channels);
- if (mono.empty()) {
- LOG_ERROR("audio mono downmix failed");
- return {};
- }
- mono = sd::audio::resample_audio(mono.data(), mono.size(), audio.sample_rate, 16000);
- if (mono.empty()) {
- LOG_ERROR("audio resample to 16 kHz failed");
- return {};
- }
- return audio_encoder->compute(n_threads, mono);
-}
-
-void StableDiffusionGGML::compute_ip_adapter_tokens(const sd::Image& image, float strength) {
- ip_adapter_tokens = {};
- ip_adapter_uncond_tokens = {};
- ip_adapter_strength = strength;
- if (ip_adapter == nullptr || clip_vision == nullptr || image.data == nullptr) {
- return;
- }
- auto image_tensor = sd_image_to_tensor(image);
- auto embed = ip_adapter->is_plus
- ? get_clip_vision_output(image_tensor, false, 2)
- : get_clip_vision_output(image_tensor, true, -1);
- if (embed.empty()) {
- return;
- }
- ip_adapter_tokens = ip_adapter->compute(n_threads, embed);
- if (ip_adapter_tokens.empty()) {
- LOG_ERROR("IP-Adapter conditional image projection failed");
- return;
- }
- auto uncond_embed = sd::Tensor<float>::zeros_like(embed);
- ip_adapter_uncond_tokens = ip_adapter->compute(n_threads, uncond_embed);
- if (ip_adapter_uncond_tokens.empty()) {
- LOG_ERROR("IP-Adapter unconditional image projection failed");
- ip_adapter_tokens = {};
- return;
- }
- LOG_INFO("IP-Adapter: %lld image tokens, strength %.2f",
- (long long)ip_adapter_tokens.shape()[1], strength);
-}
-
-std::vector<float> StableDiffusionGGML::process_timesteps(const std::vector<float>& timesteps,
- const sd::Tensor<float>& init_latent,
- const sd::Tensor<float>& denoise_mask,
- int step) {
- if (auto sefi_denoiser = std::dynamic_pointer_cast<SefiFlowDenoiser>(denoiser)) {
- int sched_idx = step > 0 ? step - 1 : 0;
- if (sched_idx >= static_cast<int>(sefi_denoiser->tex_timesteps.size())) {
- sched_idx = static_cast<int>(sefi_denoiser->tex_timesteps.size()) - 1;
- }
- return {sefi_denoiser->sem_timesteps[sched_idx],
- sefi_denoiser->tex_timesteps[sched_idx]};
- }
- if (diffusion_model->get_desc() == "Wan2.2-TI2V-5B") {
- int64_t frame_count = init_latent.shape()[2];
- auto new_timesteps = std::vector<float>(static_cast<size_t>(frame_count), timesteps[0]);
-
- if (!denoise_mask.empty() && denoise_mask.dim() >= 4 && denoise_mask.shape()[2] == frame_count) {
- for (int64_t frame = 0; frame < frame_count; ++frame) {
- float value = denoise_mask.dim() == 5 ? denoise_mask.index(0, 0, frame, 0, 0) : denoise_mask.index(0, 0, frame, 0);
- if (value == 0.f) {
- new_timesteps[static_cast<size_t>(frame)] = 0.f;
- }
- }
- }
- return new_timesteps;
- }
- if (diffusion_model->get_desc() == "Wan2.2-S2V-14B") {
- int64_t frame_count = init_latent.shape()[2];
- return std::vector<float>(static_cast<size_t>(frame_count), timesteps[0]);
- } else {
- return timesteps;
- }
-}
-
-std::vector<float> StableDiffusionGGML::process_ltxav_video_timesteps(const std::vector<float>& timesteps,
- const sd::Tensor<float>& init_latent,
- const sd::Tensor<float>& denoise_mask) {
- if (timesteps.empty() || denoise_mask.empty() || init_latent.dim() < 4 || denoise_mask.dim() < 4) {
- return timesteps;
- }
-
- int64_t width = init_latent.shape()[0];
- int64_t height = init_latent.shape()[1];
- int64_t frames = init_latent.shape()[2];
- if (denoise_mask.shape()[0] != width ||
- denoise_mask.shape()[1] != height ||
- denoise_mask.shape()[2] != frames ||
- denoise_mask.shape()[3] < 1) {
- LOG_WARN("unexpected LTXAV denoise mask shape for timestep processing");
- return timesteps;
- }
-
- std::vector<float> video_timesteps(static_cast<size_t>(width * height * frames));
- size_t idx = 0;
- for (int64_t t = 0; t < frames; ++t) {
- for (int64_t h = 0; h < height; ++h) {
- for (int64_t w = 0; w < width; ++w) {
- float mask = denoise_mask.dim() == 5 ? denoise_mask.index(w, h, t, 0, 0)
- : denoise_mask.index(w, h, t, 0);
- video_timesteps[idx++] = mask * timesteps[0];
- }
- }
- }
- return video_timesteps;
-}
-
-void StableDiffusionGGML::preview_image(int step,
- const sd::Tensor<float>& latents,
- enum SDVersion version,
- preview_t preview_mode,
- std::function<void(int, int, const sd_tensor_t* const*, bool, void*)> step_callback,
- void* step_callback_data,
- bool is_noisy) {
- bool is_video = preview_latent_tensor_is_video(latents);
- uint32_t dim = is_video ? static_cast<uint32_t>(latents.shape()[3]) : static_cast<uint32_t>(latents.shape()[2]);
- int channels = get_latent_channel();
- auto _latents = channels != dim ? is_video ? sd::ops::slice(latents, 3, 0, channels)
- : sd::ops::slice(latents, 2, 0, channels)
- : latents;
- if (preview_mode == PREVIEW_PROJ) {
- int patch_sz = 1;
- const float (*latent_rgb_proj)[3] = nullptr;
- float* latent_rgb_bias = nullptr;
- const float* latent_alpha_proj = nullptr;
- float latent_alpha_bias = 1.f;
-
- if (channels == 128) {
- if (sd_version_uses_flux2_vae(version)) {
- latent_rgb_proj = flux2_latent_rgb_proj;
- latent_rgb_bias = flux2_latent_rgb_bias;
- patch_sz = 2;
- } else if (version == VERSION_LTXAV) {
- latent_rgb_proj = ltxav_latent_rgb_proj;
- latent_rgb_bias = ltxav_latent_rgb_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels == 64) {
- if (version == VERSION_QWEN_IMAGE_2_1) {
- latent_rgb_proj = qwen21_latent_rgb_proj;
- latent_rgb_bias = qwen21_latent_rgb_bias;
- latent_alpha_proj = qwen21_latent_alpha_proj;
- latent_alpha_bias = qwen21_latent_alpha_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels == 48) {
- if (sd_version_is_wan(version)) {
- latent_rgb_proj = wan_22_latent_rgb_proj;
- latent_rgb_bias = wan_22_latent_rgb_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels == 24) {
- if (sd_version_is_minimax_h3(version)) {
- latent_rgb_proj = minimax_latent_rgb_proj;
- latent_rgb_bias = minimax_latent_rgb_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels == 16) {
- if (sd_version_is_sd3(version)) {
- latent_rgb_proj = sd3_latent_rgb_proj;
- latent_rgb_bias = sd3_latent_rgb_bias;
- } else if (sd_version_uses_flux_vae(version)) {
- latent_rgb_proj = flux_latent_rgb_proj;
- latent_rgb_bias = flux_latent_rgb_bias;
- } else if (sd_version_uses_wan_vae(version)) {
- latent_rgb_proj = wan_21_latent_rgb_proj;
- latent_rgb_bias = wan_21_latent_rgb_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels == 4) {
- if (sd_version_is_sdxl(version)) {
- latent_rgb_proj = sdxl_latent_rgb_proj;
- latent_rgb_bias = sdxl_latent_rgb_bias;
- } else if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
- latent_rgb_proj = sd_latent_rgb_proj;
- latent_rgb_bias = sd_latent_rgb_bias;
- } else {
- LOG_WARN("No latent to RGB projection known for this model");
- return;
- }
- } else if (channels != 3) {
- LOG_WARN("No latent to RGB projection known for this model (dim = %d)", dim);
- return;
- }
-
- uint32_t frames = is_video ? static_cast<uint32_t>(_latents.shape()[2]) : 1;
- uint32_t img_width = static_cast<uint32_t>(_latents.shape()[0]) * patch_sz;
- uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
-
- uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3;
- std::unique_ptr<uint8_t, decltype(&free)> data(static_cast<uint8_t*>(malloc(size_t(frames) * img_width * img_height * img_channels)), free);
- GGML_ASSERT(data != nullptr);
- preview_latent_video(data.get(), _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
- std::vector<sd::Image> images(frames);
- for (uint32_t i = 0; i < frames; i++) {
- images[i] = {img_width, img_height, img_channels, data.get() + size_t(i) * img_width * img_height * img_channels};
- }
- sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data);
- return;
- }
-
- if (preview_mode == PREVIEW_VAE || preview_mode == PREVIEW_TAE) {
- sd::Tensor<float> vae_latents;
- sd::Tensor<float> decoded;
- if (preview_vae) {
- vae_latents = preview_vae->diffusion_to_vae_latents(_latents);
- decoded = preview_vae->decode(n_threads, vae_latents, vae_tiling_params, is_video, circular_x, circular_y, true);
- } else {
- vae_latents = first_stage_model->diffusion_to_vae_latents(_latents);
- decoded = first_stage_model->decode(n_threads, vae_latents, vae_tiling_params, is_video, circular_x, circular_y, true);
- }
- if (decoded.empty()) {
- LOG_ERROR("preview decode failed at step %d", step);
- return;
- }
-
- is_video = preview_latent_tensor_is_video(decoded);
- uint32_t frames = is_video ? static_cast<uint32_t>(decoded.shape()[2]) : 1;
- auto release_images = [frames](sd::Image* images) {
- for (uint32_t i = 0; i < frames; ++i)
- free(images[i].data);
- free(images);
- };
- std::unique_ptr<sd::Image, decltype(release_images)> images(static_cast<sd::Image*>(calloc(frames, sizeof(sd::Image))), release_images);
- GGML_ASSERT(images != nullptr);
- for (uint32_t i = 0; i < frames; ++i) {
- images.get()[i] = tensor_to_sd_image(decoded, static_cast<int>(i));
- }
-
- sd::api::preview_images(step_callback, step, frames, images.get(), is_noisy, step_callback_data);
- return;
- }
-
- if (preview_mode != PREVIEW_NONE) {
- LOG_WARN("Unsupported preview mode: %d", static_cast<int>(preview_mode));
- }
-}
-
-std::vector<float> StableDiffusionGGML::prepare_sample_timesteps(float sigma,
- int shifted_timestep) {
- float t = denoiser->sigma_to_t(sigma);
- if (shifted_timestep > 0) {
- float shifted_t_float = t * (float(shifted_timestep) / float(TIMESTEPS));
- int64_t shifted_t = static_cast<int64_t>(roundf(shifted_t_float));
- shifted_t = std::max((int64_t)0, std::min((int64_t)(TIMESTEPS - 1), shifted_t));
- LOG_VERBOSE("shifting timestep from %.2f to %" PRId64 " (sigma: %.4f)", t, shifted_t, sigma);
- return std::vector<float>{(float)shifted_t};
- }
- if (sd_version_is_anima(version)) {
- return std::vector<float>{t / static_cast<float>(TIMESTEPS)};
- }
- if (sd_version_is_boogu_image(version)) {
- return std::vector<float>{t / static_cast<float>(TIMESTEPS)};
- }
- if (version == VERSION_HIDREAM_O1) {
- return std::vector<float>{1.0f - (t / static_cast<float>(TIMESTEPS))};
- }
- if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version) || sd_version_is_ideogram4(version) || version == VERSION_MING_IMAGE) {
- return std::vector<float>{1000.f - t};
- }
- return std::vector<float>{t};
-}
-
-void StableDiffusionGGML::adjust_sample_step_scalings(int shifted_timestep,
- const std::vector<float>& timesteps_vec,
- float c_in,
- float* c_skip,
- float* c_out) {
- GGML_ASSERT(c_skip != nullptr);
- GGML_ASSERT(c_out != nullptr);
- if (shifted_timestep <= 0) {
- return;
- }
-
- int64_t shifted_t_idx = static_cast<int64_t>(roundf(timesteps_vec[0]));
- float shifted_sigma = denoiser->t_to_sigma((float)shifted_t_idx);
- std::vector<float> shifted_scaling = denoiser->get_scalings(shifted_sigma);
- float shifted_c_skip = shifted_scaling[0];
- float shifted_c_out = shifted_scaling[1];
- float shifted_c_in = shifted_scaling[2];
-
- *c_skip = shifted_c_skip * c_in / shifted_c_in;
- *c_out = shifted_c_out;
-}
-
-StableDiffusionGGML::SamplePreviewContext StableDiffusionGGML::prepare_sample_preview_context() {
- return SamplePreviewContext{sd_get_preview_callback(),
- sd_get_preview_callback_data(),
- sd_get_preview_mode()};
-}
-
-void StableDiffusionGGML::report_sample_progress(int step,
- size_t total_steps,
- bool terminal_sigma_is_zero,
- int64_t* last_progress_us) {
- if (sd::preview::sample_step_is_complete(step, total_steps, terminal_sigma_is_zero)) {
- int64_t now = ggml_time_us();
- int showstep = std::abs(step);
- float step_seconds = last_progress_us != nullptr && *last_progress_us > 0
- ? (now - *last_progress_us) / 1000000.f
- : 0.f;
- pretty_progress(showstep, (int)total_steps, step_seconds);
- generation_progress.report_sample(showstep, static_cast<int64_t>(total_steps));
- if (last_progress_us != nullptr) {
- *last_progress_us = now;
- }
- }
-}
-
-static sd::Tensor<float> batch_two_condition_tensors(const sd::Tensor<float>& a, const sd::Tensor<float>& b) {
- if (a.empty() || b.empty() || a.dim() != b.dim()) {
- return {};
- }
- if (a.dim() == 1) {
- if (a.shape() != b.shape()) {
- return {};
- }
- auto batched = sd::ops::concat(a, b, 0);
- batched.reshape_({a.shape()[0], 2});
- return batched;
- }
- const int64_t batch_dim = a.dim() - 1;
- for (int64_t d = 0; d < batch_dim; d++) {
- if (a.shape()[d] != b.shape()[d]) {
- return {};
- }
- }
- return sd::ops::concat(a, b, static_cast<size_t>(batch_dim));
-}
-
-bool StableDiffusionGGML::set_control_type(sd_control_type_t type) {
- return control_net == nullptr || control_net->set_control_type(type);
-}
-
-void StableDiffusionGGML::compute_sample_controls(const sd::Tensor<float>& control_image,
- const sd::Tensor<float>& noised_input,
- const sd::Tensor<float>& timesteps_tensor,
- const SDCondition& condition,
- std::vector<sd::Tensor<float>>* controls) {
- GGML_ASSERT(controls != nullptr);
- controls->clear();
- if (control_image.empty() || control_net == nullptr) {
- return;
- }
-
- auto control_result = control_net->compute(n_threads,
- noised_input,
- control_image,
- timesteps_tensor,
- condition.c_crossattn,
- condition.c_vector);
- if (!control_result.has_value()) {
- LOG_ERROR("controlnet compute failed");
- return;
- }
-
- *controls = std::move(*control_result);
-}
-
-sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionModelRunner>& work_diffusion_model,
- bool inverse_noise_scaling,
- const sd::Tensor<float>& init_latent,
- sd::Tensor<float> noise,
- const SDCondition& cond,
- const SDCondition& uncond,
- const SDCondition& img_uncond,
- const sd::Tensor<float>& control_image,
- float control_strength,
- const sd::GuidanceConfig& guidance,
- float eta,
- int shifted_timestep,
- sample_method_t method,
- bool is_flow_denoiser,
- const char* extra_sample_args,
- const std::vector<float>& sigmas,
- const std::vector<sd::Tensor<float>>& ref_latents,
- const RefImageParams& ref_image_params,
- const sd::Tensor<float>& denoise_mask,
- const sd::Tensor<float>& vace_context,
- float vace_strength,
- int audio_length,
- float frame_rate,
- const sd::CacheConfig* cache_params,
- bool preview_final_step,
- const sd::Tensor<float>& video_positions) {
- struct RunnerEndOnExit {
- GGMLRunner* runner = nullptr;
- ~RunnerEndOnExit() {
- if (runner != nullptr) {
- runner->runner_end();
- }
- }
- };
- RunnerEndOnExit sample_diffusion_runner_end{work_diffusion_model.get()};
-
- // These inputs are immutable for this sampling run. Extensions may replace or
- // modify them per step, so those paths need an explicit stability contract first.
- const bool cache_qwen_prefix = version == VERSION_QWEN_IMAGE_2_1 &&
- std::none_of(generation_extensions.begin(), generation_extensions.end(),
- [](const auto& extension) { return extension->is_enabled(); });
- using QwenPrefixInputs = std::tuple<const sd::Tensor<float>*, const sd::Tensor<int32_t>*,
- const std::vector<sd::Tensor<float>>*>;
- std::vector<QwenPrefixInputs> qwen_prefix_inputs;
-
- RunnerEndOnExit sample_control_runner_end{!control_image.empty() && control_net != nullptr ? control_net.get() : nullptr};
-
- const bool apply_denoise_mask = !denoise_mask.empty() &&
- std::any_of(denoise_mask.values().begin(), denoise_mask.values().end(),
- [](float value) { return value != 1.f; });
-
- std::vector<int> skip_layers(guidance.slg.layers, guidance.slg.layers + guidance.slg.layer_count);
- float cfg_scale = guidance.txt_cfg;
- float img_cfg_scale = guidance.img_cfg;
- float slg_scale = guidance.slg.scale;
- bool slg_uncond = sd::guidance::parse_skip_layer_guidance_uncond_arg(extra_sample_args);
-
- std::vector<float> guidance_schedule = sd::guidance::parse_guidance_schedule(extra_sample_args);
- if (!guidance_schedule.empty() && guidance_schedule.size() != sigmas.size() - 1) {
- if (guidance_schedule.size() > sigmas.size()) {
- LOG_WARN("guidance_schedule length (%zu) is greater than number of steps (%zu)", guidance_schedule.size(), sigmas.size() - 1);
- LOG_WARN("truncating guidance_schedule to match step count");
- guidance_schedule.resize(sigmas.size() - 1);
- } else {
- LOG_INFO("padding guidance_schedule with cfg_scale");
- while (guidance_schedule.size() < sigmas.size() - 1) {
- guidance_schedule.push_back(cfg_scale);
- }
- }
- }
-
- if (!guidance_schedule.empty()) {
- std::string schedule_str = "[";
- for (size_t i = 0; i < guidance_schedule.size(); ++i) {
- schedule_str += std::to_string(guidance_schedule[i]);
- if (i < guidance_schedule.size() - 1) {
- schedule_str += ", ";
- }
- }
- schedule_str += "]";
- LOG_VERBOSE("using guidance schedule: %s", schedule_str.c_str());
- }
-
- sd_sample::SampleCacheRuntime cache_runtime = sd_sample::init_sample_cache_runtime(version,
- cache_params,
- denoiser.get(),
- sigmas);
-
- bool needs_uncond_denoised = method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD;
- // Spectrum cache is not supported for CFG++ samplers
- if (needs_uncond_denoised) {
- if (cache_runtime.spectrum_enabled) {
- LOG_WARN("Spectrum cache requested but not supported for CFG++ samplers");
- cache_runtime.spectrum_enabled = false;
- }
- }
-
- size_t steps = sigmas.size() - 1;
- bool terminal_sigma_is_zero = sigmas.back() == 0.f;
- bool has_skiplayer = (slg_scale != 0.0f || slg_uncond) && !skip_layers.empty();
- if (has_skiplayer && !sd_version_is_dit(version)) {
- has_skiplayer = false;
- LOG_WARN("SLG is incompatible with this model type");
- }
- sd::guidance::AdaptiveProjectedGuidanceParams apg_params = sd::guidance::parse_adaptive_projected_guidance_args(extra_sample_args);
- bool use_apg_guidance = sd::guidance::is_adaptive_projected_guidance_enabled(apg_params);
- if (use_apg_guidance) {
- LOG_INFO("using Adaptive Projected Guidance (APG)");
- }
- sd::guidance::ClassifierFreeGuidance classifier_free_guidance(cfg_scale, img_cfg_scale);
- sd::guidance::AdaptiveProjectedGuidance adaptive_projected_guidance(cfg_scale, img_cfg_scale, apg_params);
- const sd::guidance::BaseGuidance& primary_guidance = use_apg_guidance
- ? static_cast<const sd::guidance::BaseGuidance&>(adaptive_projected_guidance)
- : static_cast<const sd::guidance::BaseGuidance&>(classifier_free_guidance);
- sd::guidance::SkipLayerGuidance skip_layer_guidance(has_skiplayer ? skip_layers : std::vector<int>(),
- has_skiplayer ? slg_scale : 0.0f,
- guidance.slg.layer_start,
- guidance.slg.layer_end);
-
- if (version == VERSION_HIDREAM_O1 && !noise.empty()) {
- noise *= eta;
- }
-
- int64_t last_progress_us = ggml_time_us();
- sd_begin_preview_pass(static_cast<int>(steps));
- generation_progress.begin_sample(static_cast<int64_t>(steps),
- work_diffusion_model == high_noise_diffusion_model ? "high_noise" : high_noise_diffusion_model != nullptr ? "low_noise"
- : "");
- SamplePreviewContext preview = prepare_sample_preview_context();
-
- sd::Tensor<float> processed_init_latent = denoiser->process_latent_in(init_latent);
- const sd::Tensor<float>& sampling_init_latent = processed_init_latent.empty()
- ? init_latent
- : processed_init_latent;
- sd::Tensor<float> x_t = !noise.empty()
- ? denoiser->noise_scaling(sigmas[0], noise, sampling_init_latent)
- : sampling_init_latent;
- sd::Tensor<float> denoised = x_t;
-
- auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::guidance::GuiderOutput {
- if (get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_VERBOSE("cancelling generation");
- return {};
- }
-
- if (step == 1 || step == -1) {
- pretty_progress(0, (int)steps, 0);
- last_progress_us = ggml_time_us();
- }
-
- std::vector<float> scaling = denoiser->get_scalings(sigma);
- GGML_ASSERT(scaling.size() == 3);
- float c_skip = scaling[0];
- float c_out = scaling[1];
- float c_in = scaling[2];
-
- bool preview_needed = preview.callback != nullptr &&
- sd::preview::should_preview_sample_step(step,
- steps,
- terminal_sigma_is_zero,
- sd_get_preview_interval(),
- preview_final_step);
-
- std::vector<float> base_timesteps_vec = prepare_sample_timesteps(sigma, shifted_timestep);
- std::vector<float> timesteps_vec = base_timesteps_vec;
- sd::Tensor<float> audio_timesteps_tensor;
- if (sd_version_is_ltxav(version) && !denoise_mask.empty()) {
- timesteps_vec = process_ltxav_video_timesteps(base_timesteps_vec, sampling_init_latent, denoise_mask);
- audio_timesteps_tensor = sd::Tensor<float>({static_cast<int64_t>(base_timesteps_vec.size())}, base_timesteps_vec);
- } else {
- timesteps_vec = process_timesteps(timesteps_vec, sampling_init_latent, denoise_mask, step);
- }
- const std::vector<float>& scaling_timesteps_vec = (sd_version_is_ltxav(version) && !denoise_mask.empty())
- ? base_timesteps_vec
- : timesteps_vec;
- adjust_sample_step_scalings(shifted_timestep, scaling_timesteps_vec, c_in, &c_skip, &c_out);
-
- sd::Tensor<float> timesteps_tensor({static_cast<int64_t>(timesteps_vec.size())}, timesteps_vec);
- sd::Tensor<float> guidance_tensor({1}, std::vector<float>{guidance.distilled_guidance});
- sd::Tensor<float> hunyuan_timestep_r_tensor;
- if (sd_version_is_hunyuan_video(version) && step + 1 < sigmas.size()) {
- hunyuan_timestep_r_tensor = sd::Tensor<float>::from_vector({sigmas[step + 1]});
- }
- sd::Tensor<float> flux3_target_timestep;
- if (version == VERSION_FLUX3) {
- // The sign marks intermediate evaluations; the magnitude is the one-based step.
- const size_t target_index = static_cast<size_t>(std::abs(step));
- GGML_ASSERT(target_index > 0 && target_index < sigmas.size());
- flux3_target_timestep = sd::Tensor<float>::from_vector({sigmas[target_index]});
- }
- sd::Tensor<float> noised_input = x * c_in;
- if (apply_denoise_mask && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
- noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
- }
-
- if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
- cache_runtime.spectrum.predict(&denoised);
- if (apply_denoise_mask) {
- denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
- }
- if (preview_needed && sd_should_preview_denoised()) {
- preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
- }
- report_sample_progress(step, steps, terminal_sigma_is_zero, &last_progress_us);
- sd::guidance::GuiderOutput output;
- output.pred = denoised;
- return output;
- }
-
- if (preview_needed && sd_should_preview_noisy()) {
- preview_image(step, noised_input, version, preview.mode, preview.callback, preview.data, true);
- }
-
- sd::Tensor<float> cond_out;
- sd::Tensor<float> uncond_out;
- sd::Tensor<float> img_uncond_out;
- sd_sample::SampleStepCacheDispatcher step_cache(cache_runtime, step, sigma);
- std::vector<sd::Tensor<float>> controls;
- DiffusionParams diffusion_params;
- diffusion_params.x = &noised_input;
- diffusion_params.timesteps = ×teps_tensor;
- diffusion_params.ref_image_params = ref_image_params;
- sd::guidance::GuidanceInput step_guidance_input;
- step_guidance_input.step = step;
- step_guidance_input.schedule_size = sigmas.size();
- bool is_skiplayer_step = skip_layer_guidance.is_enabled_for_step(step_guidance_input);
-
- compute_sample_controls(control_image,
- noised_input,
- timesteps_tensor,
- cond,
- &controls);
- bool uncond_controls_ready = false;
-
- static const std::vector<sd::Tensor<float>> empty_ref_latents;
- bool uncond_without_ref_latents = !img_uncond.empty() &&
- !ref_latents.empty() &&
- sd_version_supports_ref_latent_img_cfg(version);
-
- auto run_condition = [&](const SDCondition& condition,
- const sd::Tensor<float>* c_concat_override = nullptr,
- const std::vector<int>* local_skip_layers = nullptr,
- const std::vector<sd::Tensor<float>>* ref_latents_override = nullptr,
- bool use_uncond_ip = false) -> sd::Tensor<float> {
- diffusion_params.context = condition.c_crossattn.empty() ? nullptr : &condition.c_crossattn;
- diffusion_params.c_concat = c_concat_override != nullptr ? c_concat_override : (condition.c_concat.empty() ? nullptr : &condition.c_concat);
- diffusion_params.y = condition.c_vector.empty() ? nullptr : &condition.c_vector;
- diffusion_params.ref_latents = ref_latents_override != nullptr ? ref_latents_override : (condition.c_ref_images.empty() ? &ref_latents : &condition.c_ref_images);
-
- if (sd_version_is_unet(version)) {
- int nvf = -1;
- if (config_->animatediff_loaded && noised_input.dim() >= 4 && noised_input.shape()[3] > 1) {
- nvf = static_cast<int>(noised_input.shape()[3]);
- }
- UNetDiffusionExtra unet_extra{nvf, &controls, control_strength};
- const auto& ip_tokens = use_uncond_ip ? ip_adapter_uncond_tokens : ip_adapter_tokens;
- if (!ip_tokens.empty()) {
- unet_extra.ip_context = &ip_tokens;
- unet_extra.ip_scale = ip_adapter_strength;
- }
- diffusion_params.extra = unet_extra;
- } else if (sd_version_is_sd3(version) || version == VERSION_MOSS_IMAGE) {
- diffusion_params.extra = SkipLayerDiffusionExtra{local_skip_layers};
- } else if (sd_version_is_flux(version) || sd_version_is_flux2(version) || sd_version_is_longcat(version) || sd_version_is_sefi_image(version)) {
- diffusion_params.extra = FluxDiffusionExtra{&guidance_tensor,
- local_skip_layers};
- } else if (sd_version_is_flux3(version)) {
- diffusion_params.extra = Flux3DiffusionExtra{flux3_target_timestep.empty() ? nullptr : &flux3_target_timestep};
- } else if (version == VERSION_QWEN_IMAGE_2_1) {
- diffusion_params.extra = QwenImage21DiffusionExtra{&condition.c_token_types};
- } else if (sd_version_is_anima(version)) {
- diffusion_params.extra = AnimaDiffusionExtra{condition.c_t5_ids.empty() ? nullptr : &condition.c_t5_ids,
- condition.c_t5_weights.empty() ? nullptr : &condition.c_t5_weights};
- } else if (sd_version_is_wan(version)) {
- diffusion_params.extra = WanDiffusionExtra{vace_context.empty() ? nullptr : &vace_context,
- vace_strength,
- condition.c_ref_audios.empty() ? nullptr : &condition.c_ref_audios[0]};
- } else if (sd_version_is_hunyuan_video(version)) {
- diffusion_params.extra = HunyuanVideoDiffusionExtra{
- &guidance_tensor,
- condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0],
- condition.c_vector.empty() ? nullptr : &condition.c_vector,
- hunyuan_timestep_r_tensor.empty() ? nullptr : &hunyuan_timestep_r_tensor};
- } else if (version == VERSION_HIDREAM_O1) {
- diffusion_params.extra = HiDreamO1DiffusionExtra{
- condition.c_input_ids.empty() ? nullptr : &condition.c_input_ids,
- condition.c_position_ids.empty() ? nullptr : &condition.c_position_ids,
- condition.c_token_types.empty() ? nullptr : &condition.c_token_types,
- condition.c_vinput_mask.empty() ? nullptr : &condition.c_vinput_mask,
- condition.c_image_embeds.empty() ? nullptr : &condition.c_image_embeds};
- } else if (version == VERSION_MING_IMAGE) {
- diffusion_params.extra = MingImageDiffusionExtra{
- condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
- } else if (sd_version_is_llada_image(version)) {
- diffusion_params.extra = LLaDAImageDiffusionExtra{
- condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
- } else if (sd_version_is_minimax_h3(version)) {
- const size_t target_index = static_cast<size_t>(std::abs(step));
- diffusion_params.extra = MiniMaxH3DiffusionExtra{
- condition.c_token_types.empty() ? nullptr : &condition.c_token_types,
- condition.c_position_ids.empty() ? nullptr : &condition.c_position_ids,
- condition.c_ref_audios.empty() ? nullptr : &condition.c_ref_audios,
- condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
- version == VERSION_MINIMAX_H3_IMAGE ? 0 : (version == VERSION_P_H3_IMAGE ? 2 : audio_length),
- std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
- version == VERSION_P_H3_IMAGE ? 1.f : 3.f,
- target_index > 0 && target_index < sigmas.size() ? sigmas[target_index] : NAN};
- } else if (sd_version_is_ltxav(version)) {
- diffusion_params.extra = LTXAVDiffusionExtra{
- nullptr,
- audio_timesteps_tensor.empty() ? nullptr : &audio_timesteps_tensor,
- audio_length,
- frame_rate,
- video_positions.empty() ? nullptr : &video_positions};
- } else if (sd_version_is_minit2i(version)) {
- diffusion_params.extra = MiniT2IDiffusionExtra{
- condition.c_vector.empty() ? nullptr : &condition.c_vector};
- } else if (sd_version_is_sensenova_u1(version)) {
- diffusion_params.extra = SenseNovaU1DiffusionExtra{
- condition.c_input_ids.empty() ? nullptr : &condition.c_input_ids};
- } else if (sd_version_is_z_image(version)) {
- if (!control_image.empty()) {
- diffusion_params.extra = ZImageDiffusionExtra{&control_image, control_strength};
- }
- } else {
- diffusion_params.extra = std::monostate{};
- }
-
- sd::Tensor<float> cached_output;
- if (step_cache.before_condition(&condition, noised_input, &cached_output)) {
- return std::move(cached_output);
- }
-
- // A re-enabled condition can miss the cache even when the positive pass was reused.
- if (!uncond_controls_ready && !uncond.empty() &&
- (&condition == &uncond || &condition == &img_uncond)) {
- compute_sample_controls(control_image,
- noised_input,
- timesteps_tensor,
- uncond,
- &controls);
- uncond_controls_ready = true;
- }
-
- for (const auto& extension : generation_extensions) {
- extension->before_diffusion(diffusion_params, step);
- }
-
- if (cache_qwen_prefix) {
- auto* extra = std::get_if<QwenImage21DiffusionExtra>(&diffusion_params.extra);
- if (extra != nullptr) {
- auto key = std::make_tuple(diffusion_params.context, extra->image_slots,
- diffusion_params.ref_image_params.pass_to_dit ? diffusion_params.ref_latents : nullptr);
- auto entry = std::find(qwen_prefix_inputs.begin(), qwen_prefix_inputs.end(), key);
- extra->prefix_id = static_cast<uint64_t>(entry - qwen_prefix_inputs.begin()) + 1;
- if (entry == qwen_prefix_inputs.end()) {
- qwen_prefix_inputs.push_back(key);
- }
- }
- }
- auto output_opt = work_diffusion_model->compute(n_threads, diffusion_params);
- if (output_opt.empty()) {
- LOG_ERROR("diffusion model compute failed");
- return sd::Tensor<float>();
- }
-
- step_cache.after_condition(&condition, noised_input, output_opt);
- return output_opt;
- };
-
- auto run_batched_condition = [&](const SDCondition& condition,
- const sd::Tensor<float>* c_concat_override) -> sd::Tensor<float> {
- const sd::Tensor<float>& condition_concat =
- c_concat_override != nullptr ? *c_concat_override : condition.c_concat;
-
- sd::Tensor<float> batched_context = batch_two_condition_tensors(condition.c_crossattn, uncond.c_crossattn);
- sd::Tensor<float> batched_y = batch_two_condition_tensors(condition.c_vector, uncond.c_vector);
- sd::Tensor<float> batched_concat = batch_two_condition_tensors(condition_concat, uncond.c_concat);
- if (!condition.c_crossattn.empty() && batched_context.empty()) {
- return {};
- }
- if ((!condition.c_vector.empty() || !uncond.c_vector.empty()) && batched_y.empty()) {
- return {};
- }
- if ((!condition_concat.empty() || !uncond.c_concat.empty()) && batched_concat.empty()) {
- return {};
- }
-
- std::vector<sd::Tensor<float>> uncond_controls;
- compute_sample_controls(control_image, noised_input, timesteps_tensor, uncond, &uncond_controls);
- if (controls.size() != uncond_controls.size()) {
- return {};
- }
- std::vector<sd::Tensor<float>> batched_controls;
- batched_controls.reserve(controls.size());
- for (size_t i = 0; i < controls.size(); i++) {
- sd::Tensor<float> batched_control = batch_two_condition_tensors(controls[i], uncond_controls[i]);
- if (batched_control.empty()) {
- return {};
- }
- batched_controls.push_back(std::move(batched_control));
- }
-
- sd::Tensor<float> batched_x =
- sd::ops::concat(noised_input, noised_input, static_cast<size_t>(noised_input.dim() - 1));
-
- DiffusionParams batched_params = diffusion_params;
- batched_params.x = &batched_x;
- batched_params.context = batched_context.empty() ? nullptr : &batched_context;
- batched_params.c_concat = batched_concat.empty() ? nullptr : &batched_concat;
- batched_params.y = batched_y.empty() ? nullptr : &batched_y;
- batched_params.ref_latents = nullptr;
- batched_params.extra = UNetDiffusionExtra{1, &batched_controls, control_strength};
-
- sd::Tensor<float> output = work_diffusion_model->compute(n_threads, batched_params);
- if (output.empty()) {
- LOG_ERROR("batched diffusion model compute failed");
- }
- return output;
- };
-
- const SDCondition* positive_condition = &cond;
- const sd::Tensor<float>* c_concat_override = nullptr;
- for (const auto& extension : generation_extensions) {
- const SDCondition& next_condition = extension->before_condition(step, *positive_condition);
- if (&next_condition != positive_condition) {
- positive_condition = &next_condition;
- if (positive_condition != &cond) {
- c_concat_override = cond.c_concat.empty() ? nullptr : &cond.c_concat;
- }
- break;
- }
- }
-
- float effective_guidance_scale = guidance_schedule.empty()
- ? cfg_scale
- : guidance_schedule[guidance_schedule.size() - 1 - step];
-
- float image_guidance_scale = img_cfg_scale;
-
- constexpr float kEpsilon = 1e-5f;
-
- bool skip_uncond = false;
- if (!uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
- if (!img_uncond.empty()) {
- skip_uncond = std::abs(image_guidance_scale - effective_guidance_scale) < kEpsilon;
- } else {
- skip_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
- }
- }
-
- bool skip_img_uncond = false;
- if (!img_uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
- if (!uncond.empty()) {
- skip_img_uncond = std::abs(image_guidance_scale - 1.0f) < kEpsilon;
- } else {
- skip_img_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
- }
- }
-
- const bool batch_cfg_ok = config_->params.batched_cfg &&
- sd_version_is_unet(version) &&
- !uncond.empty() &&
- img_uncond.empty() &&
- !skip_uncond &&
- !cache_runtime.ucache_enabled() &&
- !(is_skiplayer_step && slg_uncond) &&
- ip_adapter_tokens.empty() &&
- ip_adapter_uncond_tokens.empty() &&
- !config_->animatediff_loaded &&
- (noised_input.dim() < 4 || noised_input.shape()[3] <= 1) &&
- std::none_of(generation_extensions.begin(),
- generation_extensions.end(),
- [](const std::shared_ptr<GenerationExtension>& extension) {
- return extension->is_enabled();
- });
-
- if (batch_cfg_ok) {
- sd::Tensor<float> batched_out = run_batched_condition(*positive_condition, c_concat_override);
- if (!batched_out.empty() && batched_out.dim() >= 4 && batched_out.shape()[3] == 2) {
- auto parts = sd::ops::chunk(batched_out, 2, 3);
- cond_out = std::move(parts[0]);
- uncond_out = std::move(parts[1]);
- }
- }
-
- if (cond_out.empty()) {
- cond_out = run_condition(*positive_condition, c_concat_override);
- if (cond_out.empty()) {
- return {};
- }
- }
-
- if (uncond_out.empty() && !uncond.empty()) {
- if (!skip_uncond) {
- const std::vector<int>* uncond_skip_layers = nullptr;
- const std::vector<int> moss_skip_layers = {22};
- if (is_skiplayer_step && slg_uncond) {
- LOG_VERBOSE("Skipping layers at uncond step %d\n", step);
- uncond_skip_layers = &skip_layer_guidance.layers();
- } else if (version == VERSION_MOSS_IMAGE) {
- uncond_skip_layers = &moss_skip_layers;
- }
- uncond_out = run_condition(uncond,
- uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
- uncond_skip_layers,
- nullptr,
- true);
- if (uncond_out.empty()) {
- return {};
- }
- } else {
- step_cache.invalidate_condition(&uncond);
- }
- }
-
- if (!img_uncond.empty()) {
- if (!skip_img_uncond) {
- img_uncond_out = run_condition(img_uncond,
- img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
- nullptr,
- uncond_without_ref_latents ? &empty_ref_latents : nullptr,
- true);
- if (img_uncond_out.empty()) {
- return {};
- }
- } else {
- step_cache.invalidate_condition(&img_uncond);
- }
- }
- sd::guidance::GuidanceInput guidance_input;
- guidance_input.step = step;
- guidance_input.schedule_size = sigmas.size();
- guidance_input.pred_cond = &cond_out;
- guidance_input.pred_uncond = uncond_out.empty() ? nullptr : &uncond_out;
- guidance_input.pred_img_uncond = img_uncond_out.empty() ? nullptr : &img_uncond_out;
-
- sd::guidance::GuiderOutput guided = primary_guidance.forward(guidance_input, {}, effective_guidance_scale);
- if (guided.pred.empty()) {
- return {};
- }
-
- if (is_skiplayer_step && slg_scale != 0.0f) {
- LOG_VERBOSE("Skipping layers at step %d\n", step);
- if (!step_cache.is_step_skipped()) {
- guidance_input.predict_skip_layer = [&]() -> sd::Tensor<float> {
- return run_condition(cond,
- cond.c_concat.empty() ? nullptr : &cond.c_concat,
- &skip_layer_guidance.layers());
- };
- }
- }
-
- guided = skip_layer_guidance.forward(guidance_input, std::move(guided));
- if (guided.pred.empty()) {
- return {};
- }
-
- denoised = guided.pred * c_out + x * c_skip;
- sd::guidance::GuiderOutput output;
- output.pred = denoised;
- if (needs_uncond_denoised) {
- const sd::Tensor<float>& base_uncond = !img_uncond_out.empty()
- ? img_uncond_out
- : (!uncond_out.empty() ? uncond_out : cond_out);
- output.pred_uncond = base_uncond * c_out + x * c_skip;
- }
- if (cache_runtime.spectrum_enabled) {
- cache_runtime.spectrum.update(denoised);
- }
- if (apply_denoise_mask) {
- denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
- }
- if (preview_needed && sd_should_preview_denoised()) {
- preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
- }
- report_sample_progress(step, steps, terminal_sigma_is_zero, &last_progress_us);
- output.pred = denoised;
- return output;
- };
-
- {
- std::ostringstream oss;
- oss << std::setprecision(std::numeric_limits<float>::max_digits10);
- for (size_t i = 0; i < sigmas.size(); ++i) {
- if (i != 0) {
- oss << ",";
- }
- oss << sigmas[i];
- }
- LOG_DEBUG("sigmas: %s", oss.str().c_str());
- }
-
- auto x0_opt = sample_k_diffusion(method, denoise, x_t, sigmas, sampler_rng, eta, is_flow_denoiser, extra_sample_args, denoiser);
- if (x0_opt.empty()) {
- LOG_ERROR("Diffusion model sampling failed");
- if (control_net) {
- control_net->free_control_ctx();
- }
- return {};
- }
-
- auto x0 = std::move(x0_opt);
- sd_sample::log_sample_cache_summary(cache_runtime, steps);
- if (inverse_noise_scaling) {
- x0 = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x0);
- }
- x0 = denoiser->process_latent_out(std::move(x0));
-
- if (control_net) {
- control_net->free_control_ctx();
- }
- return x0;
-}
-
-int StableDiffusionGGML::get_vae_scale_factor() {
- if (sd_version_is_pid(version)) {
- return 1;
- }
- return first_stage_model->get_scale_factor();
-}
-
-int StableDiffusionGGML::get_diffusion_model_down_factor() {
- return find_model_definition(version)->geometry.diffusion_down_factor;
-}
-
-int StableDiffusionGGML::get_latent_channel() {
- return find_model_definition(version)->geometry.latent_channels;
-}
-
-int StableDiffusionGGML::get_image_channels() const {
- return find_model_definition(version)->geometry.image_channels;
-}
-
-int StableDiffusionGGML::get_image_seq_len(int h, int w) {
- int vae_scale_factor = get_vae_scale_factor();
- int patch_size = diffusion_model != nullptr ? diffusion_model->get_patch_size() : 1;
- int latent_h = h / vae_scale_factor;
- int latent_w = w / vae_scale_factor;
- int tokens_h = (latent_h + patch_size - 1) / patch_size;
- int tokens_w = (latent_w + patch_size - 1) / patch_size;
- return tokens_h * tokens_w;
-}
-
-sd::Tensor<float> StableDiffusionGGML::generate_init_latent(int width,
- int height,
- int frames,
- bool video) {
- int vae_scale_factor = get_vae_scale_factor();
- int W = width / vae_scale_factor;
- int H = height / vae_scale_factor;
- int T = video_frames_to_latent_frames(frames);
- int C = get_latent_channel();
- if (video) {
- return sd::zeros<float>({W, H, T, C, 1});
- }
- return sd::zeros<float>({W, H, C, 1});
-}
-
-int StableDiffusionGGML::video_frames_to_latent_frames(int frames) {
- int latent_frames = frames;
- if (sd_version_is_ltxav(version)) {
- latent_frames = ((frames - 1) / 8) + 1;
- } else if (sd_version_is_minimax_h3(version)) {
- latent_frames = frames <= 5 ? 2 : ((frames - 5) / 17) * 5 + 2;
- } else if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_hunyuan_video(version)) {
- latent_frames = ((frames - 1) / 4) + 1;
- }
- return latent_frames;
-}
-
-int StableDiffusionGGML::latent_frames_to_video_frames(int latent_frames) {
- if (latent_frames <= 0) {
- return latent_frames;
- }
- if (sd_version_is_ltxav(version)) {
- return (latent_frames - 1) * 8 + 1;
- }
- if (sd_version_is_minimax_h3(version)) {
- return latent_frames <= 2 ? 5 : ((latent_frames - 2) / 5) * 17 + 5;
- }
- if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_hunyuan_video(version)) {
- return (latent_frames - 1) * 4 + 1;
- }
- return latent_frames;
-}
-
-int StableDiffusionGGML::align_video_frames(int frames) {
- if (sd_version_is_minimax_h3(version)) {
- frames = std::max(frames, 5);
- while (frames % 17 != 5) {
- ++frames;
- }
- return frames;
- }
- return latent_frames_to_video_frames(video_frames_to_latent_frames(frames));
-}
-
-sd::Tensor<float> StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor<float>& x) {
- generation_progress.report("vae_encode");
- auto latents = first_stage_model->encode(n_threads, x, vae_tiling_params, circular_x, circular_y);
- if (latents.empty()) {
- return {};
- }
- latents = first_stage_model->vae_output_to_latents(latents, rng);
- return latents;
-}
-
-sd::Tensor<float> StableDiffusionGGML::encode_first_stage(const sd::Tensor<float>& x) {
- auto latents = encode_to_vae_latents(x);
- if (latents.empty()) {
- return {};
- }
- if (version != VERSION_SD1_PIX2PIX) {
- latents = first_stage_model->vae_to_diffusion_latents(latents);
- }
- return latents;
-}
-
-sd::Tensor<float> StableDiffusionGGML::decode_first_stage(const sd::Tensor<float>& x, bool decode_video) {
- generation_progress.report("vae_decode");
- if (sd_version_is_pid(version) || sd_version_is_minit2i(version) || version == VERSION_IRIS) {
- return sd::ops::clamp((x + 1.f) * 0.5f, 0.0f, 1.0f);
- }
- auto image_latents = version == VERSION_P_H3_IMAGE && x.dim() == 5
- ? sd::ops::slice(x, 3, 0, get_latent_channel())
- : x;
- auto latents = first_stage_model->diffusion_to_vae_latents(image_latents);
- auto tiling_params = first_stage_model->resolve_tiling_params(vae_tiling_params);
- const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 &&
- first_stage_model->can_temporal_tile_decode();
- for (;;) {
- int tile_size_w = static_cast<int>(latents.shape()[0]);
- int tile_size_h = static_cast<int>(latents.shape()[1]);
- float tile_overlap;
- if (tiling_params.enabled &&
- !first_stage_model->get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params,
- latents.shape()[0], latents.shape()[1])) {
- return {};
- }
- auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y);
- if (sd_version_is_minimax_h3_image(version) && !decoded.empty() && decoded.dim() == 5) {
- decoded = decoded.squeeze(2);
- }
- if (!decoded.empty() ||
- !sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling,
- first_stage_model->last_compute_status(),
- tile_size_w, tile_size_h, first_stage_model->get_scale_factor())) {
- return decoded;
- }
- }
-}
-
-sd::Tensor<float> StableDiffusionGGML::normalize_ltx_video_latents(const sd::Tensor<float>& x) {
- auto ltx_vae = std::dynamic_pointer_cast<LTXVideoVAE>(first_stage_model);
- if (!ltx_vae) {
- LOG_ERROR("LTX latent normalization requires LTX video VAE");
- return {};
- }
- return ltx_vae->normalize_latents(n_threads, x);
-}
-
-sd::Tensor<float> StableDiffusionGGML::un_normalize_ltx_video_latents(const sd::Tensor<float>& x) {
- auto ltx_vae = std::dynamic_pointer_cast<LTXVideoVAE>(first_stage_model);
- if (!ltx_vae) {
- LOG_ERROR("LTX latent un-normalization requires LTX video VAE");
- return {};
- }
- return ltx_vae->un_normalize_latents(n_threads, x);
-}
-
-sd::Tensor<float> StableDiffusionGGML::decode_ltx_audio_latent(const sd::Tensor<float>& audio_latent) {
- if (audio_vae_model == nullptr || audio_latent.empty()) {
- return {};
- }
- generation_progress.report("audio_decode");
- auto waveform = audio_vae_model->decode(n_threads, audio_latent);
- return waveform;
-}
-
-void StableDiffusionGGML::set_flow_shift(float flow_shift) {
- auto flow_denoiser = std::dynamic_pointer_cast<DiscreteFlowDenoiser>(denoiser);
- if (flow_denoiser) {
- if (flow_shift == INFINITY) {
- flow_shift = default_flow_shift;
- }
- flow_denoiser->set_shift(flow_shift);
- active_flow_shift = flow_shift;
- }
-}
-
-bool StableDiffusionGGML::is_flow_denoiser() {
- auto flow_denoiser = std::dynamic_pointer_cast<DiscreteFlowDenoiser>(denoiser);
- return !!flow_denoiser;
-}
-
-std::string StableDiffusionGGML::get_default_ref_image_preset(SDVersion version) const {
- return find_model_definition(version)->ref_image_preset;
-}
-
-RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_image_args) const {
- RefImageParams params;
- std::string preset_name = get_default_ref_image_preset(version);
- // Keys whose value was rejected. They are already reported, so per-model "no effect"
- // warnings must not repeat them.
- std::set<std::string> rejected_keys;
-
- for (const auto& [key, value] : parse_key_value_args(ref_image_args, "reference image args")) {
- if (key == "preset") {
- std::string requested_preset_name = value;
- if (REF_IMAGE_PRESETS.count(requested_preset_name)) {
- preset_name = requested_preset_name;
- } else if (value != "default") {
- std::string valid_list;
- for (auto const& [name, _] : REF_IMAGE_PRESETS) {
- valid_list += (valid_list.empty() ? "" : ", ") + name;
- }
- LOG_WARN("ignoring invalid reference image preset '%s'. Valid options: [%s]", value.c_str(), valid_list.c_str());
- rejected_keys.insert(key);
- }
- break;
- }
- }
- if (preset_name != "default") {
- LOG_INFO("Using '%s' preset for reference images", preset_name.c_str());
- params = REF_IMAGE_PRESETS.at(preset_name);
- }
- if (version == VERSION_QWEN_IMAGE_2_1) {
- params.vlm_use_vae_input = true;
- }
-
- for (const auto& [key, value] : parse_key_value_args(ref_image_args, "reference image args")) {
- if (key == "pass_to_vlm") {
- if (!parse_strict_bool(value, params.pass_to_vlm)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "pass_to_dit") {
- if (!parse_strict_bool(value, params.pass_to_dit)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "ref_index_mode") {
- if (value == "fixed") {
- params.ref_index_mode = Rope::RefIndexMode::FIXED;
- } else if (value == "increase") {
- params.ref_index_mode = Rope::RefIndexMode::INCREASE;
- } else if (value == "decrease") {
- params.ref_index_mode = Rope::RefIndexMode::DECREASE;
- } else {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "force_ref_timestep_zero") {
- if (!parse_strict_bool(value, params.force_ref_timestep_zero)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "resize_before_vae") {
- if (!parse_strict_bool(value, params.resize_before_vae)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "resize_vae_to_target") {
- if (!parse_strict_bool(value, params.resize_vae_to_target)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "resize_single_ref_to_target") {
- if (!parse_strict_bool(value, params.resize_single_ref_to_target)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "vae_size_buckets") {
- if (!parse_strict_bool(value, params.vae_size_buckets)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "vae_center_crop") {
- if (!parse_strict_bool(value, params.vae_center_crop)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "vlm_use_vae_input") {
- if (!parse_strict_bool(value, params.vlm_use_vae_input)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "vae_input_max_pixels") {
- if (!parse_strict_int(value, params.vae_input_max_pixels)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "vlm_resize_mode") {
- if (value == "longest_side") {
- params.vlm_resize_mode = RefImageResizeMode::LONGEST_SIDE;
- } else if (value == "area") {
- params.vlm_resize_mode = RefImageResizeMode::AREA;
- } else if (value == "none") {
- params.vlm_resize_mode = RefImageResizeMode::NONE;
- } else {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "vlm_max_size") {
- if (!parse_strict_int(value, params.vlm_max_size)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "vlm_min_size") {
- if (!parse_strict_int(value, params.vlm_min_size)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- }
- } else if (key == "ref_image_size") {
- bool valid = true;
- if (value == "match") {
- params.ref_image_size = RefImageSizeMode::MATCH;
- } else if (value == "max") {
- params.ref_image_size = RefImageSizeMode::MAX;
- } else {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- valid = false;
- }
- if (valid && !sd_version_is_minimax_h3(version)) {
- LOG_WARN("ref_image_size is only supported by MiniMax-H3 for now");
- }
- } else if (key != "preset" && key != "vlm_size") {
- LOG_WARN("ignoring unknown reference image arg '%s'", key.c_str());
- }
- }
- for (const auto& [key, value] : parse_key_value_args(ref_image_args, "reference image args")) {
- if (key == "vlm_size") {
- int vlm_size;
- if (!parse_strict_int(value, vlm_size)) {
- LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
- rejected_keys.insert(key);
- } else {
- LOG_INFO("vlm_size override: setting both min and max size to %ld", (long)vlm_size);
- params.vlm_min_size = vlm_size;
- params.vlm_max_size = vlm_size;
- }
- break;
- }
- }
- if (params.force_ref_timestep_zero && !sd_version_is_krea2(version)) {
- LOG_WARN("force_ref_timestep_zero is only supported by Krea2 architecture for now");
- }
- if (version == VERSION_MINIMAX_H3) {
- // H3 consumes references as VAE latents in its own packed sequence rather than
- // through the generic conditioner/DiT reference path, so ref_image_size is the
- // only arg it acts on. Everything else is reported as ineffective rather than
- // enumerated, so newly added RefImageParams members cannot silently do nothing.
- static const std::set<std::string> h3_effective_keys = {"ref_image_size"};
- // Keys that already have their own, more specific warning above.
- static const std::set<std::string> h3_separately_reported_keys = {
- "resize_before_vae",
- "force_ref_timestep_zero",
- };
- for (const auto& arg : parse_key_value_args(ref_image_args, "reference image args")) {
- if (h3_effective_keys.count(arg.first) == 0 &&
- h3_separately_reported_keys.count(arg.first) == 0 &&
- rejected_keys.count(arg.first) == 0) {
- LOG_WARN("reference image arg '%s' has no effect for MiniMax-H3", arg.first.c_str());
- }
- }
-
- if (!params.resize_before_vae) {
- LOG_WARN("MiniMax-H3 requires 32-pixel-aligned reference images and cannot skip resizing before VAE encoding; ignoring resize_before_vae=false");
- }
- }
- return params;
-}
-
-void StableDiffusionGGML::apply_circular_axes(bool circular_x, bool circular_y) {
- this->circular_x = circular_x;
- this->circular_y = circular_y;
- if (this->diffusion_model) {
- this->diffusion_model->set_circular_axes(circular_x, circular_y);
- }
- if (this->high_noise_diffusion_model) {
- this->high_noise_diffusion_model->set_circular_axes(circular_x, circular_y);
- }
- if (this->control_net) {
- this->control_net->set_circular_axes(circular_x, circular_y);
- }
- if (circular_x || circular_y) {
- LOG_INFO("Using circular padding for convolutions (x=%s, y=%s)",
- circular_x ? "true" : "false",
- circular_y ? "true" : "false");
- }
-}
diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h
deleted file mode 100644
index ae05dc9..0000000
--- a/src/pipeline/diffusion_engine.h
+++ /dev/null
@@ -1,534 +0,0 @@
-#ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__
-#define __SD_PIPELINE_DIFFUSION_ENGINE_H__
-
-#include "core/media.h"
-
-#include "core/option_types.h"
-
-#include <atomic>
-#include <cmath>
-#include <functional>
-#include <list>
-#include <map>
-#include <memory>
-#include <mutex>
-#include <optional>
-#include <set>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "config.h"
-#include "core/ggml_extend_backend.h"
-#include "core/ggml_graph_cut.h"
-#include "core/tensor.hpp"
-#include "core/util.h"
-#include "model/adapter/lora.hpp"
-#include "model_builders.h"
-#include "model_manager.h"
-#include "runtime/generation_progress.h"
-#include "stable-diffusion.h"
-
-class RNG;
-struct Denoiser;
-struct LoraModel;
-struct ConditionerParams;
-class ConditioningCache;
-struct SDCondition;
-struct RefImageParams;
-namespace Wav2Vec2 {
- class Wav2Vec2ModelRunner;
-}
-
-static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) {
- return version == VERSION_FLUX ||
- version == VERSION_CHROMA ||
- sd_version_is_flux2(version) ||
- (sd_version_is_qwen_image(version) && version != VERSION_QWEN_IMAGE_2_1) ||
- sd_version_is_mage_flow(version) ||
- sd_version_is_longcat(version) ||
- sd_version_is_z_image(version) ||
- sd_version_is_boogu_image(version);
-}
-
-class StableDiffusionGGML {
-public:
- SDBackendManager backend_manager;
- std::unordered_map<ggml_backend_t, bool> flash_attn_by_backend;
- SDBackendAssignment flash_attn_assignment;
- SDBackendAssignment conv_direct_assignment;
- std::unordered_map<ggml_backend_t, bool> conv2d_direct_by_backend;
- std::unordered_map<ggml_backend_t, bool> conv3d_direct_by_backend;
-
- SDVersion version;
- bool external_vae_is_invalid = false;
-
- bool circular_x = false;
- bool circular_y = false;
-
- std::shared_ptr<RNG> rng;
- std::shared_ptr<RNG> sampler_rng = nullptr;
- int n_threads = -1;
- std::unique_ptr<sd::ParallelExecutor> tensor_executor;
- float default_flow_shift = INFINITY;
- float active_flow_shift = INFINITY;
-
- std::shared_ptr<Conditioner> cond_stage_model;
- std::shared_ptr<FrozenCLIPVisionEmbedder> clip_vision; // for svd or wan2.1 i2v
- std::shared_ptr<DiffusionModelRunner> diffusion_model;
- std::shared_ptr<DiffusionModelRunner> high_noise_diffusion_model;
- std::shared_ptr<VAE> first_stage_model;
- std::shared_ptr<VAE> preview_vae;
- std::shared_ptr<AudioVAERunner> audio_vae_model;
- std::shared_ptr<Wav2Vec2::Wav2Vec2ModelRunner> audio_encoder;
- std::shared_ptr<ControlNet> control_net;
- std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
- sd::Tensor<float> ip_adapter_tokens;
- sd_prompt_image_cb_t prompt_image_callback = nullptr;
- void* prompt_image_callback_data = nullptr;
- sd::Tensor<float> ip_adapter_uncond_tokens;
- float ip_adapter_strength = 1.0f;
- std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
- struct RuntimeLora {
- ModelManager::LoraSpec spec;
- SDBackendModule module;
- std::shared_ptr<LoraModel> model;
-
- bool matches(const ModelManager::LoraSpec& other) const {
- return spec.file_id == other.file_id && spec.file_revision == other.file_revision &&
- spec.tensor_name_prefix_filter == other.tensor_name_prefix_filter;
- }
- };
- std::vector<RuntimeLora> runtime_lora_models;
- bool apply_lora_immediately = false;
- int animatediff_num_frames = 0;
-
- std::string taesd_path;
- sd::TilingConfig vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
- bool enable_mmap = false;
- sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment;
- bool disable_prefetch = false;
- bool disable_segmented_compute = false;
- bool eager_load = false;
- std::string backend_spec;
- std::string params_backend_spec;
- std::string split_mode_spec;
- bool auto_fit_enabled = false;
-
- bool diffusion_conv_direct = false;
-
- bool is_using_v_parameterization = false;
- bool is_using_edm_v_parameterization = false;
-
- std::shared_ptr<ModelManager> model_manager;
-
- enum class RunnerGroup { Core,
- VAE,
- ControlNet,
- Extensions };
- using RunnerGroups = std::set<RunnerGroup>;
-
- struct ModelConfig {
- sd::ContextConfig params{};
- std::list<std::string> strings;
- std::vector<sd_embedding_t> embeddings;
- std::map<std::string, std::string> models;
- ModelLoader::FileId control_net_file = 0;
- bool use_tae = false;
- bool use_audio_vae = false;
- bool photomaker_source_available = false;
- bool animatediff_loaded = false;
-
- explicit ModelConfig(const sd::ContextConfig& initial)
- : params(initial) {
- for (auto member : {&sd::ContextConfig::model_path, &sd::ContextConfig::clip_l_path,
- &sd::ContextConfig::clip_g_path, &sd::ContextConfig::clip_vision_path,
- &sd::ContextConfig::t5xxl_path, &sd::ContextConfig::llm_path,
- &sd::ContextConfig::llm_vision_path, &sd::ContextConfig::diffusion_model_path,
- &sd::ContextConfig::tokenizer, &sd::ContextConfig::flash_attn_mode, &sd::ContextConfig::conv_direct_mode,
- &sd::ContextConfig::high_noise_diffusion_model_path, &sd::ContextConfig::uncond_diffusion_model_path,
- &sd::ContextConfig::embeddings_connectors_path, &sd::ContextConfig::vae_path,
- &sd::ContextConfig::audio_vae_path, &sd::ContextConfig::audio_encoder_path, &sd::ContextConfig::taesd_path,
- &sd::ContextConfig::control_net_path, &sd::ContextConfig::ip_adapter_path,
- &sd::ContextConfig::motion_module_path, &sd::ContextConfig::photo_maker_path,
- &sd::ContextConfig::pulid_weights_path, &sd::ContextConfig::tensor_type_rules,
- &sd::ContextConfig::max_vram, &sd::ContextConfig::backend,
- &sd::ContextConfig::params_backend, &sd::ContextConfig::split_mode,
- &sd::ContextConfig::rpc_servers, &sd::ContextConfig::model_args}) {
- strings.emplace_back(SAFE_STR(initial.*member));
- params.*member = strings.back().c_str();
- }
- for (uint32_t i = 0; i < initial.embedding_count; ++i) {
- strings.emplace_back(SAFE_STR(initial.embeddings[i].name));
- const char* name = strings.back().c_str();
- strings.emplace_back(SAFE_STR(initial.embeddings[i].path));
- embeddings.push_back({name, strings.back().c_str()});
- }
- params.embeddings = embeddings.data();
- }
-
- ModelConfig(const ModelConfig& other)
- : ModelConfig(other.params) {
- control_net_file = other.control_net_file;
- use_tae = other.use_tae;
- use_audio_vae = other.use_audio_vae;
- photomaker_source_available = other.photomaker_source_available;
- animatediff_loaded = other.animatediff_loaded;
- models = other.models;
- }
- ModelConfig& operator=(const ModelConfig&) = delete;
-
- void record_model(const std::string& name, const std::string& path) {
- if (path.empty()) {
- models.erase(name);
- return;
- }
- const auto separator = path.find_last_of("/\\");
- models[name] = separator == std::string::npos ? path : path.substr(separator + 1);
- }
-
- void set_control_net(ModelLoader::FileId id, const std::string& path) {
- control_net_file = id;
- strings.push_back(path);
- params.control_net_path = strings.back().c_str();
- record_model("control_net", path);
- }
- };
-
- struct RunnerState {
- bool ready = false;
- uint64_t catalog_revision = 0;
- std::map<RunnerGroup, ModelLoader::FileVersions> sources;
- };
-
- std::recursive_mutex execution_mutex;
- std::unique_ptr<ModelConfig> config_;
- RunnerState runner_state_;
- std::unique_ptr<ConditioningCache> conditioning_cache_;
- std::vector<ModelManager::LoraSpec> conditioning_loras_;
- bool executing_ = false;
- sd::GenerationProgress generation_progress;
-
- std::shared_ptr<Denoiser> denoiser;
- std::vector<float> file_alphas_cumprod;
-
- StableDiffusionGGML();
- ~StableDiffusionGGML();
-
- static const std::map<RunnerGroup, std::set<ModelComponent>>& runner_components();
-
- static RunnerGroups all_runner_groups();
-
- ModelLoader::FileVersions runner_source_versions(RunnerGroup group, const ModelLoader& loader) const;
-
- void capture_runner_sources();
-
- void end_runners();
-
- bool reset_runners(const RunnerGroups& groups);
-
- bool refresh_model_sources();
-
- bool apply_model_update(ModelLoader candidate,
- std::unique_ptr<ModelConfig> next_config = nullptr,
- RunnerGroups groups = {});
-
- struct ContextOperation {
- StableDiffusionGGML& sd;
- std::unique_lock<std::recursive_mutex> lock;
- bool acquired = false;
- std::optional<sd::ParallelScope> tensor_scope;
-
- explicit ContextOperation(StableDiffusionGGML& sd)
- : sd(sd), lock(sd.execution_mutex, std::try_to_lock) {
- if (!lock.owns_lock() || sd.executing_) {
- // The caller may be a log callback, so rejecting it must not log.
- return;
- }
- sd.executing_ = true;
- acquired = true;
- tensor_scope.emplace(sd.tensor_executor.get());
- }
-
- ~ContextOperation() {
- if (acquired) {
- sd.executing_ = false;
- }
- }
- };
-
- struct ExecutionScope {
- ContextOperation operation;
- bool ready = false;
-
- explicit ExecutionScope(StableDiffusionGGML& sd)
- : operation(sd) {
- if (operation.acquired) {
- sd.generation_progress.report("preparing");
- }
- ready = operation.acquired && sd.refresh_model_sources();
- }
-
- ~ExecutionScope() {
- if (ready) {
- operation.sd.end_runners();
- }
- }
- };
-
- ggml_backend_t backend_for(SDBackendModule module);
-
- ggml_backend_t params_backend_for(SDBackendModule module);
-
- std::atomic<sd_cancel_mode_t> cancellation_flag = SD_CANCEL_RESET;
-
- void set_cancel_flag(enum sd_cancel_mode_t flag);
-
- void reset_cancel_flag();
-
- enum sd_cancel_mode_t get_cancel_flag();
-
- size_t max_graph_vram_bytes_for_module(SDBackendModule module);
-
- std::vector<size_t> layer_split_vram_limits_for_backends(const std::vector<ggml_backend_t>& backends);
-
- bool ensure_backend_pair(SDBackendModule module);
-
- template <typename T>
- bool register_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- size_t* params_mem_size = nullptr);
-
- template <typename T>
- bool register_row_split_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- const std::vector<ggml_backend_t>& module_backends,
- std::map<std::string, ggml_tensor*> group_tensors,
- const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
- ModelManager::ResidencyMode residency_mode,
- size_t* params_mem_size);
-
- // Register graph-cut layer-split tensors on the primary backend first.
- // The first real graph assigns each param tensor to a runtime backend
- // before weights are loaded or staged.
- template <typename T>
- bool register_layer_split_runner_params(ModelComponent component,
- const std::shared_ptr<T>& model,
- SDBackendModule module,
- const std::vector<ggml_backend_t>& module_backends,
- std::map<std::string, ggml_tensor*> group_tensors,
- const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
- ModelManager::ResidencyMode residency_mode,
- size_t* params_mem_size);
-
- bool unload_control_net();
-
- bool load_control_net_from_file(const std::string& path);
-
- void apply_circular_axes(bool circular_x, bool circular_y);
-
- bool init_backend();
-
- bool row_split_active();
-
- bool graph_cut_layer_split_active();
-
- std::shared_ptr<RNG> get_rng(rng_type_t rng_type);
-
- void refresh_compvis_denoiser_sigmas();
-
- void load_alphas_cumprod();
-
- bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration);
-
- bool init(const sd::ContextConfig* sd_ctx_params);
- bool set_sage_attention_enabled(bool enabled);
-
- bool uses_tae() const;
-
- bool tae_preview_only() const;
-
- void configure_weight_loading();
-
- sd::model_builders::Context model_build_context();
-
- bool build_core_runners();
-
- bool build_vae_runners();
-
- bool build_control_net_runner();
-
- bool build_extension_runners();
-
- bool validate_and_load_runners();
- bool flash_attention_for(SDBackendModule module);
- bool conv_direct_for(SDBackendModule module, bool conv3d);
-
- bool build_denoiser();
-
- bool build_runners(const RunnerGroups& groups);
-
- bool is_using_v_parameterization_for_sd2(bool is_inpaint = false);
-
- static std::string lora_log_id(const ModelManager::LoraSpec& lora);
-
- std::shared_ptr<LoraModel> load_lora_model(const ModelManager::LoraSpec& lora_spec,
- SDBackendModule module,
- LoraModel::filter_t module_filter = nullptr);
-
- void clear_lora_adapters();
-
- std::vector<std::shared_ptr<LoraModel>> load_runtime_loras_for_module(const std::vector<ModelManager::LoraSpec>& loras,
- const std::set<std::string>& model_tensor_names,
- SDBackendModule module,
- LoraModel::filter_t module_filter,
- bool& success,
- std::vector<RuntimeLora>& next_models);
-
- bool apply_loras_immediately(const std::vector<ModelManager::LoraSpec>& loras);
-
- bool apply_loras_at_runtime(const std::vector<ModelManager::LoraSpec>& loras);
-
- void lora_stat();
-
- bool apply_loras(const sd_lora_t* loras, uint32_t lora_count);
-
- SDCondition get_learned_condition(const ConditionerParams& params);
-
- void reset_generation_extensions();
-
- void prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params,
- const sd::PuLIDConfig& pulid_params,
- ConditionerParams& condition_params,
- int total_steps);
-
- sd::Tensor<float> get_clip_vision_output(const sd::Tensor<float>& image,
- bool return_pooled = true,
- int clip_skip = -1,
- bool zero_out_masked = false);
-
- sd::Tensor<float> get_audio_embedding(const sd::Audio& audio);
-
- void compute_ip_adapter_tokens(const sd::Image& image, float strength);
-
- std::vector<float> process_timesteps(const std::vector<float>& timesteps,
- const sd::Tensor<float>& init_latent,
- const sd::Tensor<float>& denoise_mask,
- int step);
-
- std::vector<float> process_ltxav_video_timesteps(const std::vector<float>& timesteps,
- const sd::Tensor<float>& init_latent,
- const sd::Tensor<float>& denoise_mask);
-
- void preview_image(int step,
- const sd::Tensor<float>& latents,
- enum SDVersion version,
- preview_t preview_mode,
- std::function<void(int, int, const sd_tensor_t* const*, bool, void*)> step_callback,
- void* step_callback_data,
- bool is_noisy);
-
- std::vector<float> prepare_sample_timesteps(float sigma,
- int shifted_timestep);
-
- void adjust_sample_step_scalings(int shifted_timestep,
- const std::vector<float>& timesteps_vec,
- float c_in,
- float* c_skip,
- float* c_out);
-
- struct SamplePreviewContext {
- sd_preview_cb_t callback = nullptr;
- void* data = nullptr;
- preview_t mode = PREVIEW_NONE;
- };
-
- SamplePreviewContext prepare_sample_preview_context();
-
- void report_sample_progress(int step,
- size_t total_steps,
- bool terminal_sigma_is_zero,
- int64_t* last_progress_us);
-
- // set the kind of the control image of the next generations (ControlNet Union),
- // false (after saying why) when the loaded ControlNet can't use it
- bool set_control_type(sd_control_type_t type);
-
- void compute_sample_controls(const sd::Tensor<float>& control_image,
- const sd::Tensor<float>& noised_input,
- const sd::Tensor<float>& timesteps_tensor,
- const SDCondition& condition,
- std::vector<sd::Tensor<float>>* controls);
-
- sd::Tensor<float> sample(const std::shared_ptr<DiffusionModelRunner>& work_diffusion_model,
- bool inverse_noise_scaling,
- const sd::Tensor<float>& init_latent,
- sd::Tensor<float> noise,
- const SDCondition& cond,
- const SDCondition& uncond,
- const SDCondition& img_uncond,
- const sd::Tensor<float>& control_image,
- float control_strength,
- const sd::GuidanceConfig& guidance,
- float eta,
- int shifted_timestep,
- sample_method_t method,
- bool is_flow_denoiser,
- const char* extra_sample_args,
- const std::vector<float>& sigmas,
- const std::vector<sd::Tensor<float>>& ref_latents,
- const RefImageParams& ref_image_params,
- const sd::Tensor<float>& denoise_mask,
- const sd::Tensor<float>& vace_context,
- float vace_strength,
- int audio_length,
- float frame_rate,
- const sd::CacheConfig* cache_params,
- bool preview_final_step,
- const sd::Tensor<float>& video_positions = {});
-
- int get_vae_scale_factor();
-
- int get_diffusion_model_down_factor();
-
- int get_latent_channel();
-
- int get_image_channels() const;
-
- int get_image_seq_len(int h, int w);
-
- sd::Tensor<float> generate_init_latent(int width,
- int height,
- int frames = 1,
- bool video = false);
-
- int video_frames_to_latent_frames(int frames);
-
- int latent_frames_to_video_frames(int latent_frames);
-
- int align_video_frames(int frames);
-
- sd::Tensor<float> encode_to_vae_latents(const sd::Tensor<float>& x);
-
- sd::Tensor<float> encode_first_stage(const sd::Tensor<float>& x);
-
- sd::Tensor<float> decode_first_stage(const sd::Tensor<float>& x, bool decode_video = false);
-
- sd::Tensor<float> normalize_ltx_video_latents(const sd::Tensor<float>& x);
-
- sd::Tensor<float> un_normalize_ltx_video_latents(const sd::Tensor<float>& x);
-
- sd::Tensor<float> decode_ltx_audio_latent(const sd::Tensor<float>& audio_latent);
-
- void set_flow_shift(float flow_shift = INFINITY);
-
- bool is_flow_denoiser();
-
- std::string get_default_ref_image_preset(SDVersion version) const;
-
- RefImageParams resolve_ref_image_params(const char* ref_image_args) const;
-};
-
-#endif // __SD_PIPELINE_DIFFUSION_ENGINE_H__
diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h
deleted file mode 100644
index 08511f2..0000000
--- a/src/pipeline/generation.h
+++ /dev/null
@@ -1,88 +0,0 @@
-#ifndef __SD_PIPELINE_GENERATION_H__
-#define __SD_PIPELINE_GENERATION_H__
-
-#include "core/media.h"
-
-#include "conditioning/conditioner.hpp"
-#include "config.h"
-#include "json.hpp"
-#include "stable-diffusion.h"
-
-class StableDiffusionGGML;
-
-namespace sd {
- struct ImageGenerationResult;
- struct VideoGenerationResult;
- sd_status_t generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept;
- sd_status_t generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept;
-}
-
-static inline bool sd_version_supports_animatediff(SDVersion version) {
- return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX;
-}
-
-namespace sd::pipeline {
-
- struct ImageGenerationLatents {
- sd::Tensor<float> init_latent;
- sd::Tensor<float> concat_latent;
- sd::Tensor<float> img_uncond_concat_latent;
- sd::Tensor<float> audio_latent;
- sd::Tensor<float> video_positions;
- sd::Tensor<float> control_image;
- std::vector<sd::Tensor<float>> ref_images;
- std::vector<sd::Tensor<float>> ref_latents;
- std::vector<sd::Tensor<float>> reference_audio_latents;
- std::vector<MiniMaxH3ReferenceBlock> minimax_reference_blocks;
- std::vector<MiniMaxH3PresentationItem> minimax_presentation_refs;
- std::vector<int32_t> keyframe_indices;
- sd::Tensor<float> denoise_mask;
- sd::Tensor<float> clip_vision_output;
- sd::Tensor<float> vace_context;
- sd::Tensor<float> s2v_audio_embed;
- int64_t ref_image_num = 0;
- int64_t video_conditioning_frame_count = 0;
- int64_t video_target_frame_count = 0;
- int audio_length = 0;
- };
-
- struct ImageGenerationEmbeds {
- SDCondition cond;
- SDCondition uncond;
- SDCondition img_uncond;
- };
-
- struct ConditionerRunnerEndOnExit {
- Conditioner* conditioner = nullptr;
- ~ConditionerRunnerEndOnExit() {
- if (conditioner != nullptr) {
- conditioner->runner_end();
- }
- }
- };
-
- // Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope.
- bool generate_image(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- sd::Image** images_out,
- int* num_images_out,
- std::vector<nlohmann::json>* infos_out = nullptr);
-
- bool generate_video(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- sd::Image** frames_out,
- int* num_frames_out,
- sd::Audio** audio_out,
- int* fps_out,
- nlohmann::json* info_out = nullptr);
-
- sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
- const char* model_path,
- const sd::Tensor<float>& packed_latent,
- int audio_length);
-
- sd::Tensor<float> ensure_image_tensor_channels(sd::Tensor<float> image, int channels);
-
-} // namespace sd::pipeline
-
-#endif // __SD_PIPELINE_GENERATION_H__
diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp
deleted file mode 100644
index f2cdcea..0000000
--- a/src/pipeline/image.cpp
+++ /dev/null
@@ -1,1239 +0,0 @@
-#include "api/result.h"
-#include "core/option_types.h"
-#include "generation.h"
-#include "generation_info.h"
-
-#include <algorithm>
-#include <cmath>
-#include <cstdlib>
-#include <optional>
-
-#include "core/rng.hpp"
-#include "diffusion_engine.h"
-#include "model/vae/vae.hpp"
-#include "pipeline/image_upscale.h"
-#include "request.h"
-#include "runtime/denoiser.hpp"
-#include "runtime/image_preprocess.h"
-
-namespace sd::pipeline {
-
- struct CircularAxesState {
- bool circular_x = false;
- bool circular_y = false;
- };
-
- static CircularAxesState configure_image_vae_axes(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- const GenerationRequest& request) {
- CircularAxesState original_axes = {sd->circular_x, sd->circular_y};
-
- if (!sd_img_gen_params->vae_tiling_params.enabled) {
- if (sd->first_stage_model) {
- sd->first_stage_model->set_circular_axes(sd->circular_x, sd->circular_y);
- }
- if (sd->preview_vae) {
- sd->preview_vae->set_circular_axes(sd->circular_x, sd->circular_y);
- }
- return original_axes;
- }
-
- int tile_size_w, tile_size_h;
- float overlap;
- int latent_size_w = request.width / request.vae_scale_factor;
- int latent_size_h = request.height / request.vae_scale_factor;
- if (!sd->first_stage_model->get_tile_sizes(tile_size_w,
- tile_size_h,
- overlap,
- sd_img_gen_params->vae_tiling_params,
- latent_size_w,
- latent_size_h)) {
- return original_axes;
- }
-
- sd->circular_x = sd->circular_x && (tile_size_w >= latent_size_w);
- sd->circular_y = sd->circular_y && (tile_size_h >= latent_size_h);
-
- if (sd->first_stage_model) {
- sd->first_stage_model->set_circular_axes(sd->circular_x, sd->circular_y);
- }
- if (sd->preview_vae) {
- sd->preview_vae->set_circular_axes(sd->circular_x, sd->circular_y);
- }
-
- sd->circular_x = original_axes.circular_x && (tile_size_w < latent_size_w);
- sd->circular_y = original_axes.circular_y && (tile_size_h < latent_size_h);
-
- return original_axes;
- }
-
- static void restore_image_vae_axes(StableDiffusionGGML* sd, const CircularAxesState& original_axes) {
- sd->circular_x = original_axes.circular_x;
- sd->circular_y = original_axes.circular_y;
- }
-
- class ImageVaeAxesGuard {
- private:
- StableDiffusionGGML* sd = nullptr;
- CircularAxesState original_axes;
-
- public:
- ImageVaeAxesGuard(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- const GenerationRequest& request)
- : sd(sd),
- original_axes(configure_image_vae_axes(sd, sd_img_gen_params, request)) {}
-
- ~ImageVaeAxesGuard() {
- restore_image_vae_axes(sd, original_axes);
- }
-
- ImageVaeAxesGuard(const ImageVaeAxesGuard&) = delete;
- ImageVaeAxesGuard& operator=(const ImageVaeAxesGuard&) = delete;
- };
-
- sd::Tensor<float> ensure_image_tensor_channels(sd::Tensor<float> image, int channels) {
- if (image.empty()) {
- return image;
- }
- GGML_ASSERT(image.dim() == 4);
- int64_t current_channels = image.shape()[2];
- if (current_channels == channels) {
- return image;
- }
- if (channels == 4) {
- sd::Tensor<float> alpha = sd::full<float>({image.shape()[0], image.shape()[1], 1, image.shape()[3]}, 1.f);
- if (current_channels == 3) {
- return sd::ops::concat(image, alpha, 2);
- }
- if (current_channels == 1) {
- sd::Tensor<float> rgb = sd::ops::concat(image, image, 2);
- rgb = sd::ops::concat(rgb, image, 2);
- return sd::ops::concat(rgb, alpha, 2);
- }
- }
- if (channels == 3 && current_channels >= 3) {
- return sd::ops::slice(image, 2, 0, 3);
- }
- GGML_ABORT("cannot convert image tensor from %lld to %d channels",
- (long long)current_channels,
- channels);
- }
-
- static sd::Tensor<float> encode_z_image_control_latent(StableDiffusionGGML* sd, const sd::Tensor<float>& control_rgb) {
- sd::Tensor<float> vae_output = sd->first_stage_model->encode(sd->n_threads,
- control_rgb,
- sd->vae_tiling_params,
- sd->circular_x,
- sd->circular_y);
- if (vae_output.empty()) {
- LOG_ERROR("failed to encode control image");
- return {};
- }
- // TAE encode already returns the latent; KL/Flux VAE returns a mean/logvar pair.
- sd::Tensor<float> ctrl_latent = sd->first_stage_model->get_desc() == "taesd"
- ? vae_output
- : sd::ops::chunk(vae_output, 2, 2)[0];
- ctrl_latent = sd->first_stage_model->vae_to_diffusion_latents(ctrl_latent);
- if (ctrl_latent.empty() || ctrl_latent.dim() != 4 || ctrl_latent.shape()[2] > 33) {
- LOG_ERROR("failed to encode control image");
- return {};
- }
- std::vector<int64_t> padded_shape = ctrl_latent.shape();
- padded_shape[2] = 33;
- sd::Tensor<float> padded(padded_shape);
- sd::ops::slice_assign(&padded, 2, 0, ctrl_latent.shape()[2], ctrl_latent);
- return padded;
- }
-
- static sd::Tensor<float> resize_ph3_reference(const sd::Tensor<float>& image, int64_t pixels) {
- double scale = std::min(1.0, std::sqrt(static_cast<double>(pixels) / (image.shape()[0] * image.shape()[1])));
- int64_t width = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[0] * scale / 32)) * 32);
- int64_t height = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[1] * scale / 32)) * 32);
- if (width == image.shape()[0] && height == image.shape()[1]) {
- return image;
- }
- return sd::ops::interpolate(image, {width, height, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Lanczos);
- }
-
- static sd::Tensor<float> pack_ph3_image_latent(const sd::Tensor<float>& image,
- const sd::Tensor<float>& previous = {}) {
- auto video = image.dim() == 4 ? image.unsqueeze(2) : image;
- GGML_ASSERT(video.dim() == 5 && video.shape()[2] == 1 && video.shape()[3] == 24 && video.shape()[4] == 1);
- const int64_t spatial = video.shape()[0] * video.shape()[1];
- // The image DiT still attends to two stereo audio latents (32 channels).
- auto shape = video.shape();
- shape[3] += (128 + spatial - 1) / spatial;
- auto packed = sd::zeros<float>(shape);
- std::copy_n(video.data(), video.numel(), packed.data());
- if (!previous.empty()) {
- const int64_t offset = previous.shape()[0] * previous.shape()[1] * 24;
- GGML_ASSERT(previous.numel() >= offset + 128);
- std::copy_n(previous.data() + offset, 128, packed.data() + video.numel());
- }
- return packed;
- }
-
- static sd::Tensor<float> resize_h3_image_reference(const sd::Tensor<float>& image, int64_t max_short_edge) {
- const int64_t short_edge = std::min(image.shape()[0], image.shape()[1]);
- const double scale = static_cast<double>(std::clamp<int64_t>(short_edge, 1024, max_short_edge)) / short_edge;
- const int64_t width = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[0] * scale / 32)) * 32);
- const int64_t height = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[1] * scale / 32)) * 32);
- if (width == image.shape()[0] && height == image.shape()[1]) {
- return image;
- }
- return sd::ops::interpolate(image, {width, height, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Lanczos);
- }
-
- static std::optional<ImageGenerationLatents> prepare_image_generation_latents(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- GenerationRequest* request,
- SamplePlan* plan,
- const RefImageParams& ref_image_params) {
- int64_t prepare_start_ms = ggml_time_ms();
-
- sd::Tensor<float> init_image_tensor;
- sd::Tensor<float> control_image_tensor;
- sd::Tensor<float> mask_image_tensor;
- int image_channels = sd->get_image_channels();
-
- if (sd_img_gen_params->init_image.data != nullptr) {
- LOG_INFO("IMG2IMG");
-
- if (request->strength < 1.f) {
- bool strength_as_noise_level = false;
- bool force_first_sigma = false;
- for (const auto& [key, value] : parse_key_value_args(sd_img_gen_params->sample_params.extra_sample_args, "img2img arg")) {
- if (key == "strength_as_noise_level") {
- if (!parse_strict_bool(value, strength_as_noise_level)) {
- LOG_WARN("ignoring invalid img2img sample arg '%s=%s'", key.c_str(), value.c_str());
- }
- } else if (key == "force_first_sigma") {
- if (!parse_strict_bool(value, force_first_sigma)) {
- LOG_WARN("ignoring invalid img2img sample arg '%s=%s'", key.c_str(), value.c_str());
- }
- }
- }
-
- size_t t_enc;
- float target_sigma = -1;
- if (!strength_as_noise_level) {
- t_enc = static_cast<size_t>(plan->sample_steps * request->strength);
- if (t_enc == static_cast<size_t>(plan->sample_steps)) {
- t_enc--;
- }
- } else {
- LOG_VERBOSE("Interpreting denoise strength as relative noise level");
- // assume x_noised = K * (x * (1-noise_level) + noise * noise_level) = K * lerp(x, noise, noise_level)
- // K = 1, noise_level = sigma for flow models
- // K = 1+sigma, noise_level=sigma/(1+sigma) for diffusion models
- float target_noise_level = request->strength;
- target_sigma = sd->denoiser->noise_level_to_sigma(target_noise_level);
- size_t start_index = 0;
- for (size_t i = 0; i < plan->sigmas.size(); ++i) {
- if (plan->sigmas[i] <= target_sigma) {
- start_index = i;
- break;
- }
- }
-
- if (start_index >= plan->sigmas.size() - 1) {
- start_index = plan->sigmas.size() - 2; // Leave at least 1 step
- }
- t_enc = plan->sample_steps - start_index - 1;
- }
- LOG_INFO("target t_enc is %zu steps", t_enc);
- std::vector<float> sigma_sched;
- sigma_sched.assign(plan->sigmas.begin() + plan->sample_steps - t_enc - 1, plan->sigmas.end());
-
- if (target_sigma > 0 && force_first_sigma && strength_as_noise_level) {
- LOG_VERBOSE("force_first_sigma to %.4f (from %.4f)", target_sigma, sigma_sched[0]);
- sigma_sched[0] = target_sigma;
- }
-
- plan->sigmas = std::move(sigma_sched);
- plan->sample_steps = static_cast<int>(plan->sigmas.size() - 1);
- }
-
- init_image_tensor = ensure_image_tensor_channels(sd_image_to_tensor(sd_img_gen_params->init_image, request->width, request->height),
- image_channels);
- }
-
- if (sd_img_gen_params->mask_image.data != nullptr) {
- mask_image_tensor = sd_image_to_tensor(sd_img_gen_params->mask_image, request->width, request->height);
- mask_image_tensor = sd::ops::round(mask_image_tensor);
- }
-
- if (sd_img_gen_params->control_image.data != nullptr) {
- control_image_tensor = sd_image_to_tensor(sd_img_gen_params->control_image, request->width, request->height);
- if (!sd->set_control_type(request->control_type)) {
- return std::nullopt;
- }
- }
-
- if (init_image_tensor.empty() || mask_image_tensor.empty()) {
- if (sd_version_is_inpaint(sd->version)) {
- LOG_WARN("inpainting model requires both an init image and a mask image.");
- }
- }
-
- if (mask_image_tensor.empty()) {
- mask_image_tensor = sd::full<float>({request->width, request->height, 1, 1}, 1.f);
- }
-
- sd::Tensor<float> latent_mask = sd::ops::interpolate(mask_image_tensor,
- {request->width / request->vae_scale_factor,
- request->height / request->vae_scale_factor,
- 1,
- 1},
- sd::ops::InterpolateMode::NearestMax);
-
- sd::Tensor<float> init_latent;
- sd::Tensor<float> control_latent;
- if (init_image_tensor.empty()) {
- if (sd->version == VERSION_QWEN_IMAGE_LAYERED) {
- init_latent = sd->generate_init_latent(request->width, request->height, request->qwen_image_layers + 1, true);
- } else {
- init_latent = sd->generate_init_latent(request->width, request->height);
- }
- } else {
- init_latent = sd->encode_first_stage(init_image_tensor);
- if (init_latent.empty()) {
- LOG_ERROR("failed to encode init image");
- return std::nullopt;
- }
- }
-
- if (sd->animatediff_num_frames > 1 &&
- init_latent.dim() >= 4 && init_latent.shape()[3] == 1) {
- int n_frames = sd->animatediff_num_frames;
- std::vector<int64_t> shape(init_latent.shape().begin(), init_latent.shape().end());
- shape[3] = n_frames;
- if (!init_image_tensor.empty()) {
- sd::Tensor<float> replicated(shape);
- for (int f = 0; f < n_frames; ++f) {
- sd::ops::slice_assign(&replicated, 3, f, f + 1, init_latent);
- }
- init_latent = std::move(replicated);
- } else {
- init_latent = sd::Tensor<float>(std::move(shape));
- }
- }
-
- if (!control_image_tensor.empty()) {
- const bool z_image_control = sd_version_is_z_image(sd->version) &&
- sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0;
- if (z_image_control) {
- control_image_tensor = encode_z_image_control_latent(sd, control_image_tensor);
- if (control_image_tensor.empty()) {
- return std::nullopt;
- }
- } else if (sd_version_is_z_image(sd->version)) {
- LOG_WARN("control image provided but the z-image model has no control net; ignoring");
- control_image_tensor = {};
- } else {
- control_latent = sd->encode_first_stage(control_image_tensor);
- if (control_latent.empty()) {
- LOG_ERROR("failed to encode control image");
- return std::nullopt;
- }
- }
- }
-
- std::vector<sd::Tensor<float>> ref_images;
- for (int i = 0; i < sd_img_gen_params->ref_images_count; i++) {
- ref_images.push_back(ensure_image_tensor_channels(sd_image_to_tensor(sd_img_gen_params->ref_images[i]),
- image_channels));
- }
-
- if (ref_images.empty() && sd_version_is_unet_edit(sd->version)) {
- LOG_WARN("This model needs at least one reference image; using an empty reference");
- ref_images.push_back(sd::zeros<float>({request->width, request->height, image_channels, 1}));
- request->guidance.img_cfg = request->guidance.txt_cfg;
- request->use_img_uncond = false;
- }
-
- if (!ref_images.empty()) {
- LOG_INFO("EDIT mode");
- }
-
- bool skip_vae_encoding = (sd->version == VERSION_HIDREAM_O1) || !(
- ref_image_params.pass_to_dit || ref_image_params.vlm_use_vae_input ||
- sd_version_is_unet_edit(sd->version) || sd_version_is_pid(sd->version));
-
- std::vector<sd::Tensor<float>> ref_latents;
- for (size_t i = 0; i < ref_images.size(); i++) {
- if (skip_vae_encoding) {
- continue;
- }
- sd::Tensor<float> ref_latent;
- if (sd->version == VERSION_MINIMAX_H3_IMAGE) {
- ref_latent = sd->encode_first_stage(resize_h3_image_reference(ref_images[i], 2048));
- if (!ref_latent.empty()) {
- const int64_t width = ref_latent.shape()[0];
- const int64_t height = ref_latent.shape()[1];
- const int64_t channels = ref_latent.shape()[3];
- auto condition_rng = std::make_shared<MT19937RNG>(static_cast<uint64_t>(request->seed));
- // Upstream augments patch rows, so the RNG follows token/channel/patch order.
- auto noise = sd::randn<float>({2, 2, channels, width / 2, height / 2}, condition_rng)
- .permute({0, 3, 1, 4, 2})
- .reshape(ref_latent.shape());
- ref_latent = ref_latent * 0.999f + noise * (1.f - 0.999f);
- }
- } else if (sd->version == VERSION_P_H3_IMAGE) {
- auto resized = resize_ph3_reference(ref_images[i], int64_t(request->width) * request->height);
- ref_latent = sd->encode_first_stage(resized);
- } else if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) {
- LOG_VERBOSE("auto resize ref images");
- double vae_width;
- double vae_height;
- int factor = sd_version_is_qwen_image(sd->version) ? 32 : 16;
- if (ref_image_params.resize_vae_to_target || (ref_image_params.resize_single_ref_to_target && ref_images.size() == 1)) {
- vae_width = request->width;
- vae_height = request->height;
- } else {
- int default_pixels = sd->version == VERSION_QWEN_IMAGE_2_1 ? request->width * request->height : 1024 * 1024;
- int target_pixels = ref_image_params.vae_input_max_pixels > 0 ? ref_image_params.vae_input_max_pixels : default_pixels;
- if (ref_image_params.vae_size_buckets) {
- auto size = sd::select_image_size_bucket(static_cast<int>(ref_images[i].shape()[0]), static_cast<int>(ref_images[i].shape()[1]), target_pixels, factor, request->width, request->height);
- vae_width = size.first;
- vae_height = size.second;
- } else {
- int vae_image_size = std::min(target_pixels, request->width * request->height);
- vae_width = sqrt(vae_image_size * ref_images[i].shape()[0] / ref_images[i].shape()[1]);
- vae_height = vae_width * ref_images[i].shape()[1] / ref_images[i].shape()[0];
- }
- }
-
- vae_height = round(vae_height / factor) * factor;
- vae_width = round(vae_width / factor) * factor;
-
- auto resized_ref_img = ref_image_params.vae_center_crop
- ? sd::resize_and_center_crop(ref_images[i], static_cast<int>(vae_width), static_cast<int>(vae_height))
- : sd::ops::interpolate(ref_images[i],
- {static_cast<int>(vae_width),
- static_cast<int>(vae_height),
- ref_images[i].shape()[2],
- ref_images[i].shape()[3]});
-
- LOG_VERBOSE("resize vae ref image %d from %" PRId64 "x%" PRId64 " to %" PRId64 "x%" PRId64,
- static_cast<int>(i),
- ref_images[i].shape()[1],
- ref_images[i].shape()[0],
- resized_ref_img.shape()[1],
- resized_ref_img.shape()[0]);
-
- ref_latent = sd->encode_first_stage(resized_ref_img);
- if (ref_image_params.vlm_use_vae_input) {
- ref_images[i] = std::move(resized_ref_img);
- }
- } else {
- ref_latent = sd->encode_first_stage(ref_images[i]);
- }
- if (ref_latent.empty()) {
- LOG_ERROR("failed to encode reference image %d", static_cast<int>(i));
- return std::nullopt;
- }
-
- ref_latents.push_back(std::move(ref_latent));
- }
-
- if (sd_version_is_pid(sd->version)) {
- if (ref_latents.empty()) {
- LOG_ERROR("PiD requires a reference image");
- return std::nullopt;
- }
- }
-
- sd::Tensor<float> concat_latent;
- sd::Tensor<float> img_uncond_concat_latent;
- if (sd_version_is_inpaint(sd->version)) {
- sd::Tensor<float> masked_init_latent;
-
- if (sd->version != VERSION_FLEX_2) {
- if (!init_image_tensor.empty()) {
- auto masked_image = ((1.0f - mask_image_tensor) * (init_image_tensor - 0.5f)) + 0.5f;
- masked_init_latent = sd->encode_first_stage(masked_image);
- if (masked_init_latent.empty()) {
- LOG_ERROR("failed to encode masked init image");
- return std::nullopt;
- }
- } else {
- masked_init_latent = sd::Tensor<float>::zeros_like(init_latent);
- }
- } else {
- masked_init_latent = ((1.0f - latent_mask) * init_latent);
- }
-
- auto uncond_masked_init_latent = sd::Tensor<float>::zeros_like(masked_init_latent);
-
- if (sd->version == VERSION_FLUX_FILL) {
- auto mask = mask_image_tensor.reshape({request->vae_scale_factor,
- request->width / request->vae_scale_factor,
- request->vae_scale_factor,
- request->height / request->vae_scale_factor});
- mask = mask.permute({1, 3, 0, 2}).reshape({request->width / request->vae_scale_factor, request->height / request->vae_scale_factor, request->vae_scale_factor * request->vae_scale_factor, 1});
-
- concat_latent = sd::ops::concat(masked_init_latent, mask, 2);
- img_uncond_concat_latent = sd::ops::concat(uncond_masked_init_latent, mask, 2);
- } else if (sd->version == VERSION_FLEX_2) {
- concat_latent = sd::ops::concat(masked_init_latent, latent_mask, 2);
- if (!control_latent.empty()) {
- concat_latent = sd::ops::concat(concat_latent, control_latent, 2);
- } else {
- concat_latent = sd::ops::concat(concat_latent, sd::Tensor<float>::zeros_like(masked_init_latent), 2);
- }
-
- img_uncond_concat_latent = sd::ops::concat(uncond_masked_init_latent, latent_mask, 2);
- img_uncond_concat_latent = sd::ops::concat(img_uncond_concat_latent, sd::Tensor<float>::zeros_like(masked_init_latent), 2);
- } else { // SD1.x SD2.x SDXL inpaint
- concat_latent = sd::ops::concat(latent_mask, masked_init_latent, 2);
- img_uncond_concat_latent = sd::ops::concat(latent_mask, uncond_masked_init_latent, 2);
- }
- }
- if (sd_version_is_unet_edit(sd->version)) {
- concat_latent = sd::ops::interpolate<float>(ref_latents[0], init_latent.shape());
- img_uncond_concat_latent = sd::Tensor<float>::zeros_like(concat_latent);
- }
- if (sd->version == VERSION_FLUX_CONTROLS) {
- if (!control_latent.empty()) {
- concat_latent = control_latent;
- } else {
- concat_latent = sd::Tensor<float>::zeros_like(init_latent);
- }
- img_uncond_concat_latent = sd::Tensor<float>::zeros_like(concat_latent);
- }
-
- if (sd_img_gen_params->init_image.data != nullptr || sd_img_gen_params->ref_images_count > 0) {
- int64_t t1 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %.2fs", (t1 - prepare_start_ms) * 1.0f / 1000);
- }
-
- ImageGenerationLatents latents;
- if (sd_version_is_minimax_h3_image(sd->version)) {
- if (sd->version == VERSION_P_H3_IMAGE) {
- init_latent = pack_ph3_image_latent(init_latent);
- } else if (init_latent.dim() == 4) {
- init_latent = init_latent.unsqueeze(2);
- }
- latent_mask = latent_mask.unsqueeze(2);
- for (size_t i = 0; i < ref_latents.size(); ++i) {
- latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE, static_cast<int32_t>(i), -1});
- }
- for (auto& image : ref_images) {
- image = sd->version == VERSION_P_H3_IMAGE ? resize_ph3_reference(image, 1000000) : resize_h3_image_reference(image, 1024);
- }
- }
- latents.init_latent = std::move(init_latent);
- latents.concat_latent = std::move(concat_latent);
- latents.img_uncond_concat_latent = std::move(img_uncond_concat_latent);
- latents.control_image = std::move(control_image_tensor);
- latents.ref_images = std::move(ref_images);
- latents.ref_latents = std::move(ref_latents);
-
- if (sd_version_is_inpaint(sd->version)) {
- latent_mask = sd::ops::max_pool_2d(latent_mask,
- {3, 3},
- {1, 1},
- {1, 1});
- }
- latents.denoise_mask = std::move(latent_mask);
-
- return latents;
- }
-
- static std::optional<ImageGenerationEmbeds> prepare_image_generation_embeds(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- GenerationRequest* request,
- SamplePlan* plan,
- ImageGenerationLatents* latents,
- const RefImageParams& ref_image_params) {
- ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
-
- ConditionerParams condition_params;
- condition_params.text = request->prompt;
- condition_params.clip_skip = request->clip_skip;
- condition_params.width = request->width;
- condition_params.height = request->height;
- if (ref_image_params.pass_to_vlm) {
- condition_params.ref_images = &latents->ref_images;
- }
-
- condition_params.ref_image_params = ref_image_params;
-
- sd->generation_progress.report("conditioning", "extensions");
- sd->prepare_generation_extensions(request->pm_params,
- request->pulid_params,
- condition_params,
- plan->total_steps);
- sd->compute_ip_adapter_tokens(sd_img_gen_params->ip_adapter_image, sd_img_gen_params->ip_adapter_strength);
- int64_t prepare_start_ms = ggml_time_ms();
- condition_params.zero_out_masked = false;
- sd->generation_progress.report("conditioning", "positive_prompt");
- auto cond = sd->get_learned_condition(condition_params);
- if (cond.empty()) {
- LOG_ERROR("failed to encode prompt");
- return std::nullopt;
- }
- if (cond.c_concat.empty() && ref_image_params.pass_to_dit) {
- cond.c_concat = latents->concat_latent; // TODO: optimize
- }
- if (sd_version_is_minimax_h3_image(sd->version) && ref_image_params.pass_to_dit) {
- cond.c_reference_blocks = latents->minimax_reference_blocks;
- }
-
- bool use_ref_latent_img_cfg = request->use_img_uncond &&
- !latents->ref_images.empty() &&
- sd_version_supports_ref_latent_img_cfg(sd->version);
-
- SDCondition uncond;
- if (request->use_uncond || request->use_high_noise_uncond) {
- if (sd_version_is_ideogram4(sd->version)) {
- uncond.c_vector = sd::Tensor<float>::from_vector({1.0f});
- } else if (sd_version_is_minit2i(sd->version)) {
- // MiniT2I derives the unconditional signal from the same T5 hidden
- // states with a zeroed prompt mask, so no extra text encode is needed.
- uncond.c_crossattn = cond.c_crossattn;
- uncond.c_vector = sd::Tensor<float>::zeros_like(cond.c_vector);
- } else if (sd->version == VERSION_MING_IMAGE) {
- uncond.c_crossattn = sd::Tensor<float>::zeros_like(cond.c_crossattn);
- for (const auto& extra : cond.extra_c_crossattns) {
- uncond.extra_c_crossattns.push_back(sd::Tensor<float>::zeros_like(extra));
- }
- } else if (sd_version_is_sensenova_u1(sd->version)) {
- auto* sensenova_conditioner = static_cast<SenseNovaU1Conditioner*>(sd->cond_stage_model.get());
- uncond = sensenova_conditioner->get_unconditional_condition(request->negative_prompt);
- } else {
- bool zero_out_masked = false;
- if (sd_version_is_sdxl(sd->version) &&
- request->negative_prompt.empty() &&
- !sd->is_using_edm_v_parameterization) {
- zero_out_masked = true;
- }
- condition_params.text = request->negative_prompt;
- condition_params.zero_out_masked = zero_out_masked;
- if (sd_version_is_llada_image(sd->version) || sd->version == VERSION_FIBO2) {
- // Keep source latents in CFG, but omit source images from the text encoder.
- condition_params.ref_images = nullptr;
- }
- sd->generation_progress.report("conditioning", "negative_prompt");
- uncond = sd->get_learned_condition(condition_params);
- if (uncond.empty()) {
- LOG_ERROR("failed to encode negative prompt");
- return std::nullopt;
- }
- }
- if (uncond.c_concat.empty() && ref_image_params.pass_to_dit) {
- uncond.c_concat = latents->concat_latent; // TODO: optimize
- }
- if (sd_version_is_minimax_h3_image(sd->version) && ref_image_params.pass_to_dit) {
- uncond.c_reference_blocks = latents->minimax_reference_blocks;
- }
- }
-
- SDCondition img_uncond;
- if (request->use_img_uncond) {
- if ((request->use_uncond || request->use_high_noise_uncond) && (latents->ref_images.empty() || !use_ref_latent_img_cfg)) {
- img_uncond = SDCondition(uncond.c_crossattn, uncond.c_vector, latents->img_uncond_concat_latent);
- } else {
- bool zero_out_masked = false;
- if (sd_version_is_sdxl(sd->version) &&
- request->negative_prompt.empty() &&
- !sd->is_using_edm_v_parameterization) {
- zero_out_masked = true;
- }
- condition_params.text = request->negative_prompt;
- condition_params.zero_out_masked = zero_out_masked;
- std::vector<sd::Tensor<float>> empty_ref_images;
- if (use_ref_latent_img_cfg) {
- condition_params.ref_images = &empty_ref_images;
- }
- sd->generation_progress.report("conditioning", "image_condition");
- img_uncond = sd->get_learned_condition(condition_params);
- if (img_uncond.empty()) {
- LOG_ERROR("failed to encode image guidance prompt");
- return std::nullopt;
- }
- if (img_uncond.c_concat.empty() && ref_image_params.pass_to_dit) {
- img_uncond.c_concat = latents->img_uncond_concat_latent; // TODO: optimize
- }
- }
- }
-
- int64_t t1 = ggml_time_ms();
- LOG_INFO("get_learned_condition completed, taking %.2fs", (t1 - prepare_start_ms) * 1.0f / 1000);
-
- ImageGenerationEmbeds embeds;
- embeds.img_uncond = std::move(img_uncond);
- embeds.cond = std::move(cond);
- embeds.uncond = std::move(uncond);
-
- return embeds;
- }
-
- static sd::Image* decode_image_outputs(StableDiffusionGGML* sd,
- const GenerationRequest& request,
- const std::vector<sd::Tensor<float>>& final_latents,
- int* num_images_out) {
- if (final_latents.empty()) {
- LOG_ERROR("no latent images to decode");
- return nullptr;
- }
- if (final_latents.size() > static_cast<size_t>(request.batch_count)) {
- LOG_ERROR("expected at most %d latents, got %zu", request.batch_count, final_latents.size());
- return nullptr;
- }
- if (final_latents.size() < static_cast<size_t>(request.batch_count)) {
- LOG_INFO("decoding %zu/%d latents", final_latents.size(), request.batch_count);
- } else {
- LOG_INFO("decoding %zu latents", final_latents.size());
- }
- std::vector<sd::Tensor<float>> decoded_images;
- int64_t t0 = ggml_time_ms();
- bool cancelled = false;
-
- for (size_t i = 0; i < final_latents.size(); i++) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling latent decodings");
- cancelled = true;
- break;
- }
- int64_t t1 = ggml_time_ms();
- if (sd->version == VERSION_QWEN_IMAGE_LAYERED) {
- int qwen_image_latent_layers = request.qwen_image_layers + 1;
- if (final_latents[i].dim() < 5 || final_latents[i].shape()[2] < qwen_image_latent_layers) {
- LOG_ERROR("qwen image layered expected at least %d latent layers, got shape dim=%d",
- qwen_image_latent_layers,
- final_latents[i].dim());
- return nullptr;
- }
- for (int layer_index = 0; layer_index < qwen_image_latent_layers; layer_index++) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling latent decodings");
- cancelled = true;
- break;
- }
- sd::Tensor<float> layer_latent = sd::ops::slice(final_latents[i], 2, layer_index, layer_index + 1);
- layer_latent.squeeze_(2);
- sd::Tensor<float> image = sd->decode_first_stage(layer_latent);
- if (image.empty()) {
- LOG_ERROR("decode_first_stage failed for latent %zu layer %d", i + 1, layer_index + 1);
- return nullptr;
- }
- decoded_images.push_back(std::move(image));
- }
- if (cancelled) {
- break;
- }
- } else if (sd->animatediff_num_frames > 1 &&
- final_latents[i].dim() >= 4 &&
- final_latents[i].shape()[3] == sd->animatediff_num_frames) {
- int n_frames = sd->animatediff_num_frames;
- for (int f = 0; f < n_frames; ++f) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling latent decodings");
- cancelled = true;
- break;
- }
- sd::Tensor<float> frame_latent = sd::ops::slice(final_latents[i], 3, f, f + 1);
- sd::Tensor<float> image = sd->decode_first_stage(frame_latent);
- if (image.empty()) {
- LOG_ERROR("decode_first_stage failed for AnimateDiff frame %d/%d", f + 1, n_frames);
- return nullptr;
- }
- decoded_images.push_back(std::move(image));
- }
- } else {
- sd::Tensor<float> image = sd->decode_first_stage(final_latents[i]);
- if (image.empty()) {
- LOG_ERROR("decode_first_stage failed for latent %" PRId64, i + 1);
- return nullptr;
- }
- decoded_images.push_back(std::move(image));
- }
- int64_t t2 = ggml_time_ms();
- LOG_INFO("latent %zu decoded, taking %.2fs", i + 1, (t2 - t1) * 1.0f / 1000);
- }
-
- int64_t t4 = ggml_time_ms();
- LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000);
- if (decoded_images.empty() || cancelled) {
- LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images");
- return nullptr;
- }
-
- int image_count = static_cast<int>(decoded_images.size());
- sd::Image* result_images = (sd::Image*)calloc(image_count, sizeof(sd::Image));
- if (result_images == nullptr) {
- return nullptr;
- }
- if (num_images_out != nullptr) {
- *num_images_out = image_count;
- }
-
- for (size_t i = 0; i < decoded_images.size(); i++) {
- result_images[i] = tensor_to_sd_image(decoded_images[i]);
- }
-
- return result_images;
- }
-
- static sd::Tensor<float> upscale_hires_latent(StableDiffusionGGML* sd,
- const sd::Tensor<float>& latent,
- const GenerationRequest& request,
- ImageUpscaleModel* upscaler) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling hires latent upscale");
- return {};
- }
-
- auto get_hires_latent_target_shape = [&]() {
- std::vector<int64_t> target_shape = latent.shape();
- if (target_shape.size() < 2) {
- target_shape.clear();
- return target_shape;
- }
- target_shape[0] = request.hires.target_width / request.vae_scale_factor;
- target_shape[1] = request.hires.target_height / request.vae_scale_factor;
- return target_shape;
- };
-
- if (request.hires.upscaler == SD_HIRES_UPSCALER_LATENT ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LATENT_NEAREST ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LATENT_ANTIALIASED ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LATENT_BICUBIC ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED) {
- std::vector<int64_t> target_shape = get_hires_latent_target_shape();
- if (target_shape.empty()) {
- LOG_ERROR("latent has invalid shape for hires upscale");
- return {};
- }
-
- sd::ops::InterpolateMode mode = sd::ops::InterpolateMode::Nearest;
- bool antialias = false;
- switch (request.hires.upscaler) {
- case SD_HIRES_UPSCALER_LATENT:
- mode = sd::ops::InterpolateMode::Bilinear;
- break;
- case SD_HIRES_UPSCALER_LATENT_NEAREST:
- mode = sd::ops::InterpolateMode::Nearest;
- break;
- case SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT:
- mode = sd::ops::InterpolateMode::NearestExact;
- break;
- case SD_HIRES_UPSCALER_LATENT_ANTIALIASED:
- mode = sd::ops::InterpolateMode::Bilinear;
- antialias = true;
- break;
- case SD_HIRES_UPSCALER_LATENT_BICUBIC:
- mode = sd::ops::InterpolateMode::Bicubic;
- break;
- case SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED:
- mode = sd::ops::InterpolateMode::Bicubic;
- antialias = true;
- break;
- default:
- break;
- }
-
- LOG_INFO("hires %s upscale %" PRId64 "x%" PRId64 " -> %" PRId64 "x%" PRId64,
- sd_hires_upscaler_name(request.hires.upscaler),
- latent.shape()[0],
- latent.shape()[1],
- target_shape[0],
- target_shape[1]);
-
- return sd::ops::interpolate(latent, target_shape, mode, false, antialias);
- } else if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL ||
- request.hires.upscaler == SD_HIRES_UPSCALER_LANCZOS ||
- request.hires.upscaler == SD_HIRES_UPSCALER_NEAREST) {
- if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL && upscaler == nullptr) {
- LOG_ERROR("hires model upscaler context is null");
- return {};
- }
-
- sd::Tensor<float> decoded = sd->decode_first_stage(latent);
- if (decoded.empty()) {
- LOG_ERROR("decode_first_stage failed before hires %s upscale",
- sd_hires_upscaler_name(request.hires.upscaler));
- return {};
- }
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling hires image upscale");
- return {};
- }
-
- sd->generation_progress.report("upscaling", "hires");
- sd::Tensor<float> upscaled_tensor;
- if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) {
- upscaled_tensor = upscale_image_tensor(*upscaler, decoded, sd->n_threads, request.hires.upscale_tile_size);
- if (upscaled_tensor.empty()) {
- LOG_ERROR("hires model upscale failed");
- return {};
- }
-
- if (upscaled_tensor.shape()[0] != request.hires.target_width ||
- upscaled_tensor.shape()[1] != request.hires.target_height) {
- upscaled_tensor = sd::ops::interpolate(upscaled_tensor,
- {request.hires.target_width,
- request.hires.target_height,
- upscaled_tensor.shape()[2],
- upscaled_tensor.shape()[3]});
- }
- } else {
- sd::ops::InterpolateMode mode = request.hires.upscaler == SD_HIRES_UPSCALER_LANCZOS
- ? sd::ops::InterpolateMode::Lanczos
- : sd::ops::InterpolateMode::Nearest;
- LOG_INFO("hires %s image upscale %" PRId64 "x%" PRId64 " -> %dx%d",
- sd_hires_upscaler_name(request.hires.upscaler),
- decoded.shape()[0],
- decoded.shape()[1],
- request.hires.target_width,
- request.hires.target_height);
- upscaled_tensor = sd::ops::interpolate(decoded,
- {request.hires.target_width,
- request.hires.target_height,
- decoded.shape()[2],
- decoded.shape()[3]},
- mode);
- upscaled_tensor = sd::ops::clamp(upscaled_tensor, 0.0f, 1.0f);
- }
-
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling hires latent encode");
- return {};
- }
- sd::Tensor<float> upscaled_latent = sd->encode_first_stage(upscaled_tensor);
- if (upscaled_latent.empty()) {
- LOG_ERROR("encode_first_stage failed after hires %s upscale",
- sd_hires_upscaler_name(request.hires.upscaler));
- }
- return upscaled_latent;
- }
-
- LOG_ERROR("unsupported hires upscaler '%s'", sd_hires_upscaler_name(request.hires.upscaler));
- return {};
- }
-
- bool generate_image(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- sd::Image** images_out,
- int* num_images_out,
- std::vector<nlohmann::json>* infos_out) {
- if (infos_out)
- infos_out->clear();
- if (images_out != nullptr) {
- *images_out = nullptr;
- }
- if (num_images_out != nullptr) {
- *num_images_out = 0;
- }
- if (sd == nullptr || sd_img_gen_params == nullptr) {
- return false;
- }
-
- if (!sd_version_supports_image_generation(sd->version)) {
- LOG_ERROR("%s cannot be run with generate_image(); use generate_video() or --mode vid_gen in the CLI",
- sd_version_name(sd->version));
- return false;
- }
-
- sd->reset_cancel_flag();
-
- int64_t t0 = ggml_time_ms();
- sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
- GenerationRequest request(sd, sd_img_gen_params);
- const auto info = sd::generation_info(request, *sd_img_gen_params);
- sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
- sd::ImageConfig processed_params = *sd_img_gen_params;
- if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
- return false;
- sd_img_gen_params = &processed_params;
- request.pm_params = processed_params.pm_params;
- LOG_INFO("generate_image %dx%d", request.width, request.height);
-
- sd->rng->manual_seed(request.seed);
- sd->sampler_rng->manual_seed(request.seed);
- if (!sd->apply_loras(sd_img_gen_params->loras, sd_img_gen_params->lora_count))
- return false;
- sd->apply_circular_axes(sd_img_gen_params->circular_x, sd_img_gen_params->circular_y);
-
- const RefImageParams ref_image_params = sd->resolve_ref_image_params(sd_img_gen_params->ref_image_args);
-
- ImageVaeAxesGuard axes_guard(sd, sd_img_gen_params, request);
-
- SamplePlan plan(sd, sd_img_gen_params, request);
- sd->generation_progress.report("preparing", "inputs");
- auto latents_opt = prepare_image_generation_latents(sd,
- sd_img_gen_params,
- &request,
- &plan,
- ref_image_params);
- if (!latents_opt.has_value()) {
- return false;
- }
- ImageGenerationLatents latents = std::move(*latents_opt);
-
- auto embeds_opt = prepare_image_generation_embeds(sd,
- sd_img_gen_params,
- &request,
- &plan,
- &latents,
- ref_image_params);
- if (!embeds_opt.has_value()) {
- return false;
- }
- ImageGenerationEmbeds embeds = std::move(*embeds_opt);
- std::vector<sd::Tensor<float>> final_latents;
- int64_t denoise_start = ggml_time_ms();
- for (int b = 0; b < request.batch_count; b++) {
- sd_cancel_mode_t cancel = sd->get_cancel_flag();
- if (cancel == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation");
- return false;
- }
- if (cancel == SD_CANCEL_NEW_LATENTS) {
- LOG_INFO("cancelling new latent generation, returning %zu/%d completed latents",
- final_latents.size(),
- request.batch_count);
- break;
- }
-
- int64_t sampling_start = ggml_time_ms();
- int64_t cur_seed = request.seed_for_batch(b);
- LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed);
-
- sd->rng->manual_seed(cur_seed);
- sd->sampler_rng->manual_seed(cur_seed);
- sd::Tensor<float> noise = sd::randn_like<float>(latents.init_latent, sd->rng);
-
- sd::Tensor<float> x_0 = sd->sample(sd->diffusion_model,
- true,
- latents.init_latent,
- std::move(noise),
- embeds.cond,
- embeds.uncond,
- embeds.img_uncond,
- latents.control_image,
- request.control_strength,
- request.guidance,
- plan.eta,
- request.shifted_timestep,
- plan.sample_method,
- sd->is_flow_denoiser(),
- plan.extra_sample_args,
- plan.sigmas,
- latents.ref_latents,
- ref_image_params,
- latents.denoise_mask,
- sd::Tensor<float>(),
- 1.f,
- 0,
- static_cast<float>(request.fps),
- request.cache_params,
- true);
- int64_t sampling_end = ggml_time_ms();
- if (!x_0.empty()) {
- LOG_INFO("sampling completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
- final_latents.push_back(std::move(x_0));
- continue;
- }
-
- LOG_ERROR("sampling for image %d/%d failed after %.2fs",
- b + 1,
- request.batch_count,
- (sampling_end - sampling_start) * 1.0f / 1000);
- return false;
- }
- int64_t denoise_end = ggml_time_ms();
- LOG_INFO("generating %zu latent images completed, taking %.2fs",
- final_latents.size(),
- (denoise_end - denoise_start) * 1.0f / 1000);
- if (final_latents.empty()) {
- LOG_ERROR("no latent images generated");
- return false;
- }
-
- if (request.hires.enabled && request.hires.target_width > 0) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before hires fix");
- return false;
- }
- LOG_INFO("hires fix: upscaling to %dx%d", request.hires.target_width, request.hires.target_height);
-
- sd->generation_progress.report("upscaling", "hires");
- std::unique_ptr<ImageUpscaleModel> hires_upscaler;
- if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before hires model load");
- return false;
- }
- LOG_INFO("hires fix: loading model upscaler from '%s'", request.hires.model_path);
- hires_upscaler = load_image_upscale_model(sd->model_manager,
- sd->backend_manager,
- request.hires.model_path,
- sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER));
- if (hires_upscaler == nullptr) {
- LOG_ERROR("load hires model upscaler failed");
- return false;
- }
- }
-
- int hires_scheduler_steps = 0;
- std::vector<float> hires_sigma_sched =
- make_hires_sigma_schedule(sd,
- request.hires,
- request.sample_params,
- plan.sample_steps,
- sd->get_image_seq_len(request.hires.target_height, request.hires.target_width),
- &hires_scheduler_steps);
- LOG_INFO("hires fix: scheduler_steps=%d, denoising_strength=%.2f, sigma_sched_size=%zu%s",
- hires_scheduler_steps,
- request.hires.denoising_strength,
- hires_sigma_sched.size(),
- request.hires.custom_sigmas_count > 0 ? ", custom_sigmas=true" : "");
-
- std::vector<sd::Tensor<float>> hires_final_latents;
- int64_t hires_denoise_start = ggml_time_ms();
- sd::Tensor<float> hires_control_image;
- if (!latents.control_image.empty() &&
- sd_version_is_z_image(sd->version) &&
- sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0) {
- const int64_t hires_latent_w = request.hires.target_width / request.vae_scale_factor;
- const int64_t hires_latent_h = request.hires.target_height / request.vae_scale_factor;
- if (latents.control_image.shape()[0] != hires_latent_w ||
- latents.control_image.shape()[1] != hires_latent_h) {
- LOG_INFO("hires fix: re-encoding control latent for %dx%d", request.hires.target_width, request.hires.target_height);
- hires_control_image = encode_z_image_control_latent(sd,
- sd_image_to_tensor(sd_img_gen_params->control_image,
- request.hires.target_width,
- request.hires.target_height));
- if (hires_control_image.empty()) {
- std::vector<int64_t> ctrl_shape = latents.control_image.shape();
- ctrl_shape[0] = hires_latent_w;
- ctrl_shape[1] = hires_latent_h;
- hires_control_image = sd::ops::interpolate(latents.control_image, ctrl_shape, sd::ops::InterpolateMode::Bicubic);
- }
- } else {
- hires_control_image = latents.control_image;
- }
- }
- for (int b = 0; b < (int)final_latents.size(); b++) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation during hires fix");
- return false;
- }
- int64_t cur_seed = request.seed_for_batch(b);
- sd->rng->manual_seed(cur_seed);
- sd->sampler_rng->manual_seed(cur_seed);
-
- auto hires_source = sd->version == VERSION_P_H3_IMAGE
- ? sd::ops::slice(final_latents[b], 3, 0, 24)
- : final_latents[b];
- sd::Tensor<float> upscaled = upscale_hires_latent(sd,
- hires_source,
- request,
- hires_upscaler.get());
- if (upscaled.empty()) {
- return false;
- }
- if (sd->version == VERSION_P_H3_IMAGE) {
- upscaled = pack_ph3_image_latent(upscaled, final_latents[b]);
- }
-
- sd::Tensor<float> noise = sd::randn_like<float>(upscaled, sd->rng);
-
- sd::Tensor<float> hires_denoise_mask;
- if (!latents.denoise_mask.empty()) {
- std::vector<int64_t> mask_shape = latents.denoise_mask.shape();
- mask_shape[0] = upscaled.shape()[0];
- mask_shape[1] = upscaled.shape()[1];
- hires_denoise_mask = sd::ops::interpolate(latents.denoise_mask,
- mask_shape,
- sd::ops::InterpolateMode::NearestMax);
- }
-
- int64_t hires_sample_start = ggml_time_ms();
- sd::Tensor<float> x_0 = sd->sample(sd->diffusion_model,
- true,
- upscaled,
- std::move(noise),
- embeds.cond,
- embeds.uncond,
- embeds.img_uncond,
- !hires_control_image.empty() ? hires_control_image : latents.control_image,
- request.control_strength,
- request.guidance,
- plan.eta,
- request.shifted_timestep,
- plan.sample_method,
- sd->is_flow_denoiser(),
- plan.extra_sample_args,
- hires_sigma_sched,
- latents.ref_latents,
- ref_image_params,
- hires_denoise_mask,
- sd::Tensor<float>(),
- 1.f,
- 0,
- static_cast<float>(request.fps),
- request.cache_params,
- false);
- int64_t hires_sample_end = ggml_time_ms();
- if (!x_0.empty()) {
- LOG_INFO("hires sampling %d/%d completed, taking %.2fs",
- b + 1,
- (int)final_latents.size(),
- (hires_sample_end - hires_sample_start) * 1.0f / 1000);
- hires_final_latents.push_back(std::move(x_0));
- continue;
- }
-
- LOG_ERROR("hires sampling for image %d/%d failed after %.2fs",
- b + 1,
- (int)final_latents.size(),
- (hires_sample_end - hires_sample_start) * 1.0f / 1000);
- return false;
- }
- int64_t hires_denoise_end = ggml_time_ms();
- LOG_INFO("hires fix completed, taking %.2fs", (hires_denoise_end - hires_denoise_start) * 1.0f / 1000);
-
- final_latents = std::move(hires_final_latents);
- }
-
- int num_images = 0;
- auto result = decode_image_outputs(sd, request, final_latents, &num_images);
- if (result == nullptr) {
- return false;
- }
-
- if (infos_out) {
- try {
- infos_out->assign(num_images, info);
- // Layers and frames decoded from the same latent share its seed.
- const int images_per_latent = num_images / static_cast<int>(final_latents.size());
- for (int i = 0; i < num_images; ++i) {
- (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent));
- }
- } catch (...) {
- sd::free_images(result, num_images);
- throw;
- }
- }
- sd->lora_stat();
-
- int64_t t1 = ggml_time_ms();
- LOG_INFO("generate_image completed in %.2fs", (t1 - t0) * 1.0f / 1000);
- if (num_images_out != nullptr) {
- *num_images_out = num_images;
- }
- if (images_out != nullptr) {
- *images_out = result;
- } else {
- sd::free_images(result, num_images);
- }
- return true;
- }
-
-} // namespace sd::pipeline
diff --git a/src/pipeline/relay_vsr.cpp b/src/pipeline/relay_vsr.cpp
deleted file mode 100644
index 20d70ea..0000000
--- a/src/pipeline/relay_vsr.cpp
+++ /dev/null
@@ -1,210 +0,0 @@
-#include "api/result.h"
-#include "pipeline/video_upscale.h"
-
-#include <climits>
-
-#include "core/rng_philox.hpp"
-#include "generation_info.h"
-#include "model/diffusion/relay_vsr.hpp"
-#include "model/upscaler/flash_vsr.hpp"
-#include "name_conversion.h"
-
-namespace sd {
- namespace pipeline {
-
- static sd::Tensor<float> prepare_vsr_frame(const sd::Image& frame, int width, int height) {
- auto result = sd_image_to_tensor(frame, -1, -1, false);
- result /= 127.5f;
- result -= 1.f;
- if (width > int(frame.width)) {
- auto edge = sd::ops::slice(result, 0, frame.width - 1, frame.width);
- auto padding = sd::zeros<float>({width - frame.width, frame.height, 3, 1}) + edge;
- result = sd::ops::concat(result, padding, 0);
- }
- if (height > int(frame.height)) {
- auto edge = sd::ops::slice(result, 1, frame.height - 1, frame.height);
- auto padding = sd::zeros<float>({width, height - frame.height, 3, 1}) + edge;
- result = sd::ops::concat(result, padding, 1);
- }
- return result;
- }
-
- static sd::Tensor<float> prepare_vsr_lq(const sd::Tensor<float>& frame, int scale) {
- auto input = frame;
- if (scale == 4)
- input = sd::ops::interpolate(frame, {frame.shape()[0] * 4, frame.shape()[1] * 4, 3, 1}, sd::ops::InterpolateMode::Bilinear);
- const int patch = 4 * scale;
- const int width = int(input.shape()[0]), height = int(input.shape()[1]);
- input.reshape_({patch, width / patch, patch, height / patch, 3, 1});
- input.permute_({1, 3, 0, 2, 4, 5});
- input.reshape_({width / patch, height / patch, 3 * patch * patch, 1});
- return input;
- }
-
- class RelayVSRUpscaler : public VideoUpscaleModel {
- std::unique_ptr<RelayVSR::WanVSRRunner> wan_;
- std::unique_ptr<RelayVSR::FlashVSRRunner> flash_;
- int scale_ = 0;
- int n_threads_ = 1;
-
- public:
- bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& source_weights, int upscale_factor) override {
- if (upscale_factor != 2 && upscale_factor != 4) {
- LOG_ERROR("RelayVSR requires an explicit upscale factor of 2 or 4 matching both checkpoints");
- return false;
- }
- scale_ = upscale_factor;
- n_threads_ = runtime.n_threads;
- auto wan_backend = runtime.backends.runtime_backend(SDBackendModule::DIFFUSION);
- auto flash_backend = runtime.backends.runtime_backend(SDBackendModule::VAE);
- auto wan_params = runtime.backends.params_backend(SDBackendModule::DIFFUSION);
- auto flash_params = runtime.backends.params_backend(SDBackendModule::VAE);
- String2TensorStorage weights;
- std::map<std::string, std::string> source_names;
- for (const auto& [name, storage] : source_weights) {
- auto canonical = name;
- if (starts_with(canonical, "diffusion."))
- canonical = "wan." + canonical.substr(10);
- else if (starts_with(canonical, "vae."))
- canonical = "flash." + canonical.substr(4);
- canonical = convert_relayvsr_tensor_name(canonical);
- if (!source_names.emplace(canonical, name).second)
- throw std::runtime_error("Duplicate RelayVSR weight name: " + canonical);
- weights[canonical] = storage;
- weights[canonical].name = canonical;
- }
- wan_ = std::make_unique<RelayVSR::WanVSRRunner>(wan_backend, weights, upscale_factor, runtime.manager);
- flash_ = std::make_unique<RelayVSR::FlashVSRRunner>(flash_backend, weights, upscale_factor, runtime.manager);
- auto register_model = [&](auto& runner, ModelComponent component, SDBackendModule module, ggml_backend_t compute, ggml_backend_t params) {
- std::map<std::string, ggml_tensor*> tensors, sources;
- runner->get_param_tensors(tensors);
- for (const auto& [name, tensor] : tensors) {
- auto source = source_names.find(name);
- if (source == source_names.end())
- throw std::runtime_error("Missing RelayVSR tensor: " + name);
- sources.emplace(source->second, tensor);
- ggml_set_name(tensor, source->second.c_str());
- }
- runtime.configure_runner(*runner, module, true, module == SDBackendModule::DIFFUSION);
- return runtime.manager->register_param_tensors(component, std::move(sources),
- runtime.backends.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend,
- compute, params);
- };
- if (!register_model(wan_, ModelComponent::Diffusion, SDBackendModule::DIFFUSION, wan_backend, wan_params) ||
- !register_model(flash_, ModelComponent::VAE, SDBackendModule::VAE, flash_backend, flash_params) ||
- !runtime.manager->validate_registered_tensors())
- return false;
- return true;
- }
-
- void end() override {
- if (wan_)
- wan_->runner_end();
- if (flash_)
- flash_->runner_end();
- }
-
- bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override {
- const auto* frames = params.frames;
- const int frame_count = params.frame_count, fps = params.fps, window_size = params.window_size;
- const int64_t seed = params.seed;
- if (window_size < 2) {
- LOG_ERROR("RelayVSR requires a reference window of at least two frames");
- return false;
- }
- if (frames[0].width > uint32_t(INT_MAX / scale_ - 32) || frames[0].height > uint32_t(INT_MAX / scale_ - 32))
- return false;
- struct Reset {
- RelayVSRUpscaler* model;
- ~Reset() {
- model->wan_->reset();
- model->flash_->reset();
- }
- } reset{this};
- wan_->reset();
- flash_->reset();
- const int stride = 16 / scale_;
- const int alignment = 2 * stride;
- const int width = (int(frames[0].width) + alignment - 1) / alignment * alignment;
- const int height = (int(frames[0].height) + alignment - 1) / alignment * alignment;
- PhiloxRNG rng;
- rng.manual_seed(uint64_t(seed));
- auto keyframe = [&](int index, int slot) {
- LOG_INFO("RelayVSR reference frame %d/%d", index + 1, frame_count);
- auto input = prepare_vsr_frame(frames[index], width, height);
- sd::Tensor<float> noise({width / stride, height / stride, 1, 48});
- auto values = rng.randn(uint32_t(noise.numel()));
- std::copy(values.begin(), values.end(), noise.data());
- auto latent = wan_->compute(n_threads_, noise, prepare_vsr_lq(input, scale_));
- if (latent.empty())
- return false;
- latent.reshape_({width / stride, height / stride, 48, 1});
- return !flash_->compute(n_threads_, latent, slot).empty();
- };
- int start_slot = 0;
- if (!keyframe(0, start_slot))
- return false;
- for (int first = 0; first < frame_count;) {
- const int last = first + std::min(window_size - 1, frame_count - 1 - first);
- int end_slot = last == first ? start_slot : 1 - start_slot;
- if (last != first && !keyframe(last, end_slot))
- return false;
- flash_->history = false;
- for (int i = first; i <= last; ++i) {
- auto input = prepare_vsr_frame(frames[i], width, height);
- auto residual = flash_->compute(n_threads_, input, -1, start_slot, end_slot);
- if (residual.empty())
- return false;
- if (first != 0 && i == first)
- continue;
- auto output = sd::ops::interpolate(input, {width * scale_, height * scale_, 3, 1}, sd::ops::InterpolateMode::Bicubic);
- auto& image = result->frames[i];
- image.width = frames[i].width * scale_;
- image.height = frames[i].height * scale_;
- image.channel = 3;
- output += residual;
- output += 1.f;
- output *= 127.5f;
- output = sd::ops::slice(sd::ops::slice(output, 0, 0, image.width), 1, 0, image.height);
- if (!std::all_of(output.values().begin(), output.values().end(), [](float value) { return std::isfinite(value); }))
- throw std::runtime_error("RelayVSR produced non-finite pixels");
- output = sd::ops::clamp(output, 0.f, 255.f);
- output.permute_({2, 0, 1, 3});
- image.data = static_cast<uint8_t*>(malloc(size_t(image.width) * image.height * 3));
- if (!image.data)
- throw std::bad_alloc();
- std::transform(output.values().begin(), output.values().end(), image.data, [](float value) { return uint8_t(value); });
- LOG_INFO("RelayVSR output frame %d/%d", i + 1, frame_count);
- if (auto progress = sd_get_progress_callback())
- progress(i + 1, frame_count, 0.f, sd_get_progress_callback_data());
- }
- if (last == frame_count - 1)
- break;
- first = last;
- start_slot = end_slot;
- }
- auto info = sd::upscale_generation_info(scale_);
- info["kind"] = "video";
- info["upscale"]["model"] = "relayvsr";
- info["upscale"]["window_size"] = window_size;
- info["seed"] = seed;
- info["fps"] = fps;
- info["video_frames"] = frame_count;
- result->generation_info = sd::copy_generation_info(info);
- if (!result->generation_info)
- throw std::bad_alloc();
- return true;
- }
- };
-
- bool is_relay_vsr_model(const String2TensorStorage& weights) {
- return (weights.count("diffusion.positive_conditioning") || weights.count("diffusion.prompt_embeds")) &&
- (weights.count("diffusion.lq_proj_in.conv1.weight") || weights.count("diffusion.lq_proj.conv1.weight"));
- }
-
- std::unique_ptr<VideoUpscaleModel> create_relay_vsr_upscaler() {
- return std::make_unique<RelayVSRUpscaler>();
- }
-
- } // namespace pipeline
-} // namespace sd
diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp
deleted file mode 100644
index bbf3ab7..0000000
--- a/src/pipeline/request.cpp
+++ /dev/null
@@ -1,524 +0,0 @@
-#include "request.h"
-#include "conditioning/conditioner.hpp"
-#include "config.h"
-#include "core/option_types.h"
-
-#include <algorithm>
-#include <cmath>
-#include <cstdlib>
-#include <ctime>
-
-#include "diffusion_engine.h"
-#include "generation.h"
-#include "model/vae/vae.hpp"
-#include "model_definition.h"
-#include "runtime/denoiser.hpp"
-
-namespace sd::pipeline {
-
- const char* sampling_methods_str[] = {
- "Euler",
- "Euler A",
- "Heun",
- "DPM2",
- "DPM++ (2s)",
- "DPM++ (2M)",
- "modified DPM++ (2M)",
- "iPNDM",
- "iPNDM_v",
- "LCM",
- "DDIM \"trailing\"",
- "TCD",
- "Res Multistep",
- "Res 2s",
- "ER-SDE",
- "Euler CFG++",
- "Euler A CFG++",
- "Euler GE",
- "DPM++ (2M) SDE",
- "DPM++ (2M) SDE BT",
- "LMS",
- };
-
- static_assert(SAMPLE_METHOD_COUNT == sizeof(sampling_methods_str) / sizeof(sampling_methods_str[0]),
- "\nnumber of elements in sampling_methods_str[] != SAMPLE_METHOD_COUNT");
-
- static bool sd_version_supports_img_cfg(SDVersion version, bool has_ref_images) {
- return sd_version_is_inpaint_or_unet_edit(version) ||
- (has_ref_images && sd_version_supports_ref_latent_img_cfg(version));
- }
-
- enum sample_method_t default_sample_method(const StableDiffusionGGML* sd) {
- const auto* definition = sd ? find_model_definition(sd->version) : nullptr;
- return definition ? definition->sampling.default_sample_method : EULER_A_SAMPLE_METHOD;
- }
-
- enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method) {
- if (sd != nullptr) {
- auto edm_v_denoiser = std::dynamic_pointer_cast<EDMVDenoiser>(sd->denoiser);
- if (edm_v_denoiser) {
- return EXPONENTIAL_SCHEDULER;
- }
- }
- if (sample_method == LCM_SAMPLE_METHOD || sample_method == TCD_SAMPLE_METHOD) {
- return LCM_SCHEDULER;
- } else if (sample_method == DDIM_TRAILING_SAMPLE_METHOD) {
- return SIMPLE_SCHEDULER;
- }
- if (const auto* definition = sd ? find_model_definition(sd->version) : nullptr) {
- return definition->sampling.default_scheduler;
- }
- return DISCRETE_SCHEDULER;
- }
-
- int64_t resolve_seed(int64_t seed) {
- if (seed >= 0) {
- return seed;
- }
- srand((int)time(nullptr));
- return rand();
- }
-
- static enum sample_method_t resolve_sample_method(StableDiffusionGGML* sd, enum sample_method_t sample_method) {
- if (sample_method == SAMPLE_METHOD_COUNT) {
- return default_sample_method(sd);
- }
- return sample_method;
- }
-
- static scheduler_t resolve_scheduler(StableDiffusionGGML* sd,
- scheduler_t scheduler,
- enum sample_method_t sample_method) {
- if (scheduler == SCHEDULER_COUNT) {
- return default_scheduler(sd, sample_method);
- }
- return scheduler;
- }
-
- static float resolve_eta(StableDiffusionGGML* sd,
- float eta,
- enum sample_method_t sample_method) {
- if (eta == INFINITY) {
- if (sd->version == VERSION_HIDREAM_O1) {
- return 8.f;
- }
- switch (sample_method) {
- case DDIM_TRAILING_SAMPLE_METHOD:
- case TCD_SAMPLE_METHOD:
- case RES_MULTISTEP_SAMPLE_METHOD:
- case RES_2S_SAMPLE_METHOD:
- return 0.0f;
- case EULER_A_SAMPLE_METHOD:
- case DPMPP2S_A_SAMPLE_METHOD:
- case ER_SDE_SAMPLE_METHOD:
- case EULER_A_CFG_PP_SAMPLE_METHOD:
- case DPMPP2M_SDE_SAMPLE_METHOD:
- case DPMPP2M_SDE_BT_SAMPLE_METHOD:
- return 1.0f;
- default:;
- }
- return 0.0f;
- }
- return eta;
- }
-
- GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params) {
- prompt = SAFE_STR(sd_img_gen_params->prompt);
- negative_prompt = SAFE_STR(sd_img_gen_params->negative_prompt);
- width = sd_img_gen_params->width;
- height = sd_img_gen_params->height;
- vae_scale_factor = sd->get_vae_scale_factor();
- diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
- seed = sd_img_gen_params->seed;
- batch_count = sd_img_gen_params->batch_count;
- qwen_image_layers = std::max(0, sd_img_gen_params->qwen_image_layers);
- is_layered_image = sd->version == VERSION_QWEN_IMAGE_LAYERED;
- clip_skip = sd_img_gen_params->clip_skip;
- shifted_timestep = sd_img_gen_params->sample_params.shifted_timestep;
- strength = sd_img_gen_params->strength;
- control_strength = sd_img_gen_params->control_strength;
- control_type = sd_img_gen_params->control_type;
- has_ref_images = sd_img_gen_params->ref_images_count > 0;
- guidance = sd_img_gen_params->sample_params.guidance;
- pm_params = sd_img_gen_params->pm_params;
- pulid_params = sd_img_gen_params->pulid_params;
- hires = sd_img_gen_params->hires;
- vae_tiling_params = sd_img_gen_params->vae_tiling_params;
- cache_params = &sd_img_gen_params->cache;
- sample_params = sd_img_gen_params->sample_params;
- resolve(sd);
- if (!sd_img_gen_params->control_image.data)
- models.erase("control_net");
- if (!sd_img_gen_params->ip_adapter_image.data)
- models.erase("ip_adapter");
- if (!pm_params.id_images || pm_params.id_images_count <= 0)
- models.erase("photo_maker");
- if (!pulid_params.id_embedding_path || !*pulid_params.id_embedding_path)
- models.erase("pulid_weights");
- }
-
- GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params) {
- prompt = SAFE_STR(sd_vid_gen_params->prompt);
- negative_prompt = SAFE_STR(sd_vid_gen_params->negative_prompt);
- width = sd_vid_gen_params->width;
- height = sd_vid_gen_params->height;
- requested_frames = std::max(1, sd_vid_gen_params->video_frames);
- frames = sd->align_video_frames(requested_frames);
- clip_skip = sd_vid_gen_params->clip_skip;
- fps = std::max(1, sd_vid_gen_params->fps);
- if (sd->version == VERSION_WAN2_2_S2V && sd_vid_gen_params->fps != 16) {
- LOG_WARN("Wan2.2 S2V uses 16 fps; overriding requested fps %d", sd_vid_gen_params->fps);
- fps = 16;
- } else if (sd_version_is_minimax_h3(sd->version) && fps != 24) {
- LOG_WARN("MiniMax-H3 uses 24 fps; overriding requested fps %d", fps);
- fps = 24;
- }
- vae_scale_factor = sd->get_vae_scale_factor();
- diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
- seed = sd_vid_gen_params->seed;
- strength = sd_vid_gen_params->strength;
- cache_params = &sd_vid_gen_params->cache;
- vace_strength = sd_vid_gen_params->vace_strength;
- guidance = sd_vid_gen_params->sample_params.guidance;
- high_noise_guidance = sd_vid_gen_params->high_noise_sample_params.guidance;
- hires = sd_vid_gen_params->hires;
- vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
- sample_params = sd_vid_gen_params->sample_params;
- high_noise_sample_params = sd_vid_gen_params->high_noise_sample_params;
- use_high_noise = sd->high_noise_diffusion_model != nullptr;
- use_vace = sd->diffusion_model && (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
- sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
- shifted_timestep = sample_params.shifted_timestep;
- if (sd->config_ && sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version))
- frames = std::min(requested_frames, 32);
- resolve(sd);
- models.erase("control_net");
- models.erase("ip_adapter");
- models.erase("photo_maker");
- models.erase("pulid_weights");
- if (frames != requested_frames) {
- LOG_WARN("align video frames from %d to %d for %s",
- requested_frames,
- frames,
- sd_version_name(sd->version));
- }
- }
-
- void GenerationRequest::align_generation_request_size() {
- align_image_size(&width, &height, "generation request");
- }
-
- void GenerationRequest::align_image_size(int* target_width, int* target_height, const char* label) {
- int spatial_multiple = vae_scale_factor * diffusion_model_down_factor;
- int width_offset = align_up_offset(*target_width, spatial_multiple);
- int height_offset = align_up_offset(*target_height, spatial_multiple);
- if (width_offset <= 0 && height_offset <= 0) {
- return;
- }
-
- int original_width = *target_width;
- int original_height = *target_height;
-
- *target_width += width_offset;
- *target_height += height_offset;
- LOG_WARN("align %s up %dx%d to %dx%d (multiple=%d)",
- label,
- original_width,
- original_height,
- *target_width,
- *target_height,
- spatial_multiple);
- }
-
- int64_t GenerationRequest::seed_for_batch(int index) const {
- return static_cast<int64_t>((static_cast<uint64_t>(seed) + index) & INT64_MAX);
- }
-
- void GenerationRequest::resolve_hires() {
- if (!hires.enabled) {
- return;
- }
- if (hires.upscaler == SD_HIRES_UPSCALER_NONE) {
- hires.enabled = false;
- return;
- }
- if (hires.upscaler < SD_HIRES_UPSCALER_NONE || hires.upscaler >= SD_HIRES_UPSCALER_COUNT) {
- LOG_WARN("hires upscaler '%d' is invalid, disabling hires", hires.upscaler);
- hires.enabled = false;
- return;
- }
- if (hires.upscaler == SD_HIRES_UPSCALER_MODEL && strlen(SAFE_STR(hires.model_path)) == 0) {
- LOG_WARN("hires model upscaler requires a model path, disabling hires");
- hires.enabled = false;
- return;
- }
- if (hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0) {
- LOG_WARN("hires scale must be positive when no target size is set, disabling hires");
- hires.enabled = false;
- return;
- }
- if (hires.custom_sigmas_count < 0) {
- LOG_WARN("hires custom sigmas count is negative, ignoring custom sigmas");
- hires.custom_sigmas = nullptr;
- hires.custom_sigmas_count = 0;
- }
- if (hires.custom_sigmas_count > 0 && hires.custom_sigmas == nullptr) {
- LOG_WARN("hires custom sigmas count is positive but custom sigmas are null, ignoring custom sigmas");
- hires.custom_sigmas_count = 0;
- }
- if (hires.custom_sigmas_count == 1) {
- LOG_WARN("hires custom sigmas requires at least two values, ignoring custom sigmas");
- hires.custom_sigmas = nullptr;
- hires.custom_sigmas_count = 0;
- }
- hires.denoising_strength = std::clamp(hires.denoising_strength, 0.0001f, 1.f);
- hires.steps = std::max(0, hires.steps);
-
- if (hires.target_width > 0 && hires.target_height > 0) {
- // pass
- } else if (hires.target_width > 0) {
- hires.target_height = hires.target_width;
- } else if (hires.target_height > 0) {
- hires.target_width = hires.target_height;
- } else {
- hires.target_width = static_cast<int>(std::round(width * hires.scale));
- hires.target_height = static_cast<int>(std::round(height * hires.scale));
- }
-
- if (hires.target_width <= 0 || hires.target_height <= 0) {
- LOG_WARN("hires target size is not positive, disabling hires");
- hires.enabled = false;
- return;
- }
- align_image_size(&hires.target_width, &hires.target_height, "hires target");
- }
-
- void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd,
- sd::GuidanceConfig* guidance,
- bool* use_uncond,
- bool* use_img_uncond,
- bool has_ref_images,
- const char* stage_name) {
- GGML_ASSERT(guidance != nullptr);
- GGML_ASSERT(use_uncond != nullptr);
- GGML_ASSERT(use_img_uncond != nullptr);
- // out_img_uncond + text_cfg_scale * (out_cond - out_uncond) + image_cfg_scale * (out_uncond - out_img_uncond)
- // -> text_cfg_scale * out_cond + (image_cfg_scale - text_cfg_scale) * out_uncond + (1 - image_cfg_scale) * out_img_uncond
- // out_cond : prompt, image latent
- // out_uncond : negative prompt, image latent
- // out_img_uncond : negative prompt, zero image latent
- // image_cfg_scale == 1 reduces 3-cond CFG to 2-cond CFG.
- bool img_cfg_was_set = std::isfinite(guidance->img_cfg);
- if (!img_cfg_was_set) {
- guidance->img_cfg = 1.f;
- }
-
- if (!sd_version_supports_img_cfg(sd->version, has_ref_images)) {
- if (img_cfg_was_set && guidance->img_cfg != 1.f) {
- LOG_WARN("3-conditioning CFG is not supported with this model, disabling it for better performance");
- }
- guidance->img_cfg = 1.f;
- }
-
- if (guidance->img_cfg != guidance->txt_cfg) {
- *use_uncond = true;
- }
-
- if (guidance->img_cfg != 1.f) {
- *use_img_uncond = true;
- }
-
- if (guidance->txt_cfg < 1.f) {
- const char* prefix = stage_name == nullptr ? "" : stage_name;
- if (guidance->txt_cfg == 0.f) {
- LOG_WARN("%sunconditioned mode, images won't follow the prompt (use cfg-scale=1 for distilled models)",
- prefix);
- } else {
- LOG_WARN("%scfg value out of expected range may produce unexpected results", prefix);
- }
- }
- }
-
- void GenerationRequest::resolve(StableDiffusionGGML* sd) {
- if (sd->config_) {
- models = sd->config_->models;
- if (!use_high_noise)
- models.erase("high_noise_diffusion_model");
- if (!sd->clip_vision)
- models.erase("clip_vision");
- if (!sd->audio_vae_model || frames <= 0)
- models.erase("audio_vae");
- if (!sd->audio_encoder || frames <= 0)
- models.erase("audio_encoder");
- if (!sd->config_->animatediff_loaded || !sd_version_supports_animatediff(sd->version) || frames <= 0)
- models.erase("motion_module");
- if (sd->uses_tae() && !sd->tae_preview_only())
- models.erase("vae");
- else
- models.erase("taesd");
- }
- if (sd_version_is_minit2i(sd->version) && (width != 512 || height != 512)) {
- width = 512;
- height = 512;
- LOG_WARN("align to fixed size 512x512 for MiniT2I");
- }
- align_generation_request_size();
- resolve_hires();
- if (sd->first_stage_model)
- vae_tiling_params = sd->first_stage_model->resolve_tiling_params(vae_tiling_params);
- seed = resolve_seed(seed);
- if (sd->cond_stage_model)
- clip_skip = sd->cond_stage_model->resolve_clip_skip(clip_skip);
-
- resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
- if (sd->high_noise_diffusion_model) {
- resolve_guidance(sd,
- &high_noise_guidance,
- &use_high_noise_uncond,
- &use_high_noise_img_uncond,
- has_ref_images,
- "high noise: ");
- }
-
- if (shifted_timestep > 0 && !sd_version_is_sdxl(sd->version)) {
- LOG_WARN("timestep shifting is only supported for SDXL models!");
- shifted_timestep = 0;
- }
- sample_params.guidance = guidance;
- sample_params.shifted_timestep = shifted_timestep;
- sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method);
- sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method);
- sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method);
- sd->set_flow_shift(sample_params.flow_shift);
- if (sd->is_flow_denoiser())
- sample_params.flow_shift = sd->active_flow_shift;
- if (frames > 0) {
- high_noise_sample_params.guidance = high_noise_guidance;
- high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
- high_noise_sample_params.scheduler = sample_params.scheduler;
- high_noise_sample_params.flow_shift = sample_params.flow_shift;
- high_noise_sample_params.shifted_timestep = sample_params.shifted_timestep;
- high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
- }
- }
-
- SamplePlan::SamplePlan(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- const GenerationRequest& request) {
- sample_method = request.sample_params.sample_method;
- extra_sample_args = request.sample_params.extra_sample_args;
- eta = request.sample_params.eta;
- sample_steps = request.sample_params.sample_steps;
- resolve(sd, &request, &request.sample_params);
- }
-
- SamplePlan::SamplePlan(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- const GenerationRequest& request) {
- sample_method = request.sample_params.sample_method;
- extra_sample_args = request.sample_params.extra_sample_args;
- eta = request.sample_params.eta;
- sample_steps = request.sample_params.sample_steps;
- if (sd->high_noise_diffusion_model) {
- high_noise_sample_steps = request.high_noise_sample_params.sample_steps;
- high_noise_sample_method = request.high_noise_sample_params.sample_method;
- high_noise_extra_sample_args = request.high_noise_sample_params.extra_sample_args;
- high_noise_eta = request.high_noise_sample_params.eta;
- }
- moe_boundary = sd_vid_gen_params->moe_boundary;
- resolve(sd, &request, &request.sample_params);
- }
-
- void SamplePlan::resolve(StableDiffusionGGML* sd,
- const GenerationRequest* request,
- const sd::SampleConfig* sample_params) {
- scheduler = sample_params->scheduler;
- total_steps = sample_steps + std::max(0, high_noise_sample_steps);
-
- if (sample_params->custom_sigmas_count > 0) {
- sigmas = std::vector<float>(sample_params->custom_sigmas,
- sample_params->custom_sigmas + sample_params->custom_sigmas_count);
- if (total_steps != sample_params->custom_sigmas_count - 1) {
- total_steps = sample_params->custom_sigmas_count - 1;
- LOG_WARN("total_steps != custom_sigmas_count - 1, set total_steps to %d", total_steps);
- if ((sample_steps > total_steps) || (high_noise_sample_steps <= 0 && sample_steps != total_steps)) {
- sample_steps = total_steps;
- LOG_WARN("total_steps != custom_sigmas_count - 1, set sample_steps to %d", sample_steps);
- }
- if (high_noise_sample_steps > 0) {
- high_noise_sample_steps = total_steps - sample_steps;
- LOG_WARN("total_steps != custom_sigmas_count - 1, set high_noise_sample_steps to %d", high_noise_sample_steps);
- }
- }
- } else {
- int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
- if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
- int latent_frames = ((request->frames - 1) / 8) + 1;
- sample_seq_len *= latent_frames;
- } else if (sd_version_is_minimax_h3(sd->version) && request->frames > 0) {
- sample_seq_len *= sd->video_frames_to_latent_frames(request->frames);
- }
- sigmas = sd->denoiser->get_sigmas(total_steps,
- sample_seq_len,
- scheduler,
- sd->version,
- sample_params->extra_sample_args);
- }
-
- if (high_noise_sample_steps < 0) {
- for (size_t i = 0; i < sigmas.size(); ++i) {
- if (sigmas[i] < moe_boundary) {
- high_noise_sample_steps = static_cast<int>(i);
- break;
- }
- }
- LOG_VERBOSE("switching from high noise model at step %d", high_noise_sample_steps);
- }
-
- LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]);
- if (high_noise_sample_steps > 0) {
- LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]);
- }
- }
-
- std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
- const sd::HiresConfig& hires,
- const sd::SampleConfig& sample_params,
- int default_steps,
- int sample_seq_len,
- int* scheduler_steps_out) {
- if (scheduler_steps_out != nullptr) {
- *scheduler_steps_out = 0;
- }
-
- if (hires.custom_sigmas_count > 0 && hires.custom_sigmas != nullptr) {
- std::vector<float> custom_sigmas(hires.custom_sigmas,
- hires.custom_sigmas + hires.custom_sigmas_count);
- if (scheduler_steps_out != nullptr) {
- *scheduler_steps_out = static_cast<int>(custom_sigmas.size()) - 1;
- }
- return custom_sigmas;
- }
-
- int effective_steps = hires.steps > 0 ? hires.steps : default_steps;
- effective_steps = std::max(1, effective_steps);
-
- // sd-webui behavior: scale up total steps so trimming by denoising_strength yields exactly hires_steps effective steps,
- // unlike img2img which trims from a fixed step count.
- int scheduler_steps = static_cast<int>(effective_steps / hires.denoising_strength);
- scheduler_steps = std::max(1, scheduler_steps);
-
- scheduler_t scheduler = sample_params.scheduler;
- std::vector<float> sigmas = sd->denoiser->get_sigmas(scheduler_steps,
- sample_seq_len,
- scheduler,
- sd->version,
- sample_params.extra_sample_args);
- if (scheduler_steps_out != nullptr) {
- *scheduler_steps_out = scheduler_steps;
- }
- return std::vector<float>(sigmas.begin() + std::max(0, scheduler_steps - effective_steps),
- sigmas.end());
- }
-
-} // namespace sd::pipeline
diff --git a/src/pipeline/request.h b/src/pipeline/request.h
deleted file mode 100644
index 1e15db8..0000000
--- a/src/pipeline/request.h
+++ /dev/null
@@ -1,121 +0,0 @@
-#ifndef __SD_PIPELINE_REQUEST_H__
-#define __SD_PIPELINE_REQUEST_H__
-
-#include "core/option_types.h"
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "config.h"
-#include "stable-diffusion.h"
-
-class StableDiffusionGGML;
-
-namespace sd::pipeline {
-
- extern const char* sampling_methods_str[];
-
- enum sample_method_t default_sample_method(const StableDiffusionGGML* sd);
-
- enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
-
- int64_t resolve_seed(int64_t seed);
-
- struct GenerationRequest {
- std::string prompt;
- std::string negative_prompt;
- std::map<std::string, std::string> models;
- int width = -1;
- int height = -1;
- int clip_skip = -1;
- int vae_scale_factor = -1;
- int diffusion_model_down_factor = -1;
- int64_t seed = -1;
- bool use_uncond = false;
- bool use_img_uncond = false;
- bool use_high_noise_uncond = false;
- bool use_high_noise_img_uncond = false;
- bool has_ref_images = false;
- bool is_layered_image = false;
- bool use_high_noise = false;
- bool use_vace = false;
- const sd::CacheConfig* cache_params = nullptr;
- int batch_count = 1;
- int qwen_image_layers = 3;
- int shifted_timestep = 0;
- float strength = 1.f;
- float control_strength = 0.f;
- sd_control_type_t control_type = SD_CONTROL_TYPE_NONE;
- sd::SampleConfig sample_params = {};
- sd::SampleConfig high_noise_sample_params = {};
- sd::GuidanceConfig guidance = {};
- sd::GuidanceConfig high_noise_guidance = {};
- sd::PhotoMakerConfig pm_params = {};
- sd::PuLIDConfig pulid_params = {};
- sd::HiresConfig hires = {};
- sd::TilingConfig vae_tiling_params = {};
- int frames = -1;
- int requested_frames = -1;
- int fps = 16;
- float vace_strength = 1.f;
-
- GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params);
-
- GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params);
-
- int64_t seed_for_batch(int index) const;
-
- void align_generation_request_size();
-
- void align_image_size(int* target_width, int* target_height, const char* label);
-
- void resolve_hires();
-
- static void resolve_guidance(StableDiffusionGGML* sd,
- sd::GuidanceConfig* guidance,
- bool* use_uncond,
- bool* use_img_uncond,
- bool has_ref_images,
- const char* stage_name = nullptr);
-
- void resolve(StableDiffusionGGML* sd);
- };
-
- struct SamplePlan {
- enum sample_method_t sample_method = SAMPLE_METHOD_COUNT;
- enum sample_method_t high_noise_sample_method = SAMPLE_METHOD_COUNT;
- const char* extra_sample_args = nullptr;
- const char* high_noise_extra_sample_args = nullptr;
- float eta = 0.f;
- float high_noise_eta = 0.f;
- int sample_steps = 0;
- int high_noise_sample_steps = 0;
- int total_steps = 0;
- scheduler_t scheduler = SCHEDULER_COUNT;
- float moe_boundary = 0.f;
- std::vector<float> sigmas;
-
- SamplePlan(StableDiffusionGGML* sd,
- const sd::ImageConfig* sd_img_gen_params,
- const GenerationRequest& request);
-
- SamplePlan(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- const GenerationRequest& request);
-
- void resolve(StableDiffusionGGML* sd,
- const GenerationRequest* request,
- const sd::SampleConfig* sample_params);
- };
-
- std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
- const sd::HiresConfig& hires,
- const sd::SampleConfig& sample_params,
- int default_steps,
- int sample_seq_len,
- int* scheduler_steps_out);
-
-} // namespace sd::pipeline
-
-#endif // __SD_PIPELINE_REQUEST_H__
diff --git a/src/pipeline/seedvr2.cpp b/src/pipeline/seedvr2.cpp
deleted file mode 100644
index cb07876..0000000
--- a/src/pipeline/seedvr2.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-#include "api/result.h"
-#include "pipeline/video_upscale.h"
-
-#include <climits>
-
-#include "core/rng_philox.hpp"
-#include "generation_info.h"
-#include "model/diffusion/seedvr2.hpp"
-#include "model/vae/seedvr2_vae.hpp"
-
-namespace sd {
- namespace pipeline {
-
- class SeedVR2Upscaler : public VideoUpscaleModel {
- std::unique_ptr<SeedVR2::SeedVR2Runner> dit_;
- std::unique_ptr<SeedVR2::SeedVR2VAERunner> vae_;
- int scale_ = 0;
- int n_threads_ = 1;
-
- public:
- bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& weights, int upscale_factor) override {
- scale_ = upscale_factor;
- n_threads_ = runtime.n_threads;
- dit_ = std::make_unique<SeedVR2::SeedVR2Runner>(runtime.backends.runtime_backend(SDBackendModule::DIFFUSION), weights, runtime.manager);
- vae_ = std::make_unique<SeedVR2::SeedVR2VAERunner>(runtime.backends.runtime_backend(SDBackendModule::VAE), weights, runtime.manager);
- std::set<std::string> used;
- auto register_model = [&](auto& runner, ModelComponent component, SDBackendModule module) {
- std::map<std::string, ggml_tensor*> tensors;
- runner->get_param_tensors(tensors);
- for (const auto& [name, tensor] : tensors) {
- if (!weights.count(name))
- throw std::runtime_error("Missing SeedVR2 tensor: " + name);
- used.insert(name);
- }
- runtime.configure_runner(*runner, module, module == SDBackendModule::VAE, module == SDBackendModule::VAE);
- return runtime.manager->register_param_tensors(component, std::move(tensors),
- runtime.backends.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend,
- runtime.backends.runtime_backend(module), runtime.backends.params_backend(module));
- };
- if (!register_model(dit_, ModelComponent::Diffusion, SDBackendModule::DIFFUSION) ||
- !register_model(vae_, ModelComponent::VAE, SDBackendModule::VAE) || !runtime.manager->validate_registered_tensors())
- return false;
- for (const auto& [name, storage] : weights) {
- if (!used.count(name) && name != "diffusion.negative_conditioning" && !ends_with(name, ".comfy_quant"))
- throw std::runtime_error("Unused SeedVR2 tensor: " + name);
- }
- LOG_INFO("SeedVR2 validated %zu parameter tensors; negative conditioning is unused at guidance 1", used.size());
- return true;
- }
-
- void end() override {
- if (dit_)
- dit_->runner_end();
- if (vae_)
- vae_->runner_end();
- }
-
- bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override {
- const int chunk = params.temporal_chunk_size;
- if (chunk < 1 || chunk > INT_MAX - 3 || (chunk - 1) % 4) {
- LOG_ERROR("SeedVR2 temporal chunk size must be 1 or 4n+1 input frames");
- return false;
- }
- struct Reset {
- SeedVR2Upscaler* model;
- ~Reset() { model->end(); }
- } reset{this};
- end();
- const int width = int(params.frames[0].width) * scale_, height = int(params.frames[0].height) * scale_;
- if (width > INT_MAX - 15 || height > INT_MAX - 15)
- return false;
- const int pw = (width + 15) / 16 * 16, ph = (height + 15) / 16 * 16;
- PhiloxRNG rng;
- rng.manual_seed(uint64_t(params.seed));
- for (int start = 0; start < params.frame_count;) {
- const int count = std::min(chunk, params.frame_count - start);
- const int padded_count = count == 1 ? 1 : ((count - 1 + 3) / 4) * 4 + 1;
- LOG_INFO("SeedVR2 chunk %d..%d/%d", start + 1, start + count, params.frame_count);
- const int64_t begin = ggml_time_us();
- std::vector<sd::Tensor<float>> images;
- for (int i = 0; i < count; ++i) {
- auto image = sd_image_to_tensor(params.frames[start + i], -1, -1, false);
- image /= 255.f;
- image = sd::ops::interpolate(image, {width, height, 3, 1}, sd::ops::InterpolateMode::Bicubic);
- image = sd::ops::clamp(image, 0.f, 1.f);
- if (pw != width)
- image = sd::ops::concat(image, sd::zeros<float>({pw - width, height, 3, 1}), 0);
- if (ph != height)
- image = sd::ops::concat(image, sd::zeros<float>({pw, ph - height, 3, 1}), 1);
- image *= 2.f;
- image -= 1.f;
- images.push_back(image);
- }
- auto pixels = images.front();
- for (int i = 1; i < padded_count; ++i)
- pixels = sd::ops::concat(pixels, images[std::min(i, count - 1)], 3);
- pixels.permute_({0, 1, 3, 2});
- const int64_t prepared = ggml_time_us();
- auto latent = vae_->compute(n_threads_, pixels, false);
- if (latent.empty())
- return false;
- latent *= 0.9152f;
- const int64_t encoded = ggml_time_us();
- const int lw = pw / 8, lh = ph / 8, lt = (padded_count - 1) / 4 + 1;
- sd::Tensor<float> noise({lw, lh, lt, 16});
- auto values = rng.randn(uint32_t(noise.numel()));
- std::copy(values.begin(), values.end(), noise.data());
- auto mask = sd::zeros<float>({lw, lh, lt, 1});
- mask += 1.f;
- auto input = sd::ops::concat(sd::ops::concat(noise, latent, 3), mask, 3);
- input.reshape_({2, lw / 2, 2, lh / 2, lt, 33});
- input.permute_({5, 0, 2, 1, 3, 4});
- input.reshape_({132, (lw / 2) * (lh / 2) * lt});
- auto prediction = dit_->compute(n_threads_, input, lw / 2, lh / 2, lt);
- if (prediction.empty())
- return false;
- prediction.reshape_({16, 2, 2, lw / 2, lh / 2, lt});
- prediction.permute_({1, 3, 2, 4, 5, 0});
- prediction.reshape_({lw, lh, lt, 16});
- latent = (noise - prediction) / 0.9152f;
- const int64_t denoised = ggml_time_us();
- auto decoded = vae_->compute(n_threads_, latent, true);
- if (decoded.empty())
- return false;
- const int64_t decoded_at = ggml_time_us();
- if (!std::all_of(decoded.values().begin(), decoded.values().end(), [](float x) { return std::isfinite(x); }))
- throw std::runtime_error("SeedVR2 produced non-finite pixels");
- decoded = sd::ops::clamp((decoded + 1.f) * 127.5f, 0.f, 255.f);
- for (int i = 0; i < count; ++i) {
- auto output = sd::ops::slice(sd::ops::slice(sd::ops::slice(decoded, 2, i, i + 1), 0, 0, width), 1, 0, height);
- output.permute_({3, 0, 1, 2});
- auto& image = result->frames[start + i];
- image.width = width;
- image.height = height;
- image.channel = 3;
- image.data = static_cast<uint8_t*>(malloc(size_t(width) * height * 3));
- if (!image.data)
- throw std::bad_alloc();
- std::transform(output.values().begin(), output.values().end(), image.data, [](float x) { return uint8_t(x); });
- }
- if (auto progress = sd_get_progress_callback())
- progress(start + count, params.frame_count, 0.f, sd_get_progress_callback_data());
- LOG_INFO("SeedVR2 seconds: preprocess %.3f, VAE encode %.3f, DiT and latent transforms %.3f, VAE decode %.3f, postprocess %.3f",
- (prepared - begin) / 1e6, (encoded - prepared) / 1e6, (denoised - encoded) / 1e6, (decoded_at - denoised) / 1e6, (ggml_time_us() - decoded_at) / 1e6);
- start += count;
- }
- auto info = sd::upscale_generation_info(scale_);
- info["kind"] = "video";
- info["upscale"]["model"] = "seedvr2";
- info["upscale"]["temporal_chunk_size"] = chunk;
- info["seed"] = params.seed;
- info["fps"] = params.fps;
- info["video_frames"] = params.frame_count;
- result->generation_info = sd::copy_generation_info(info);
- return result->generation_info != nullptr;
- }
- };
-
- bool is_seedvr2_model(const String2TensorStorage& weights) {
- return weights.count("diffusion.vid_in.proj.weight") && weights.count("diffusion.blocks.0.attn.rope.rope.freqs") &&
- weights.count("diffusion.positive_conditioning");
- }
-
- std::unique_ptr<VideoUpscaleModel> create_seedvr2_upscaler() {
- return std::make_unique<SeedVR2Upscaler>();
- }
- }
-}
diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp
deleted file mode 100644
index 529bbb6..0000000
--- a/src/pipeline/video.cpp
+++ /dev/null
@@ -1,2317 +0,0 @@
-#include "core/option_types.h"
-#include "generation.h"
-#include "generation_info.h"
-
-#include <algorithm>
-#include <cmath>
-#include <cstdlib>
-#include <optional>
-#include <utility>
-
-#include "conditioning/wan_audio.h"
-#include "core/rng.hpp"
-#include "core/rng_philox.hpp"
-#include "diffusion_engine.h"
-#include "model/diffusion/minimax_h3.hpp"
-#include "model/upscaler/h3_latent_upscaler.hpp"
-#include "model/upscaler/ltx_latent_upscaler.hpp"
-#include "model/vae/audio_vae.hpp"
-#include "model/vae/vae.hpp"
-#include "request.h"
-#include "runtime/denoiser.hpp"
-#include "runtime/image_preprocess.h"
-
-namespace sd::pipeline {
-
- static sd::Audio* waveform_to_sd_audio(const StableDiffusionGGML* sd,
- const sd::Tensor<float>& waveform) {
- if (sd == nullptr || waveform.empty()) {
- return nullptr;
- }
-
- int64_t sample_count = waveform.shape()[0];
- int64_t channels = waveform.shape().size() > 1 ? waveform.shape()[1] : 1;
- if (sample_count <= 0 || channels <= 0) {
- return nullptr;
- }
-
- sd::Audio* audio = (sd::Audio*)malloc(sizeof(sd::Audio));
- if (audio == nullptr) {
- return nullptr;
- }
-
- audio->sample_rate = static_cast<uint32_t>(sd->audio_vae_model != nullptr ? sd->audio_vae_model->output_sample_rate() : 0);
- audio->channels = static_cast<uint32_t>(channels);
- audio->sample_count = static_cast<uint64_t>(sample_count);
- size_t sample_bytes = waveform.numel() * sizeof(float);
- audio->data = (float*)malloc(sample_bytes);
- if (audio->data == nullptr) {
- free(audio);
- return nullptr;
- }
-
- auto wavaform_t = waveform.permute({1, 0, 2, 3});
- std::memcpy(audio->data, wavaform_t.data(), sample_bytes);
-
- return audio;
- }
-
- static float ltxv_latent_corner_to_pixel_frame(int64_t corner_index,
- int temporal_scale,
- bool causal_temporal_positioning) {
- float pixel_t = static_cast<float>(corner_index * temporal_scale);
- if (causal_temporal_positioning) {
- pixel_t = std::max(0.f, pixel_t + 1.f - static_cast<float>(temporal_scale));
- }
- return pixel_t;
- }
-
- static void set_ltxv_video_position(sd::Tensor<float>* positions,
- int64_t token,
- float t_start,
- float t_end,
- float h_start,
- float h_end,
- float w_start,
- float w_end) {
- positions->index(0, 0, token, 0) = t_start;
- positions->index(1, 0, token, 0) = t_end;
- positions->index(0, 1, token, 0) = h_start;
- positions->index(1, 1, token, 0) = h_end;
- positions->index(0, 2, token, 0) = w_start;
- positions->index(1, 2, token, 0) = w_end;
- }
-
- static sd::Tensor<float> build_ltxv_video_positions(int64_t width,
- int64_t height,
- int64_t target_latent_frames,
- int64_t keyframe_latent_frames,
- int keyframe_frame_idx,
- int keyframe_pixel_frames,
- int fps,
- int spatial_scale,
- int temporal_scale,
- bool causal_temporal_positioning) {
- GGML_ASSERT(width > 0 && height > 0 && target_latent_frames > 0);
- GGML_ASSERT(keyframe_latent_frames > 0);
- GGML_ASSERT(fps > 0);
-
- int64_t total_tokens = width * height * (target_latent_frames + keyframe_latent_frames);
- sd::Tensor<float> positions({2, 3, total_tokens, 1});
- int64_t token = 0;
-
- for (int64_t t = 0; t < target_latent_frames; t++) {
- float t_start = ltxv_latent_corner_to_pixel_frame(t, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
- float t_end = ltxv_latent_corner_to_pixel_frame(t + 1, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
- for (int64_t h = 0; h < height; h++) {
- float h_start = static_cast<float>(h * spatial_scale);
- float h_end = static_cast<float>((h + 1) * spatial_scale);
- for (int64_t w = 0; w < width; w++) {
- float w_start = static_cast<float>(w * spatial_scale);
- float w_end = static_cast<float>((w + 1) * spatial_scale);
- set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
- }
- }
- }
-
- for (int64_t t = 0; t < keyframe_latent_frames; t++) {
- float t_start = static_cast<float>(keyframe_frame_idx + t * temporal_scale);
- float t_end = static_cast<float>(keyframe_frame_idx + (t + 1) * temporal_scale);
- if (keyframe_pixel_frames == 1) {
- t_end = t_start + 1.f;
- }
- t_start /= static_cast<float>(fps);
- t_end /= static_cast<float>(fps);
- for (int64_t h = 0; h < height; h++) {
- float h_start = static_cast<float>(h * spatial_scale);
- float h_end = static_cast<float>((h + 1) * spatial_scale);
- for (int64_t w = 0; w < width; w++) {
- float w_start = static_cast<float>(w * spatial_scale);
- float w_end = static_cast<float>((w + 1) * spatial_scale);
- set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
- }
- }
- }
-
- return positions;
- }
-
- static sd::Tensor<float> pack_ltxav_audio_and_video_latents(const sd::Tensor<float>& video_latent,
- const sd::Tensor<float>& audio_latent) {
- if (audio_latent.empty()) {
- return video_latent;
- }
-
- GGML_ASSERT(video_latent.dim() == 4 || video_latent.dim() == 5);
- GGML_ASSERT(audio_latent.dim() == 3 || audio_latent.dim() == 4);
- if (video_latent.dim() == 5) {
- GGML_ASSERT(video_latent.shape()[4] == 1);
- }
- if (audio_latent.dim() == 4) {
- GGML_ASSERT(audio_latent.shape()[3] == 1);
- }
-
- int64_t width = video_latent.shape()[0];
- int64_t height = video_latent.shape()[1];
- int64_t frames = video_latent.shape()[2];
- int64_t video_ch = video_latent.shape()[3];
- int64_t spatial_size = width * height * frames;
- int64_t audio_values = audio_latent.numel();
- int64_t extra_ch = (audio_values + spatial_size - 1) / spatial_size;
-
- std::vector<int64_t> packed_shape = video_latent.shape();
- packed_shape[3] = video_ch + extra_ch;
- sd::Tensor<float> packed = sd::zeros<float>(packed_shape);
-
- std::copy_n(video_latent.data(), video_latent.numel(), packed.data());
- std::copy_n(audio_latent.data(), audio_latent.numel(), packed.data() + video_latent.numel());
- return packed;
- }
-
- static sd::Tensor<float> pack_ltxav_audio_and_video_denoise_mask(const sd::Tensor<float>& video_mask,
- const sd::Tensor<float>& video_latent,
- const sd::Tensor<float>& audio_latent) {
- if (video_mask.empty() || audio_latent.empty()) {
- return video_mask;
- }
-
- GGML_ASSERT(video_latent.dim() == 4 || video_latent.dim() == 5);
- GGML_ASSERT(audio_latent.dim() == 3 || audio_latent.dim() == 4);
- if (video_latent.dim() == 5) {
- GGML_ASSERT(video_latent.shape()[4] == 1);
- }
- if (audio_latent.dim() == 4) {
- GGML_ASSERT(audio_latent.shape()[3] == 1);
- }
-
- int64_t width = video_latent.shape()[0];
- int64_t height = video_latent.shape()[1];
- int64_t frames = video_latent.shape()[2];
- int64_t video_ch = video_latent.shape()[3];
- int64_t spatial_size = width * height * frames;
- int64_t audio_values = audio_latent.numel();
- int64_t extra_ch = (audio_values + spatial_size - 1) / spatial_size;
-
- GGML_ASSERT(video_mask.dim() == video_latent.dim());
- GGML_ASSERT(video_mask.shape()[0] == width);
- GGML_ASSERT(video_mask.shape()[1] == height);
- GGML_ASSERT(video_mask.shape()[2] == frames);
- if (video_mask.dim() == 5) {
- GGML_ASSERT(video_mask.shape()[4] == video_latent.shape()[4]);
- }
-
- int64_t mask_ch = video_mask.shape()[3];
- if (mask_ch == video_ch + extra_ch) {
- return video_mask;
- }
- GGML_ASSERT(mask_ch == 1 || mask_ch == video_ch);
-
- sd::Tensor<float> video_mask_full = video_mask;
- if (mask_ch == 1 && video_ch != 1) {
- video_mask_full = video_mask * sd::Tensor<float>::ones(video_latent.shape());
- }
-
- std::vector<int64_t> audio_mask_shape = video_latent.shape();
- audio_mask_shape[3] = extra_ch;
- auto audio_mask = sd::Tensor<float>::ones(audio_mask_shape);
- return sd::ops::concat(video_mask_full, audio_mask, 3);
- }
-
- static sd::Tensor<float> make_ltxav_video_denoise_mask(const sd::Tensor<float>& video_latent, float value = 1.f) {
- if (video_latent.empty()) {
- return {};
- }
- return sd::full<float>({video_latent.shape()[0],
- video_latent.shape()[1],
- video_latent.shape()[2],
- 1,
- 1},
- value);
- }
-
- static sd::Tensor<float> encode_ltxav_condition_image(StableDiffusionGGML* sd,
- const sd::Tensor<float>& image,
- const char* name) {
- if (sd == nullptr || image.empty()) {
- return {};
- }
- auto condition_image = image.reshape({image.shape()[0],
- image.shape()[1],
- 1,
- image.shape()[2],
- image.shape()[3]});
- auto condition_latent = sd->encode_first_stage(condition_image);
- if (condition_latent.empty()) {
- LOG_ERROR("failed to encode LTXAV %s image", name);
- }
- return condition_latent;
- }
-
- static bool apply_ltxav_condition_by_latent_index(sd::Tensor<float>* video_latent,
- sd::Tensor<float>* video_mask,
- const sd::Tensor<float>& condition_latent,
- int64_t latent_idx,
- const char* name,
- float conditioned_mask) {
- if (video_latent == nullptr || video_mask == nullptr || video_latent->empty() || video_mask->empty()) {
- return false;
- }
- if (condition_latent.empty() ||
- condition_latent.shape()[0] != video_latent->shape()[0] ||
- condition_latent.shape()[1] != video_latent->shape()[1] ||
- condition_latent.shape()[3] != video_latent->shape()[3]) {
- LOG_ERROR("invalid LTXAV %s condition latent shape", name);
- return false;
- }
- int64_t latent_frames = video_latent->shape()[2];
- int64_t condition_frames = condition_latent.shape()[2];
- if (latent_idx < 0 || condition_frames <= 0 || latent_idx + condition_frames > latent_frames) {
- LOG_ERROR("invalid LTXAV %s image latent range: start=%" PRId64 ", length=%" PRId64 ", latent_frames=%" PRId64,
- name,
- latent_idx,
- condition_frames,
- latent_frames);
- return false;
- }
-
- sd::ops::slice_assign(video_latent, 2, latent_idx, latent_idx + condition_frames, condition_latent);
- sd::ops::fill_slice(video_mask, 2, latent_idx, latent_idx + condition_frames, conditioned_mask);
- return true;
- }
-
- static bool apply_ltxav_condition_image_by_latent_index(StableDiffusionGGML* sd,
- const sd::Tensor<float>& image,
- sd::Tensor<float>* video_latent,
- sd::Tensor<float>* video_mask,
- int64_t latent_idx,
- const char* name,
- float strength) {
- auto condition_latent = encode_ltxav_condition_image(sd, image, name);
- return !condition_latent.empty() &&
- apply_ltxav_condition_by_latent_index(video_latent,
- video_mask,
- condition_latent,
- latent_idx,
- name,
- 1.0f - std::clamp(strength, 0.f, 1.f));
- }
-
- static sd::Tensor<float> unpack_ltxav_audio_latent(const sd::Tensor<float>& packed_latent,
- int audio_length,
- int video_channels) {
- if (packed_latent.empty() || audio_length <= 0) {
- return {};
- }
-
- GGML_ASSERT(packed_latent.dim() == 4 || packed_latent.dim() == 5);
- int64_t width = packed_latent.shape()[0];
- int64_t height = packed_latent.shape()[1];
- int64_t frames = packed_latent.shape()[2];
- int64_t total_channels = packed_latent.shape()[3];
- int64_t spatial_size = width * height * frames;
- if (total_channels <= video_channels) {
- return {};
- }
-
- constexpr int kLtxavAudioFrequencyBins = 16;
- constexpr int kLtxavAudioChannels = 8;
- int64_t required_values = static_cast<int64_t>(audio_length) * kLtxavAudioFrequencyBins * kLtxavAudioChannels;
- int64_t packed_values = (total_channels - video_channels) * spatial_size;
- if (packed_values < required_values) {
- return {};
- }
-
- sd::Tensor<float> audio_latent({kLtxavAudioFrequencyBins, audio_length, kLtxavAudioChannels, 1});
- const float* audio_src = packed_latent.data() + static_cast<size_t>(video_channels) * static_cast<size_t>(spatial_size);
- std::copy_n(audio_src, static_cast<size_t>(required_values), audio_latent.data());
- return audio_latent;
- }
-
- static sd::Tensor<float> make_ltxav_empty_audio_latent(int audio_length) {
- if (audio_length <= 0) {
- return {};
- }
- constexpr int kLtxavAudioFrequencyBins = 16;
- constexpr int kLtxavAudioChannels = 8;
- return sd::zeros<float>({kLtxavAudioFrequencyBins, audio_length, kLtxavAudioChannels, 1});
- }
-
- static sd::Tensor<float> resize_ltxav_audio_latent(const sd::Tensor<float>& audio_latent,
- int target_audio_length) {
- auto resized = make_ltxav_empty_audio_latent(target_audio_length);
- if (resized.empty() || audio_latent.empty()) {
- return resized;
- }
- GGML_ASSERT(audio_latent.dim() == 3 || audio_latent.dim() == 4);
- int copy_length = std::min(static_cast<int>(audio_latent.shape()[1]), target_audio_length);
- if (copy_length > 0) {
- auto copied = sd::ops::slice(audio_latent, 1, 0, copy_length);
- sd::ops::slice_assign(&resized, 1, 0, copy_length, copied);
- }
- return resized;
- }
-
- static int get_ltxav_num_audio_latents(int frames, int fps) {
- GGML_ASSERT(frames > 0);
- GGML_ASSERT(fps > 0);
- constexpr float kSampleRate = 16000.0f;
- constexpr float kMelHopLength = 160.0f;
- constexpr float kAudioLatentDownsample = 4.0f;
- constexpr float kLatentsPerSecond = kSampleRate / kMelHopLength / kAudioLatentDownsample;
- return static_cast<int>(std::ceil((static_cast<float>(frames) / static_cast<float>(fps)) * kLatentsPerSecond));
- }
-
- static int get_minimax_h3_num_audio_latents(int frames, int fps) {
- GGML_ASSERT(frames > 0 && fps > 0);
- return std::max(1,
- static_cast<int>(std::lround(
- static_cast<double>(frames) * 40.0 / fps)));
- }
-
- static sd::Tensor<float> make_minimax_h3_empty_audio_latent(int audio_length) {
- if (audio_length <= 0) {
- return {};
- }
- return sd::zeros<float>({audio_length, 2, 32, 1});
- }
-
- static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd::Audio& audio,
- int target_sample_rate = 32000) {
- if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) {
- return {};
- }
- uint64_t output_samples = static_cast<uint64_t>(std::llround(
- static_cast<long double>(audio.sample_count) * target_sample_rate / audio.sample_rate));
- output_samples = std::max<uint64_t>(1, output_samples);
- uint64_t padded_samples = (output_samples + 799) / 800 * 800;
- // Keep stereo streams planar for the mono-per-stream audio encoder:
- // [samples, 1, stereo, batch]. This avoids flattening interleaved L/R
- // storage into alternating samples when the encoder folds streams into
- // its batch dimension.
- sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 1, 2, 1});
-
- for (uint64_t i = 0; i < output_samples; ++i) {
- long double source_pos = static_cast<long double>(i) * audio.sample_rate / target_sample_rate;
- uint64_t source0 = std::min<uint64_t>(static_cast<uint64_t>(source_pos), audio.sample_count - 1);
- uint64_t source1 = std::min<uint64_t>(source0 + 1, audio.sample_count - 1);
- float fraction = static_cast<float>(source_pos - source0);
- for (uint32_t channel = 0; channel < 2; ++channel) {
- uint32_t source_channel = audio.channels == 1 ? 0 : std::min<uint32_t>(channel, audio.channels - 1);
- float a = audio.data[source0 * audio.channels + source_channel];
- float b = audio.data[source1 * audio.channels + source_channel];
- waveform.index(static_cast<int64_t>(i), 0, channel, 0) =
- std::clamp(a + (b - a) * fraction, -1.f, 1.f);
- }
- }
- return waveform;
- }
-
- static sd::Tensor<float> unpack_minimax_h3_audio_latent(const sd::Tensor<float>& packed_latent,
- int audio_length,
- int video_channels) {
- if (packed_latent.empty() || audio_length <= 0) {
- return {};
- }
- GGML_ASSERT(packed_latent.dim() == 4 || packed_latent.dim() == 5);
- int64_t spatial_size = packed_latent.shape()[0] * packed_latent.shape()[1] * packed_latent.shape()[2];
- int64_t required = static_cast<int64_t>(audio_length) * 2 * 32;
- int64_t available = (packed_latent.shape()[3] - video_channels) * spatial_size;
- if (available < required) {
- return {};
- }
- sd::Tensor<float> audio({audio_length, 2, 32, 1});
- const float* source = packed_latent.data() +
- static_cast<size_t>(video_channels) * static_cast<size_t>(spatial_size);
- std::copy_n(source, static_cast<size_t>(required), audio.data());
- return audio;
- }
-
- // Build the first 16 fps audio window, zero-padding past the track end.
- static sd::Tensor<float> build_s2v_audio_window(const sd::Tensor<float>& stacked, int64_t batch_frames) {
- const int64_t embed_dim = stacked.shape()[0];
- const int64_t in_frames = stacked.shape()[1];
- const int64_t num_layers = stacked.shape()[2];
- if (embed_dim <= 0 || in_frames <= 0 || num_layers <= 0 || batch_frames <= 0) {
- return {};
- }
- std::vector<float> layer_first(static_cast<size_t>(num_layers) * in_frames * embed_dim);
- for (int64_t l = 0; l < num_layers; ++l) {
- for (int64_t f = 0; f < in_frames; ++f) {
- const float* src = stacked.data() + l * embed_dim * in_frames + f * embed_dim;
- std::copy_n(src,
- static_cast<size_t>(embed_dim),
- layer_first.data() + (static_cast<size_t>(l) * in_frames + f) * embed_dim);
- }
- }
- sd::wan_audio::BucketPlan plan;
- std::vector<float> buckets = sd::wan_audio::build_audio_buckets(layer_first.data(),
- static_cast<int>(num_layers),
- static_cast<int>(in_frames),
- static_cast<int>(embed_dim),
- static_cast<int>(batch_frames),
- &plan);
- if (buckets.empty() || plan.bucket_frames < batch_frames) {
- return {};
- }
- // Reorder frame-major buckets into sd::Tensor's [dim, frame, layer] layout.
- sd::Tensor<float> window({embed_dim, batch_frames, num_layers});
- for (int64_t f = 0; f < batch_frames; ++f) {
- for (int64_t l = 0; l < num_layers; ++l) {
- const float* src = buckets.data() + (static_cast<size_t>(f) * num_layers + l) * embed_dim;
- float* dst = window.data() + l * embed_dim * batch_frames + f * embed_dim;
- std::copy_n(src, static_cast<size_t>(embed_dim), dst);
- }
- }
- return window;
- }
-
- // ComfyUI-compatible reference sizing: per-axis rounding to 32 px keeps the reference
- // divisible by VAE 16 x patch 2 without padding in the H3 graph. Both modes round rather
- // than floor: flooring would under-size in-limit references and diverge from the
- // upstream node, which rounds in both branches.
- static std::pair<int, int> minimax_h3_reference_size(int source_w,
- int source_h,
- int target_w,
- int target_h,
- RefImageSizeMode mode) {
- constexpr double kCanvasMultiple = 32.0;
- constexpr double kRefImageShortEdge = 2048.0;
- double scale = 1.0;
- if (mode == RefImageSizeMode::MAX) {
- scale = std::min(1.0, kRefImageShortEdge / std::min(source_w, source_h));
- } else {
- double source_area = static_cast<double>(source_w) * source_h;
- double target_area = static_cast<double>(target_w) * target_h;
- scale = std::min(1.0, std::sqrt(target_area / source_area));
- }
- auto align32 = [kCanvasMultiple](double value) {
- return std::max(32, static_cast<int>(std::round(value / kCanvasMultiple)) * 32);
- };
- return {align32(source_w * scale), align32(source_h * scale)};
- }
-
- static std::optional<ImageGenerationLatents> prepare_video_generation_latents(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- GenerationRequest* request,
- const RefImageParams& ref_image_params) {
- ImageGenerationLatents latents;
- int64_t prepare_start_ms = ggml_time_ms();
-
- sd::Tensor<float> start_image;
- sd::Tensor<float> end_image;
-
- if (sd_vid_gen_params->init_image.data) {
- start_image = ensure_image_tensor_channels(
- sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height),
- sd->get_image_channels());
- }
-
- if (sd_vid_gen_params->end_image.data) {
- end_image = ensure_image_tensor_channels(
- sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height),
- sd->get_image_channels());
- }
-
- if (sd_version_is_minimax_h3(sd->version)) {
- if (sd_vid_gen_params->ref_images_count < 0 || sd_vid_gen_params->ref_videos_count < 0 ||
- sd_vid_gen_params->ref_audios_count < 0 ||
- (sd_vid_gen_params->ref_images_count > 0 && sd_vid_gen_params->ref_images == nullptr) ||
- (sd_vid_gen_params->ref_videos_count > 0 && sd_vid_gen_params->ref_videos == nullptr) ||
- (sd_vid_gen_params->ref_audios_count > 0 && sd_vid_gen_params->ref_audios == nullptr)) {
- LOG_ERROR("invalid MiniMax-H3 Ref2VA input arrays");
- return std::nullopt;
- }
-
- latents.audio_length = get_minimax_h3_num_audio_latents(request->frames,
- request->fps);
- latents.audio_latent = make_minimax_h3_empty_audio_latent(latents.audio_length);
-
- bool has_references = sd_vid_gen_params->ref_images_count > 0 ||
- sd_vid_gen_params->ref_videos_count > 0 ||
- sd_vid_gen_params->ref_audios_count > 0;
- if (has_references && (!start_image.empty() || !end_image.empty())) {
- LOG_ERROR("MiniMax-H3 keyframes and Ref2VA references cannot be used together");
- return std::nullopt;
- }
-
- if (sd_vid_gen_params->control_frames_size > 0) {
- LOG_ERROR("MiniMax-H3 control_frames are not implemented");
- return std::nullopt;
- }
-
- auto add_visual_noise = [&](sd::Tensor<float> latent) {
- auto condition_rng = std::make_shared<PhiloxRNG>();
- condition_rng->manual_seed(static_cast<uint64_t>(request->seed));
- return latent * MiniMaxH3::VISUAL_COND_TIMESTEP +
- sd::Tensor<float>::randn_like(latent, condition_rng) *
- (1.f - MiniMaxH3::VISUAL_COND_TIMESTEP);
- };
-
- auto add_keyframe = [&](const sd::Tensor<float>& image,
- int32_t frame_index,
- const char* name) -> bool {
- if (image.empty()) {
- return true;
- }
- auto video_image = image.reshape({image.shape()[0],
- image.shape()[1],
- 1,
- image.shape()[2],
- image.shape()[3]});
- auto latent = sd->encode_first_stage(video_image);
- if (latent.empty()) {
- LOG_ERROR("failed to encode MiniMax-H3 %s keyframe", name);
- return false;
- }
- latents.ref_images.push_back(image);
- latents.ref_latents.push_back(add_visual_noise(std::move(latent)));
- latents.keyframe_indices.push_back(frame_index);
- return true;
- };
-
- auto resize_reference = [&](const sd::Tensor<float>& image,
- int width,
- int height) {
- return sd::ops::interpolate(
- image,
- std::vector<int64_t>{width, height, image.shape()[2], image.shape()[3]});
- };
-
- auto encode_reference_audio = [&](const sd::Audio& audio,
- int32_t* audio_index) -> bool {
- if (sd->audio_vae_model == nullptr) {
- LOG_ERROR("MiniMax-H3 Ref2VA audio requires --audio-vae with encoder weights");
- return false;
- }
- auto waveform = prepare_minimax_h3_reference_waveform(
- audio,
- sd->audio_vae_model->input_sample_rate());
- if (waveform.empty()) {
- LOG_ERROR("invalid MiniMax-H3 reference audio");
- return false;
- }
- auto encoded = sd->audio_vae_model->encode(sd->n_threads, waveform);
- if (encoded.empty()) {
- LOG_ERROR("failed to encode MiniMax-H3 reference audio");
- return false;
- }
- *audio_index = static_cast<int32_t>(latents.reference_audio_latents.size());
- latents.reference_audio_latents.push_back(std::move(encoded));
- return true;
- };
-
- if (has_references) {
- LOG_INFO("MiniMax-H3 Ref2VA: %d image(s), %d video(s), %d audio clip(s)",
- sd_vid_gen_params->ref_images_count,
- sd_vid_gen_params->ref_videos_count,
- sd_vid_gen_params->ref_audios_count);
-
- int64_t ref_image_tokens = 0;
- for (int i = 0; i < sd_vid_gen_params->ref_images_count; ++i) {
- auto image = ensure_image_tensor_channels(
- sd_image_to_tensor(sd_vid_gen_params->ref_images[i]),
- 3);
- if (image.empty()) {
- LOG_ERROR("failed to load MiniMax-H3 reference image %d", i + 1);
- return std::nullopt;
- }
- int source_w = static_cast<int>(image.shape()[0]);
- int source_h = static_cast<int>(image.shape()[1]);
- auto [width, height] =
- minimax_h3_reference_size(source_w, source_h, request->width, request->height, ref_image_params.ref_image_size);
- LOG_INFO("MiniMax-H3 reference image %d: %dx%d -> %dx%d",
- i + 1,
- source_w,
- source_h,
- width,
- height);
- image = resize_reference(image, width, height);
- auto latent = sd->encode_first_stage(image);
- if (latent.empty()) {
- LOG_ERROR("failed to encode MiniMax-H3 reference image %d", i + 1);
- return std::nullopt;
- }
- int32_t video_index = static_cast<int32_t>(latents.ref_latents.size());
- latents.ref_latents.push_back(add_visual_noise(std::move(latent)));
- latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE,
- video_index,
- -1});
- // The DiT patchifies latents 2 x 2, so the token count is a quarter of the
- // latent grid.
- ref_image_tokens += static_cast<int64_t>(latents.ref_latents.back().shape()[0] / 2) *
- (latents.ref_latents.back().shape()[1] / 2);
- MiniMaxH3PresentationItem item;
- item.kind = MiniMaxH3PresentationKind::IMAGE;
- item.frames.push_back(std::move(image));
- latents.minimax_presentation_refs.push_back(std::move(item));
- }
-
- for (int video_idx = 0; video_idx < sd_vid_gen_params->ref_videos_count; ++video_idx) {
- const auto& reference = sd_vid_gen_params->ref_videos[video_idx];
- if (reference.frames == nullptr || reference.frame_count < 1) {
- LOG_ERROR("invalid MiniMax-H3 reference video %d", video_idx + 1);
- return std::nullopt;
- }
- int source_fps = reference.fps > 0 ? reference.fps : 24;
- int normalized_frames = static_cast<int>(std::lround(
- static_cast<double>(reference.frame_count) * 24.0 / source_fps));
- normalized_frames = std::min(normalized_frames, request->frames);
- if (normalized_frames < 5) {
- LOG_ERROR("MiniMax-H3 reference video %d needs at least 5 frames at 24 fps",
- video_idx + 1);
- return std::nullopt;
- }
- while (normalized_frames % 17 != 5) {
- --normalized_frames;
- }
-
- auto first = ensure_image_tensor_channels(sd_image_to_tensor(reference.frames[0]), 3);
- if (first.empty()) {
- LOG_ERROR("invalid first frame in MiniMax-H3 reference video %d", video_idx + 1);
- return std::nullopt;
- }
- int source_w = static_cast<int>(first.shape()[0]);
- int source_h = static_cast<int>(first.shape()[1]);
- double ratio = static_cast<double>(source_w) / source_h;
- double nominal_w = ratio >= 1.0 ? 768.0 * ratio : 768.0;
- double nominal_h = ratio >= 1.0 ? 768.0 : 768.0 / ratio;
- if (nominal_w * nominal_h > 768.0 * 1344.0) {
- double scale = std::sqrt((768.0 * 1344.0) / (nominal_w * nominal_h));
- nominal_w *= scale;
- nominal_h *= scale;
- }
- int width = std::max(32, static_cast<int>(std::round(nominal_w / 32.0)) * 32);
- int height = std::max(32, static_cast<int>(std::round(nominal_h / 32.0)) * 32);
- if (source_w * source_h < width * height) {
- width = std::max(32, static_cast<int>(std::round(source_w / 32.0)) * 32);
- height = std::max(32, static_cast<int>(std::round(source_h / 32.0)) * 32);
- }
-
- sd::Tensor<float> video({width, height, normalized_frames, 3, 1});
- for (int frame = 0; frame < normalized_frames; ++frame) {
- int source_index = std::min(reference.frame_count - 1,
- static_cast<int>(std::floor(frame * source_fps / 24.0)));
- auto source = ensure_image_tensor_channels(
- sd_image_to_tensor(reference.frames[source_index]),
- 3);
- if (source.empty()) {
- LOG_ERROR("invalid frame %d in MiniMax-H3 reference video %d",
- source_index + 1,
- video_idx + 1);
- return std::nullopt;
- }
- source = resize_reference(source, width, height);
- sd::ops::slice_assign(&video, 2, frame, frame + 1, source.unsqueeze(2));
- }
- auto video_latent = sd->encode_first_stage(video);
- if (video_latent.empty()) {
- LOG_ERROR("failed to encode MiniMax-H3 reference video %d", video_idx + 1);
- return std::nullopt;
- }
- int32_t audio_index = -1;
- bool has_audio = reference.audio.data != nullptr && reference.audio.sample_count > 0;
- if (has_audio) {
- if (!encode_reference_audio(reference.audio, &audio_index)) {
- return std::nullopt;
- }
- MiniMaxH3PresentationItem audio_item;
- audio_item.kind = MiniMaxH3PresentationKind::AUDIO;
- latents.minimax_presentation_refs.push_back(std::move(audio_item));
- }
-
- MiniMaxH3PresentationItem video_item;
- video_item.kind = MiniMaxH3PresentationKind::VIDEO;
- for (int frame = 0; frame < normalized_frames; frame += 12) {
- auto sampled = sd::ops::slice(video, 2, frame, frame + 1)
- .reshape({width, height, 3, 1});
- video_item.frames.push_back(std::move(sampled));
- video_item.timestamps.push_back(frame / 24.f);
- }
- latents.minimax_presentation_refs.push_back(std::move(video_item));
-
- int32_t video_index = static_cast<int32_t>(latents.ref_latents.size());
- latents.ref_latents.push_back(add_visual_noise(std::move(video_latent)));
- latents.minimax_reference_blocks.push_back({has_audio ? MiniMaxH3ReferenceKind::VIDEO_AUDIO
- : MiniMaxH3ReferenceKind::VIDEO,
- video_index,
- audio_index});
- }
-
- for (int audio_idx = 0; audio_idx < sd_vid_gen_params->ref_audios_count; ++audio_idx) {
- int32_t encoded_index = -1;
- if (!encode_reference_audio(sd_vid_gen_params->ref_audios[audio_idx], &encoded_index)) {
- return std::nullopt;
- }
- MiniMaxH3PresentationItem item;
- item.kind = MiniMaxH3PresentationKind::AUDIO;
- latents.minimax_presentation_refs.push_back(std::move(item));
- latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::AUDIO,
- -1,
- encoded_index});
- }
-
- if (ref_image_tokens > 0) {
- int64_t target_tokens = sd->get_image_seq_len(request->width, request->height) / 4 *
- sd->video_frames_to_latent_frames(request->frames);
- if (target_tokens > 0 && ref_image_tokens > target_tokens * 4) {
- LOG_WARN("MiniMax-H3 reference images add %" PRId64
- " tokens versus %" PRId64
- " for the generated video; sampling cost scales with every attended token",
- ref_image_tokens,
- target_tokens);
- }
- }
- }
-
- if (!has_references && (!start_image.empty() || !end_image.empty())) {
- LOG_INFO(!start_image.empty() && !end_image.empty() ? "MiniMax-H3 FL2VA" : !start_image.empty() ? "MiniMax-H3 I2VA"
- : "MiniMax-H3 end-frame conditioning");
- }
- if (!has_references &&
- (!add_keyframe(start_image, 0, "start") ||
- !add_keyframe(end_image, request->frames - 1, "end"))) {
- return std::nullopt;
- }
- }
-
- if (sd_version_is_ltxav(sd->version)) {
- latents.audio_length = get_ltxav_num_audio_latents(request->frames, request->fps);
- latents.audio_latent = make_ltxav_empty_audio_latent(latents.audio_length);
- }
-
- if (sd_version_is_ltxav(sd->version)) {
- if (sd_vid_gen_params->control_frames_size > 0) {
- LOG_ERROR("LTXAV control_frames are not implemented");
- return std::nullopt;
- }
-
- if (!start_image.empty() || !end_image.empty()) {
- if (!start_image.empty() && !end_image.empty()) {
- LOG_INFO("FLF2V");
- } else if (!start_image.empty()) {
- LOG_INFO("IMG2VID");
- } else {
- LOG_INFO("END2VID");
- }
-
- int64_t t1 = ggml_time_ms();
- latents.init_latent = sd->generate_init_latent(request->width, request->height, request->frames, true);
-
- float conditioning_strength = std::clamp(request->strength, 0.f, 1.f);
- float conditioned_mask = 1.0f - conditioning_strength;
- latents.denoise_mask = make_ltxav_video_denoise_mask(latents.init_latent, 1.f);
-
- auto apply_video_condition_by_keyframe_index = [&](const sd::Tensor<float>& keyframes,
- int frame_idx,
- const char* name) -> bool {
- int64_t keyframe_frames = keyframes.shape()[2];
- if (keyframe_frames <= 0 || keyframes.shape()[0] != latents.init_latent.shape()[0] ||
- keyframes.shape()[1] != latents.init_latent.shape()[1] ||
- keyframes.shape()[3] != latents.init_latent.shape()[3]) {
- LOG_ERROR("invalid LTXAV %s keyframe latent shape", name);
- return false;
- }
-
- latents.video_target_frame_count = latents.init_latent.shape()[2];
- latents.video_conditioning_frame_count = keyframe_frames;
- latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2);
-
- auto keyframe_mask = sd::full<float>({keyframes.shape()[0],
- keyframes.shape()[1],
- keyframes.shape()[2],
- 1,
- 1},
- conditioned_mask);
- latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2);
- latents.video_positions = build_ltxv_video_positions(latents.init_latent.shape()[0],
- latents.init_latent.shape()[1],
- latents.video_target_frame_count,
- keyframe_frames,
- frame_idx,
- 1,
- request->fps,
- request->vae_scale_factor,
- 8,
- true);
- return true;
- };
-
- if (!start_image.empty()) {
- if (!apply_ltxav_condition_image_by_latent_index(sd,
- start_image,
- &latents.init_latent,
- &latents.denoise_mask,
- 0,
- "init",
- conditioning_strength)) {
- return std::nullopt;
- }
- }
-
- if (!end_image.empty()) {
- auto end_image_latent = encode_ltxav_condition_image(sd, end_image, "end");
- if (end_image_latent.empty()) {
- return std::nullopt;
- }
-
- int frame_idx = request->frames - 1;
- bool ok = frame_idx == 0 ? apply_ltxav_condition_by_latent_index(&latents.init_latent,
- &latents.denoise_mask,
- end_image_latent,
- 0,
- "end",
- conditioned_mask)
- : apply_video_condition_by_keyframe_index(end_image_latent, frame_idx, "end");
- if (!ok) {
- return std::nullopt;
- }
- }
-
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
- }
- }
-
- if (sd_version_is_hunyuan_video(sd->version) &&
- (!start_image.empty() || !end_image.empty())) {
- LOG_INFO("Hunyuan Video IMG2VID");
-
- int64_t t1 = ggml_time_ms();
- auto concat_latent = sd->generate_init_latent(request->width,
- request->height,
- request->frames,
- true);
- auto encode_condition_frame = [&](const sd::Tensor<float>& image,
- int64_t latent_frame,
- const char* name) -> bool {
- auto encoded = sd->encode_first_stage(image.unsqueeze(2));
- if (encoded.empty()) {
- LOG_ERROR("failed to encode Hunyuan Video %s conditioning frame", name);
- return false;
- }
- if (encoded.dim() == 4) {
- encoded.unsqueeze_(2);
- }
- if (encoded.dim() != 5 ||
- encoded.shape()[0] != concat_latent.shape()[0] ||
- encoded.shape()[1] != concat_latent.shape()[1] ||
- encoded.shape()[3] != concat_latent.shape()[3]) {
- LOG_ERROR("invalid Hunyuan Video %s conditioning latent shape", name);
- return false;
- }
- sd::ops::slice_assign(&concat_latent,
- 2,
- latent_frame,
- latent_frame + 1,
- sd::ops::slice(encoded, 2, 0, 1));
- return true;
- };
-
- if (!start_image.empty() && !encode_condition_frame(start_image, 0, "start")) {
- return std::nullopt;
- }
- if (!end_image.empty() &&
- !encode_condition_frame(end_image, concat_latent.shape()[2] - 1, "end")) {
- return std::nullopt;
- }
-
- sd::Tensor<float> concat_mask = sd::zeros<float>({concat_latent.shape()[0],
- concat_latent.shape()[1],
- concat_latent.shape()[2],
- 1,
- 1});
- if (!start_image.empty()) {
- sd::ops::fill_slice(&concat_mask, 2, 0, 1, 1.0f);
- }
- if (!end_image.empty()) {
- sd::ops::fill_slice(&concat_mask, 2, concat_mask.shape()[2] - 1, concat_mask.shape()[2], 1.0f);
- }
- latents.concat_latent = sd::ops::concat(concat_latent, concat_mask, 3);
-
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
- }
-
- if (sd->diffusion_model->get_desc() == "Wan2.1-I2V-14B" ||
- sd->diffusion_model->get_desc() == "Wan2.2-I2V-14B" ||
- sd->diffusion_model->get_desc() == "Wan2.1-I2V-1.3B" ||
- sd->diffusion_model->get_desc() == "Wan2.1-FLF2V-14B") {
- LOG_INFO("IMG2VID");
-
- if (sd->diffusion_model->get_desc() == "Wan2.1-I2V-14B" ||
- sd->diffusion_model->get_desc() == "Wan2.1-I2V-1.3B" ||
- sd->diffusion_model->get_desc() == "Wan2.1-FLF2V-14B") {
- if (!start_image.empty()) {
- auto clip_vision_output = sd->get_clip_vision_output(start_image, false, -2);
- if (clip_vision_output.empty()) {
- LOG_ERROR("failed to compute clip vision output for init image");
- return std::nullopt;
- }
- latents.clip_vision_output = std::move(clip_vision_output);
- } else {
- latents.clip_vision_output = sd->get_clip_vision_output(start_image, false, -2, true);
- }
-
- if (sd->diffusion_model->get_desc() == "Wan2.1-FLF2V-14B") {
- sd::Tensor<float> end_image_clip_vision_output;
- if (!end_image.empty()) {
- end_image_clip_vision_output = sd->get_clip_vision_output(end_image, false, -2);
- if (end_image_clip_vision_output.empty()) {
- LOG_ERROR("failed to compute clip vision output for end image");
- return std::nullopt;
- }
- } else {
- end_image_clip_vision_output = sd->get_clip_vision_output(end_image, false, -2, true);
- }
- latents.clip_vision_output = sd::ops::concat(latents.clip_vision_output, end_image_clip_vision_output, 1);
- }
-
- int64_t t1 = ggml_time_ms();
- LOG_INFO("get_clip_vision_output completed, taking %" PRId64 " ms", t1 - prepare_start_ms);
- }
-
- int64_t t1 = ggml_time_ms();
- sd::Tensor<float> image = sd::full<float>({request->width, request->height, request->frames, 3, 1}, 0.5f);
- if (!start_image.empty()) {
- sd::ops::slice_assign(&image, 2, 0, 1, start_image.unsqueeze(2));
- }
- if (!end_image.empty()) {
- sd::ops::slice_assign(&image, 2, request->frames - 1, request->frames, end_image.unsqueeze(2));
- }
-
- auto concat_latent = sd->encode_first_stage(image); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor]
- if (concat_latent.empty()) {
- LOG_ERROR("failed to encode video conditioning frames");
- return std::nullopt;
- }
- latents.concat_latent = std::move(concat_latent);
-
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
-
- sd::Tensor<float> concat_mask = sd::zeros<float>({latents.concat_latent.shape()[0],
- latents.concat_latent.shape()[1],
- latents.concat_latent.shape()[2],
- 4,
- 1}); // [b, 4, t, h/vae_scale_factor, w/vae_scale_factor]
- if (!start_image.empty()) {
- sd::ops::fill_slice(&concat_mask, 2, 0, 1, 1.0f);
- }
- if (!end_image.empty()) {
- auto last_channel = sd::ops::slice(concat_mask, 3, 3, 4);
- sd::ops::fill_slice(&last_channel, 2, last_channel.shape()[2] - 1, last_channel.shape()[2], 1.0f);
- sd::ops::slice_assign(&concat_mask, 3, 3, 4, last_channel);
- }
- latents.concat_latent = sd::ops::concat(concat_mask, latents.concat_latent, 3); // [b, 4+c, t, h/vae_scale_factor, w/vae_scale_factor]
- } else if (sd->diffusion_model->get_desc() == "Wan2.2-TI2V-5B" && !start_image.empty()) {
- LOG_INFO("IMG2VID");
-
- int64_t t1 = ggml_time_ms();
- auto init_img = start_image.reshape({start_image.shape()[0], start_image.shape()[1], 1, start_image.shape()[2], 1});
- auto init_image_latent = sd->encode_first_stage(init_img); // [b, c, 1, h/vae_scale_factor, w/vae_scale_factor]
- if (init_image_latent.empty()) {
- LOG_ERROR("failed to encode init video frame");
- return std::nullopt;
- }
-
- latents.init_latent = sd->generate_init_latent(request->width, request->height, request->frames, true); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor]
- sd::ops::slice_assign(&latents.init_latent, 2, 0, init_image_latent.shape()[2], init_image_latent);
-
- latents.denoise_mask = sd::full<float>({latents.init_latent.shape()[0], latents.init_latent.shape()[1], latents.init_latent.shape()[2], 1, 1}, 1.f);
- sd::ops::fill_slice(&latents.denoise_mask, 2, 0, init_image_latent.shape()[2], 0.0f);
-
- if (!end_image.empty()) {
- auto end_img = end_image.reshape({end_image.shape()[0], end_image.shape()[1], 1, end_image.shape()[2], 1});
- auto end_image_latent = sd->encode_first_stage(end_img); // [b, c, 1, h/vae_scale_factor, w/vae_scale_factor]
- if (end_image_latent.empty()) {
- LOG_ERROR("failed to encode end video frame");
- return std::nullopt;
- }
- sd::ops::slice_assign(&latents.init_latent, 2, latents.init_latent.shape()[2] - 1, latents.init_latent.shape()[2], end_image_latent);
- sd::ops::fill_slice(&latents.denoise_mask, 2, latents.init_latent.shape()[2] - 1, latents.init_latent.shape()[2], 0.0f);
- }
-
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
- } else if (sd_version_is_lingbot_video(sd->version) && !start_image.empty()) {
- LOG_INFO("LingBot Video IMG2VID");
-
- int64_t t1 = ggml_time_ms();
- auto init_img = start_image.reshape({start_image.shape()[0], start_image.shape()[1], 1, start_image.shape()[2], 1});
- auto init_image_latent = sd->encode_first_stage(init_img);
- if (init_image_latent.empty()) {
- LOG_ERROR("failed to encode init video frame");
- return std::nullopt;
- }
-
- latents.init_latent = sd->generate_init_latent(request->width, request->height, request->frames, true);
- sd::ops::slice_assign(&latents.init_latent, 2, 0, init_image_latent.shape()[2], init_image_latent);
-
- latents.denoise_mask = sd::full<float>({latents.init_latent.shape()[0], latents.init_latent.shape()[1], latents.init_latent.shape()[2], 1, 1}, 1.f);
- sd::ops::fill_slice(&latents.denoise_mask, 2, 0, init_image_latent.shape()[2], 0.0f);
-
- latents.ref_images.push_back(start_image);
-
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
- } else if (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
- sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B") {
- LOG_INFO("VACE");
- int64_t t1 = ggml_time_ms();
- sd::Tensor<float> ref_image_latent;
- if (!start_image.empty()) {
- auto ref_img = start_image.reshape({start_image.shape()[0], start_image.shape()[1], 1, start_image.shape()[2], 1});
- auto encoded_ref = sd->encode_first_stage(ref_img); // [b, c, 1, h/vae_scale_factor, w/vae_scale_factor]
- if (encoded_ref.empty()) {
- LOG_ERROR("failed to encode VACE reference image");
- return std::nullopt;
- }
- ref_image_latent = sd::ops::concat(encoded_ref, sd::zeros<float>(encoded_ref.shape()), 3); // [b, 2*c, 1, h/vae_scale_factor, w/vae_scale_factor]
- }
-
- sd::Tensor<float> control_video = sd::full<float>({request->width, request->height, request->frames, 3, 1}, 0.5f);
- int64_t control_frame_count = std::min<int64_t>(request->frames, sd_vid_gen_params->control_frames_size);
- for (int64_t i = 0; i < control_frame_count; ++i) {
- auto control_frame = sd_image_to_tensor(sd_vid_gen_params->control_frames[i], request->width, request->height);
- sd::ops::slice_assign(&control_video, 2, i, i + 1, control_frame.unsqueeze(2));
- }
-
- sd::Tensor<float> mask = sd::full<float>({request->width, request->height, request->frames, 1, 1}, 1.0f);
-
- control_video = control_video - 0.5f;
- sd::Tensor<float> inactive = control_video * (1.0f - mask) + 0.5f;
- sd::Tensor<float> reactive = control_video * mask + 0.5f;
-
- inactive = sd->encode_first_stage(inactive); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor]
- if (inactive.empty()) {
- LOG_ERROR("failed to encode VACE inactive context");
- return std::nullopt;
- }
-
- reactive = sd->encode_first_stage(reactive); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor]
- if (reactive.empty()) {
- LOG_ERROR("failed to encode VACE reactive context");
- return std::nullopt;
- }
-
- int64_t length = inactive.shape()[2];
- if (!ref_image_latent.empty()) {
- length += 1;
- request->frames = static_cast<int>((length - 1) * 4 + 1);
- latents.ref_image_num = 1;
- }
- auto vace_context = sd::ops::concat(inactive, reactive, 3); // [b, 2*c, t, h/vae_scale_factor, w/vae_scale_factor]
-
- mask = sd::full<float>({request->width, request->height, inactive.shape()[2], 1, 1}, 1.0f);
- auto mask_context = mask.reshape({request->vae_scale_factor,
- inactive.shape()[0],
- request->vae_scale_factor,
- inactive.shape()[1],
- inactive.shape()[2]}); // [t, h/vae_scale_factor, vae_scale_factor, w/vae_scale_factor, vae_scale_factor]
- mask_context = mask_context.permute({1, 3, 4, 0, 2}) // [vae_scale_factor, vae_scale_factor, t, h/vae_scale_factor, w/vae_scale_factor]
- .reshape({inactive.shape()[0],
- inactive.shape()[1],
- inactive.shape()[2],
- request->vae_scale_factor * request->vae_scale_factor}); // [vae_scale_factor*vae_scale_factor, t, h/vae_scale_factor, w/vae_scale_factor]
-
- if (!ref_image_latent.empty()) {
- vace_context = sd::ops::concat(ref_image_latent, vace_context, 2); // [b, 2*c, t+1, h/vae_scale_factor, w/vae_scale_factor]
- auto mask_pad = sd::zeros<float>({mask_context.shape()[0],
- mask_context.shape()[1],
- 1,
- mask_context.shape()[3]}); // [vae_scale_factor*vae_scale_factor, 1, h/vae_scale_factor, w/vae_scale_factor]
- mask_context = sd::ops::concat(mask_pad, mask_context, 2); // [vae_scale_factor*vae_scale_factor, t + 1, h/vae_scale_factor, w/vae_scale_factor]
- }
-
- mask_context.unsqueeze_(mask_context.dim()); // [b, vae_scale_factor*vae_scale_factor, t + 1 or t, h/vae_scale_factor, w/vae_scale_factor]
-
- latents.vace_context = sd::ops::concat(vace_context, mask_context, 3); // [b, 2*c + vae_scale_factor*vae_scale_factor, t + 1 or t, h/vae_scale_factor, w/vae_scale_factor]
- int64_t t2 = ggml_time_ms();
- LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
- } else if (sd->diffusion_model->get_desc() == "Wan2.2-S2V-14B") {
- LOG_INFO("S2V");
- if (!end_image.empty()) {
- LOG_WARN("Wan2.2 S2V ignores end_image");
- }
- if (sd_vid_gen_params->ref_audios_count > 1) {
- LOG_ERROR("Wan2.2 S2V supports a single driving audio track");
- return std::nullopt;
- }
- int64_t t1 = ggml_time_ms();
- if (!start_image.empty()) {
- auto ref_img = start_image.reshape({start_image.shape()[0],
- start_image.shape()[1],
- 1,
- start_image.shape()[2],
- 1});
- auto encoded_ref = sd->encode_first_stage(ref_img);
- if (encoded_ref.empty()) {
- LOG_ERROR("failed to encode S2V reference image");
- return std::nullopt;
- }
- // Wan consumes reference latents in 4D.
- latents.ref_latents.push_back(encoded_ref.reshape({encoded_ref.shape()[0],
- encoded_ref.shape()[1],
- encoded_ref.shape()[2],
- encoded_ref.shape()[3]}));
- }
- if (sd_vid_gen_params->ref_audios_count == 1) {
- if (sd->audio_encoder == nullptr) {
- LOG_ERROR("S2V audio conditioning requires --audio-encoder (wav2vec2)");
- return std::nullopt;
- }
- auto stacked = sd->get_audio_embedding(sd_vid_gen_params->ref_audios[0]);
- if (stacked.empty()) {
- LOG_ERROR("failed to compute wav2vec2 embedding for driving audio");
- return std::nullopt;
- }
- int64_t latent_t = sd->video_frames_to_latent_frames(request->frames);
- int64_t batch_frames = latent_t * 4;
- latents.s2v_audio_embed = build_s2v_audio_window(stacked, batch_frames);
- if (latents.s2v_audio_embed.empty()) {
- LOG_ERROR("failed to build S2V audio window");
- return std::nullopt;
- }
- }
- int64_t t2 = ggml_time_ms();
- LOG_INFO("s2v conditioning prepared, taking %" PRId64 " ms", t2 - t1);
- }
-
- if (latents.init_latent.empty()) {
- latents.init_latent = sd->generate_init_latent(request->width, request->height, request->frames, true);
- }
-
- if ((sd_version_is_ltxav(sd->version) || sd_version_is_minimax_h3(sd->version)) &&
- !latents.audio_latent.empty()) {
- if (!latents.denoise_mask.empty()) {
- latents.denoise_mask = pack_ltxav_audio_and_video_denoise_mask(latents.denoise_mask,
- latents.init_latent,
- latents.audio_latent);
- }
- latents.init_latent = pack_ltxav_audio_and_video_latents(latents.init_latent, latents.audio_latent);
- }
-
- return latents;
- }
-
- static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- const GenerationRequest& request,
- const ImageGenerationLatents& latents,
- const RefImageParams& ref_image_params) {
- ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
-
- ImageGenerationEmbeds embeds;
- ConditionerParams condition_params;
- condition_params.clip_skip = request.clip_skip;
- condition_params.text = request.prompt;
- condition_params.zero_out_masked = true;
- condition_params.ref_images = &latents.ref_images;
- condition_params.minimax_h3_references = &latents.minimax_presentation_refs;
- condition_params.ref_image_params = ref_image_params;
-
- int64_t prepare_start_ms = ggml_time_ms();
- sd->generation_progress.report("conditioning", "positive_prompt");
- embeds.cond = sd->get_learned_condition(condition_params);
- if (embeds.cond.empty()) {
- LOG_ERROR("failed to encode video prompt");
- return std::nullopt;
- }
- embeds.cond.c_concat = latents.concat_latent;
- embeds.cond.c_vector = latents.clip_vision_output;
- if (sd_version_is_minimax_h3(sd->version)) {
- embeds.cond.c_ref_images = latents.ref_latents;
- embeds.cond.c_ref_audios = latents.reference_audio_latents;
- embeds.cond.c_reference_blocks = latents.minimax_reference_blocks;
- if (!latents.keyframe_indices.empty()) {
- embeds.cond.c_position_ids = sd::Tensor<int32_t>(
- {static_cast<int64_t>(latents.keyframe_indices.size())},
- latents.keyframe_indices);
- }
- }
- if (sd->version == VERSION_WAN2_2_S2V) {
- embeds.cond.c_ref_images = latents.ref_latents;
- if (!latents.s2v_audio_embed.empty()) {
- embeds.cond.c_ref_audios = {latents.s2v_audio_embed};
- }
- }
- if (request.use_uncond) {
- condition_params.text = request.negative_prompt;
- sd->generation_progress.report("conditioning", "negative_prompt");
- embeds.uncond = sd->get_learned_condition(condition_params);
- if (embeds.uncond.empty()) {
- LOG_ERROR("failed to encode negative video prompt");
- return std::nullopt;
- }
- embeds.uncond.c_concat = latents.concat_latent;
- embeds.uncond.c_vector = latents.clip_vision_output;
- if (sd_version_is_minimax_h3(sd->version)) {
- embeds.uncond.c_ref_images = latents.ref_latents;
- embeds.uncond.c_ref_audios = latents.reference_audio_latents;
- embeds.uncond.c_reference_blocks = latents.minimax_reference_blocks;
- embeds.uncond.c_position_ids = embeds.cond.c_position_ids;
- }
- if (sd->version == VERSION_WAN2_2_S2V) {
- embeds.uncond.c_ref_images = latents.ref_latents;
- if (!latents.s2v_audio_embed.empty()) {
- embeds.uncond.c_ref_audios = {sd::Tensor<float>::zeros_like(latents.s2v_audio_embed)};
- }
- }
- }
-
- int64_t t1 = ggml_time_ms();
- LOG_INFO("get_learned_condition completed, taking %.2fs", (t1 - prepare_start_ms) * 1.0f / 1000);
-
- return embeds;
- }
-
- static sd::Image* decode_video_outputs(StableDiffusionGGML* sd,
- const GenerationRequest& request,
- const sd::Tensor<float>& final_latent,
- int* num_frames_out) {
- if (final_latent.empty()) {
- LOG_ERROR("no latent video to decode");
- return nullptr;
- }
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling video decode");
- return nullptr;
- }
- sd::Tensor<float> video_latent = final_latent;
- if ((sd_version_is_ltxav(sd->version) || sd_version_is_minimax_h3(sd->version)) &&
- video_latent.shape()[3] > sd->get_latent_channel()) {
- video_latent = sd::ops::slice(video_latent, 3, 0, sd->get_latent_channel());
- }
- LOG_VERBOSE("decode_video_outputs latent %dx%dx%dx%d",
- (int)video_latent.shape()[0],
- (int)video_latent.shape()[1],
- (int)video_latent.shape()[2],
- (int)video_latent.shape()[3]);
- // auto z = sd::load_tensor_from_file_as_tensor<float>("ltx_vae_z.bin");
- int64_t t4 = ggml_time_ms();
- sd::Tensor<float> vid = sd->decode_first_stage(video_latent, true);
- int64_t t5 = ggml_time_ms();
- LOG_INFO("decode_first_stage completed, taking %.2fs", (t5 - t4) * 1.0f / 1000);
- if (vid.empty()) {
- LOG_ERROR("decode_first_stage failed for video");
- return nullptr;
- }
- LOG_VERBOSE("decode_video_outputs decoded %dx%dx%dx%d",
- (int)vid.shape()[0],
- (int)vid.shape()[1],
- (int)vid.shape()[2],
- (int)vid.shape()[3]);
- if (request.frames > 0 &&
- vid.shape()[2] > request.frames) {
- vid = sd::ops::slice(vid, 2, 0, request.frames);
- }
-
- sd::Image* result_images = (sd::Image*)calloc(vid.shape()[2], sizeof(sd::Image));
- if (result_images == nullptr) {
- return nullptr;
- }
- if (num_frames_out != nullptr) {
- *num_frames_out = static_cast<int>(vid.shape()[2]);
- }
-
- for (int64_t i = 0; i < vid.shape()[2]; i++) {
- result_images[i] = tensor_to_sd_image(vid, static_cast<int>(i));
- }
-
- return result_images;
- }
-
- static bool hires_upscaler_is_latent_interpolation(enum sd_hires_upscaler_t upscaler) {
- switch (upscaler) {
- case SD_HIRES_UPSCALER_LATENT:
- case SD_HIRES_UPSCALER_LATENT_NEAREST:
- case SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT:
- case SD_HIRES_UPSCALER_LATENT_ANTIALIASED:
- case SD_HIRES_UPSCALER_LATENT_BICUBIC:
- case SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED:
- return true;
- default:
- return false;
- }
- }
-
- static sd::ops::InterpolateMode hires_latent_interpolate_mode(enum sd_hires_upscaler_t upscaler,
- bool* antialias) {
- if (antialias != nullptr) {
- *antialias = false;
- }
- switch (upscaler) {
- case SD_HIRES_UPSCALER_LATENT_NEAREST:
- return sd::ops::InterpolateMode::Nearest;
- case SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT:
- return sd::ops::InterpolateMode::NearestExact;
- case SD_HIRES_UPSCALER_LATENT_BICUBIC:
- return sd::ops::InterpolateMode::Bicubic;
- case SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED:
- if (antialias != nullptr) {
- *antialias = true;
- }
- return sd::ops::InterpolateMode::Bicubic;
- case SD_HIRES_UPSCALER_LATENT_ANTIALIASED:
- if (antialias != nullptr) {
- *antialias = true;
- }
- return sd::ops::InterpolateMode::Bilinear;
- default:
- return sd::ops::InterpolateMode::Bilinear;
- }
- }
-
- static sd::Tensor<float> upscale_spatial_video_latent(StableDiffusionGGML* sd,
- const GenerationRequest& request,
- const sd::Tensor<float>& packed_latent,
- int audio_length) {
- if (sd == nullptr || packed_latent.empty()) {
- return {};
- }
- if (packed_latent.dim() != 4 && packed_latent.dim() != 5) {
- LOG_ERROR("video latent spatial upscale expects a 4D or 5D latent");
- return {};
- }
- int64_t target_width = request.hires.target_width / request.vae_scale_factor;
- int64_t target_height = request.hires.target_height / request.vae_scale_factor;
- if (target_width <= 0 || target_height <= 0) {
- LOG_ERROR("video latent spatial upscale target must be positive: %dx%d",
- request.hires.target_width,
- request.hires.target_height);
- return {};
- }
- if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) {
- LOG_DEBUG("video latent spatial upscale: target equals the draft latent size, skipping");
- return packed_latent;
- }
-
- int latent_channels = sd->get_latent_channel();
- sd::Tensor<float> video_latent = packed_latent;
- sd::Tensor<float> audio_latent;
- if (packed_latent.shape()[3] > latent_channels) {
- video_latent = sd::ops::slice(packed_latent, 3, 0, latent_channels);
- audio_latent = sd_version_is_minimax_h3(sd->version)
- ? unpack_minimax_h3_audio_latent(packed_latent, audio_length, latent_channels)
- : unpack_ltxav_audio_latent(packed_latent, audio_length, latent_channels);
- if (audio_latent.empty()) {
- LOG_ERROR("failed to unpack audio latent before spatial latent upscale");
- return {};
- }
- }
-
- bool antialias = false;
- auto mode = hires_latent_interpolate_mode(request.hires.upscaler, &antialias);
- std::vector<int64_t> target_shape = video_latent.shape();
- target_shape[0] = target_width;
- target_shape[1] = target_height;
-
- LOG_INFO("video latent %s spatial upscale: %" PRId64 "x%" PRId64 "x%" PRId64 "x%" PRId64 " -> %" PRId64 "x%" PRId64 "x%" PRId64 "x%" PRId64,
- sd_hires_upscaler_name(request.hires.upscaler),
- (int64_t)video_latent.shape()[0],
- (int64_t)video_latent.shape()[1],
- (int64_t)video_latent.shape()[2],
- (int64_t)video_latent.shape()[3],
- target_shape[0],
- target_shape[1],
- target_shape[2],
- target_shape[3]);
-
- sd::Tensor<float> upscaled = sd::ops::interpolate(video_latent, target_shape, mode, false, antialias);
- if (upscaled.empty()) {
- LOG_ERROR("video latent spatial upscale failed");
- return {};
- }
- if (audio_latent.empty()) {
- return upscaled;
- }
- return pack_ltxav_audio_and_video_latents(upscaled, audio_latent);
- }
-
- sd::Tensor<float> upscale_h3_learned_video_latent(StableDiffusionGGML* sd,
- const char* model_path,
- const sd::Tensor<float>& packed_latent,
- int audio_length,
- int64_t target_width,
- int64_t target_height) {
- if (sd == nullptr || sd->model_manager == nullptr || packed_latent.empty()) {
- return {};
- }
- if (strlen(SAFE_STR(model_path)) == 0) {
- LOG_ERROR("H3 learned latent upscale requires a hires model path");
- return {};
- }
- // The learned network is not an identity mapping at scale 1.
- if (packed_latent.shape()[0] == target_width && packed_latent.shape()[1] == target_height) {
- LOG_DEBUG("H3 learned latent upscale: target equals the draft latent size, skipping");
- return packed_latent;
- }
- if (!sd->ensure_backend_pair(SDBackendModule::UPSCALER)) {
- return {};
- }
-
- int latent_channels = sd->get_latent_channel();
- sd::Tensor<float> video_latent = packed_latent;
- sd::Tensor<float> audio_latent;
- if (packed_latent.shape()[3] > latent_channels) {
- video_latent = sd::ops::slice(packed_latent, 3, 0, latent_channels);
- audio_latent = unpack_minimax_h3_audio_latent(packed_latent, audio_length, latent_channels);
- if (audio_latent.empty()) {
- LOG_ERROR("failed to unpack H3 audio latent before learned latent upscale");
- return {};
- }
- }
-
- LOG_INFO("H3 learned latent spatial upscale: latent %dx%dx%dx%d -> %" PRId64 "x%" PRId64 " model output",
- (int)video_latent.shape()[0],
- (int)video_latent.shape()[1],
- (int)video_latent.shape()[2],
- (int)video_latent.shape()[3],
- target_width,
- target_height);
-
- if (sd->first_stage_model) {
- sd->first_stage_model->runner_end();
- }
-
- auto model_manager = sd->model_manager;
- struct UpsamplerScope {
- ModelManager& manager;
- ModelLoader::FileId owned_source = 0;
- std::unique_ptr<H3VUpsampler::H3LatentUpscalerRunner> runner;
- std::vector<ggml_tensor*> params;
-
- ~UpsamplerScope() {
- if (runner) {
- runner->runner_end();
- }
- GGML_ASSERT(manager.unregister_param_tensors(params));
- if (owned_source != 0) {
- GGML_ASSERT(manager.del_file(owned_source));
- }
- }
- } scope{*model_manager};
-
- const std::string prefix = "h3_latent_upscaler";
- ModelLoader candidate = model_manager->loader();
- ModelLoader::FileId source_file = 0;
- if (!candidate.add_file(model_path, prefix + ".", &source_file)) {
- LOG_ERROR("init H3 latent upsampler model loader from file failed: '%s'", model_path);
- return {};
- }
- const bool owns_source = model_manager->loader().file_revision(source_file) == 0;
- if (!model_manager->set_loader(std::move(candidate))) {
- return {};
- }
- scope.owned_source = owns_source ? source_file : 0;
-
- auto& upsampler = scope.runner;
- upsampler = std::make_unique<H3VUpsampler::H3LatentUpscalerRunner>(sd->backend_for(SDBackendModule::UPSCALER),
- model_manager->loader().get_tensor_storage_map(),
- prefix,
- model_manager);
- const size_t max_graph_vram_bytes = sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER);
- upsampler->set_max_graph_vram_bytes(max_graph_vram_bytes);
- if (upsampler->model == nullptr) {
- LOG_ERROR("init H3 latent upsampler from metadata failed");
- return {};
- }
-
- std::map<std::string, ggml_tensor*> tensors;
- upsampler->get_param_tensors(tensors);
- for (const auto& entry : tensors) {
- scope.params.push_back(entry.second);
- }
- if (!model_manager->register_param_tensors(ModelComponent::LatentUpsampler,
- std::move(tensors),
- ModelManager::ResidencyMode::ParamBackend,
- sd->backend_for(SDBackendModule::UPSCALER),
- sd->params_backend_for(SDBackendModule::UPSCALER)) ||
- !model_manager->validate_registered_tensors()) {
- LOG_ERROR("register H3 latent upsampler tensors with model manager failed");
- return {};
- }
-
- sd::Tensor<float> upscaled = upsampler->compute(sd->n_threads, video_latent, target_width, target_height);
- upsampler->runner_end();
- if (upscaled.empty()) {
- LOG_ERROR("H3 learned latent spatial upscale failed");
- return {};
- }
-
- if (!audio_latent.empty()) {
- upscaled = pack_ltxav_audio_and_video_latents(upscaled, audio_latent);
- }
- return upscaled;
- }
-
- sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
- const char* model_path,
- const sd::Tensor<float>& packed_latent,
- int audio_length) {
- if (sd == nullptr || sd->model_manager == nullptr || packed_latent.empty()) {
- return {};
- }
- if (strlen(SAFE_STR(model_path)) == 0) {
- LOG_ERROR("LTX latent spatial upscale requires a model path");
- return {};
- }
- if (!sd->ensure_backend_pair(SDBackendModule::UPSCALER)) {
- return {};
- }
-
- int latent_channels = sd->get_latent_channel();
- sd::Tensor<float> video_latent = packed_latent;
- sd::Tensor<float> audio_latent;
- if (packed_latent.shape()[3] > latent_channels) {
- video_latent = sd::ops::slice(packed_latent, 3, 0, latent_channels);
- audio_latent = unpack_ltxav_audio_latent(packed_latent, audio_length, latent_channels);
- }
-
- LOG_INFO("LTX latent spatial upscale: latent %dx%dx%dx%d -> model output",
- (int)video_latent.shape()[0],
- (int)video_latent.shape()[1],
- (int)video_latent.shape()[2],
- (int)video_latent.shape()[3]);
-
- sd::Tensor<float> unnormalized = sd->un_normalize_ltx_video_latents(video_latent);
- if (sd->first_stage_model) {
- sd->first_stage_model->runner_end();
- }
- if (unnormalized.empty()) {
- LOG_ERROR("LTX latent un-normalization failed before spatial upscale");
- return {};
- }
-
- auto model_manager = sd->model_manager;
- struct UpsamplerScope {
- ModelManager& manager;
- ModelLoader::FileId owned_source = 0;
- std::unique_ptr<LTXVUpsampler::LatentUpsamplerRunner> runner;
- std::vector<ggml_tensor*> params;
-
- ~UpsamplerScope() {
- if (runner) {
- runner->runner_end();
- }
- GGML_ASSERT(manager.unregister_param_tensors(params));
- if (owned_source != 0) {
- GGML_ASSERT(manager.del_file(owned_source));
- }
- }
- } scope{*model_manager};
-
- const std::string prefix = "ltx_latent_upsampler";
- ModelLoader candidate = model_manager->loader();
- ModelLoader::FileId source_file = 0;
- if (!candidate.add_file(model_path, prefix + ".", &source_file)) {
- LOG_ERROR("init LTX latent upsampler model loader from file failed: '%s'", model_path);
- return {};
- }
- const bool owns_source = model_manager->loader().file_revision(source_file) == 0;
- if (!model_manager->set_loader(std::move(candidate))) {
- return {};
- }
- scope.owned_source = owns_source ? source_file : 0;
-
- auto& upsampler = scope.runner;
- upsampler = std::make_unique<LTXVUpsampler::LatentUpsamplerRunner>(sd->backend_for(SDBackendModule::UPSCALER),
- model_manager->loader().get_tensor_storage_map(),
- prefix,
- model_manager);
- const size_t max_graph_vram_bytes = sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER);
- upsampler->set_max_graph_vram_bytes(max_graph_vram_bytes);
- if (upsampler->model == nullptr) {
- LOG_ERROR("init LTX latent upsampler from metadata failed");
- return {};
- }
-
- std::map<std::string, ggml_tensor*> tensors;
- upsampler->get_param_tensors(tensors);
- for (const auto& entry : tensors) {
- scope.params.push_back(entry.second);
- }
- if (!model_manager->register_param_tensors(ModelComponent::LatentUpsampler,
- std::move(tensors),
- ModelManager::ResidencyMode::ParamBackend,
- sd->backend_for(SDBackendModule::UPSCALER),
- sd->params_backend_for(SDBackendModule::UPSCALER)) ||
- !model_manager->validate_registered_tensors()) {
- LOG_ERROR("register LTX latent upsampler tensors with model manager failed");
- return {};
- }
-
- sd::Tensor<float> upscaled = upsampler->compute(sd->n_threads, unnormalized);
- upsampler->runner_end();
- if (upscaled.empty()) {
- LOG_ERROR("LTX latent spatial upscale failed");
- return {};
- }
-
- upscaled = sd->normalize_ltx_video_latents(upscaled);
- sd->first_stage_model->runner_end();
- if (upscaled.empty()) {
- LOG_ERROR("LTX latent normalization failed after spatial upscale");
- return {};
- }
-
- if (!audio_latent.empty()) {
- upscaled = pack_ltxav_audio_and_video_latents(upscaled, audio_latent);
- }
- return upscaled;
- }
-
- static bool apply_ltxv_refine_image_conditioning(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- const GenerationRequest& request,
- const ImageGenerationLatents& latents,
- sd::Tensor<float>* latent,
- sd::Tensor<float>* denoise_mask,
- sd::Tensor<float>* video_positions) {
- if (sd == nullptr || sd_vid_gen_params == nullptr ||
- latent == nullptr || latent->empty() || denoise_mask == nullptr || video_positions == nullptr) {
- return true;
- }
- if (sd_vid_gen_params->init_image.data == nullptr &&
- sd_vid_gen_params->end_image.data == nullptr) {
- return true;
- }
- constexpr float conditioning_strength = 1.f;
- int latent_channels = sd->get_latent_channel();
- sd::Tensor<float> video_latent = *latent;
- sd::Tensor<float> audio_latent;
- if (latent->shape()[3] > latent_channels) {
- video_latent = sd::ops::slice(*latent, 3, 0, latent_channels);
- audio_latent = unpack_ltxav_audio_latent(*latent, latents.audio_length, latent_channels);
- if (audio_latent.empty()) {
- LOG_ERROR("failed to unpack LTXAV audio latent before image-to-video inplace conditioning");
- return false;
- }
- }
-
- int image_width = static_cast<int>(video_latent.shape()[0]) * request.vae_scale_factor;
- int image_height = static_cast<int>(video_latent.shape()[1]) * request.vae_scale_factor;
- sd::Tensor<float> video_mask = make_ltxav_video_denoise_mask(video_latent, 1.f);
-
- if (sd_vid_gen_params->init_image.data != nullptr) {
- sd::Tensor<float> start_image = ensure_image_tensor_channels(
- sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height),
- sd->get_image_channels());
- if (!apply_ltxav_condition_image_by_latent_index(sd,
- start_image,
- &video_latent,
- &video_mask,
- 0,
- "init",
- conditioning_strength)) {
- return false;
- }
- }
-
- if (sd_vid_gen_params->end_image.data != nullptr) {
- sd::Tensor<float> end_image = ensure_image_tensor_channels(
- sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height),
- sd->get_image_channels());
- sd::Tensor<float> end_image_latent = encode_ltxav_condition_image(sd, end_image, "end");
- if (end_image_latent.empty()) {
- return false;
- }
-
- int frame_idx = request.frames - 1;
- if (frame_idx == 0) {
- if (!apply_ltxav_condition_by_latent_index(&video_latent,
- &video_mask,
- end_image_latent,
- 0,
- "end",
- 1.f - conditioning_strength)) {
- return false;
- }
- } else {
- if (latents.video_conditioning_frame_count <= 0 || latents.video_target_frame_count <= 0) {
- LOG_ERROR("LTXV FLF2V refine conditioning requires low-resolution keyframe conditioning metadata");
- return false;
- }
- int64_t target_latent_frames = latents.video_target_frame_count;
- if (!apply_ltxav_condition_by_latent_index(&video_latent,
- &video_mask,
- end_image_latent,
- target_latent_frames,
- "end",
- 1.f - conditioning_strength)) {
- return false;
- }
- *video_positions = build_ltxv_video_positions(video_latent.shape()[0],
- video_latent.shape()[1],
- target_latent_frames,
- end_image_latent.shape()[2],
- frame_idx,
- 1,
- request.fps,
- request.vae_scale_factor,
- 8,
- true);
- }
- }
-
- if (!audio_latent.empty()) {
- *latent = pack_ltxav_audio_and_video_latents(video_latent, audio_latent);
- *denoise_mask = pack_ltxav_audio_and_video_denoise_mask(video_mask, video_latent, audio_latent);
- } else {
- *latent = std::move(video_latent);
- *denoise_mask = std::move(video_mask);
- }
- LOG_INFO("LTXV refine image conditioning applied at %dx%d", image_width, image_height);
- return true;
- }
-
- static bool generate_animatediff_video(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- sd::Image** frames_out,
- int* num_frames_out) {
- int n_frames = sd_vid_gen_params->video_frames;
- if (n_frames < 1) {
- LOG_ERROR("AnimateDiff: --video-frames must be >= 1");
- return false;
- }
- if (n_frames > 32) {
- LOG_WARN("AnimateDiff motion modules have a 32-frame positional-encoding context; capping to 32");
- n_frames = 32;
- }
-
- sd::ImageConfig img_gen_params;
-
- img_gen_params.loras = sd_vid_gen_params->loras;
- img_gen_params.lora_count = sd_vid_gen_params->lora_count;
- img_gen_params.prompt = sd_vid_gen_params->prompt;
- img_gen_params.negative_prompt = sd_vid_gen_params->negative_prompt;
- img_gen_params.clip_skip = sd_vid_gen_params->clip_skip;
- img_gen_params.width = sd_vid_gen_params->width;
- img_gen_params.height = sd_vid_gen_params->height;
- img_gen_params.sample_params = sd_vid_gen_params->sample_params;
- img_gen_params.strength = sd_vid_gen_params->strength;
- img_gen_params.init_image = sd_vid_gen_params->init_image;
- img_gen_params.seed = sd_vid_gen_params->seed;
- img_gen_params.batch_count = 1;
- img_gen_params.control_strength = 1.0f;
- img_gen_params.vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
- img_gen_params.cache = sd_vid_gen_params->cache;
- img_gen_params.hires = sd_vid_gen_params->hires;
- img_gen_params.qwen_image_layers = 0;
- img_gen_params.circular_x = sd_vid_gen_params->circular_x;
- img_gen_params.circular_y = sd_vid_gen_params->circular_y;
- img_gen_params.image_preprocess = sd_vid_gen_params->image_preprocess;
- img_gen_params.ref_image_args = sd_vid_gen_params->ref_image_args;
-
- sd->animatediff_num_frames = n_frames;
- bool ok = false;
- try {
- ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
- } catch (...) {
- sd->animatediff_num_frames = 0;
- throw;
- }
- sd->animatediff_num_frames = 0;
- return ok;
- }
-
- bool generate_video(StableDiffusionGGML* sd,
- const sd::VideoConfig* sd_vid_gen_params,
- sd::Image** frames_out,
- int* num_frames_out,
- sd::Audio** audio_out,
- int* fps_out,
- nlohmann::json* info_out) {
- if (info_out)
- *info_out = nlohmann::json();
- GenerationRequest request(sd, sd_vid_gen_params);
- const auto info = sd::generation_info(request, *sd_vid_gen_params);
- if (sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) {
- LOG_INFO("AnimateDiff dispatch: %d frames, %dx%d",
- sd_vid_gen_params->video_frames, sd_vid_gen_params->width, sd_vid_gen_params->height);
- if (fps_out != nullptr) {
- *fps_out = std::max(1, sd_vid_gen_params->fps);
- }
- auto params = *sd_vid_gen_params;
- params.seed = request.seed;
- const bool ok = generate_animatediff_video(sd, ¶ms, frames_out, num_frames_out);
- if (ok && info_out)
- *info_out = info;
- return ok;
- }
-
- sd->reset_cancel_flag();
-
- const RefImageParams ref_image_params = sd->resolve_ref_image_params(sd_vid_gen_params->ref_image_args);
-
- int64_t t0 = ggml_time_ms();
- sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
- sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
- sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
- sd::VideoConfig processed_params = *sd_vid_gen_params;
- if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
- return false;
- sd_vid_gen_params = &processed_params;
- if (fps_out != nullptr) {
- *fps_out = request.fps;
- }
- bool latent_upscale_enabled = request.hires.enabled;
- GenerationRequest hires_request = request;
- if (latent_upscale_enabled) {
- const bool is_ltxav = sd_version_is_ltxav(sd->version);
- const bool is_h3 = sd_version_is_minimax_h3(sd->version);
- if (!is_ltxav && !is_h3) {
- LOG_ERROR("video latent spatial upscale is only supported for LTX and MiniMax-H3 video models");
- return false;
- }
- if (is_ltxav) {
- if (request.hires.upscaler != SD_HIRES_UPSCALER_MODEL) {
- LOG_ERROR("LTX latent spatial upscale currently requires hires upscaler MODEL");
- return false;
- }
- if (strlen(SAFE_STR(request.hires.model_path)) == 0) {
- LOG_ERROR("LTX latent spatial upscale is enabled but hires model path was not provided");
- return false;
- }
- } else {
- const bool model_upscaler = request.hires.upscaler == SD_HIRES_UPSCALER_MODEL;
- if (model_upscaler) {
- if (strlen(SAFE_STR(request.hires.model_path)) == 0) {
- LOG_ERROR("MiniMax-H3 learned hires upscaler is enabled but hires model path was not provided");
- return false;
- }
- } else if (!hires_upscaler_is_latent_interpolation(request.hires.upscaler)) {
- LOG_ERROR("MiniMax-H3 hires requires an interpolating latent upscaler (e.g. \"Latent\" or \"Latent (bicubic)\") or a MODEL upscaler checkpoint");
- return false;
- }
- if (sd_vid_gen_params->init_image.data != nullptr ||
- sd_vid_gen_params->end_image.data != nullptr) {
- LOG_ERROR("MiniMax-H3 hires does not support --init-img/--end-img keyframes yet; use T2VA or Ref2VA");
- return false;
- }
- if (request.hires.target_width <= 0 || request.hires.target_height <= 0) {
- LOG_ERROR("MiniMax-H3 hires requires a positive hires target size");
- return false;
- }
- }
- }
-
- sd->rng->manual_seed(request.seed);
- sd->sampler_rng->manual_seed(request.seed);
- if (!sd->apply_loras(sd_vid_gen_params->loras, sd_vid_gen_params->lora_count))
- return false;
- sd->reset_generation_extensions();
-
- SamplePlan plan(sd, sd_vid_gen_params, request);
- sd->generation_progress.report("preparing", "inputs");
- auto latent_inputs_opt = prepare_video_generation_latents(sd, sd_vid_gen_params, &request, ref_image_params);
- if (!latent_inputs_opt.has_value()) {
- return false;
- }
- ImageGenerationLatents latents = std::move(*latent_inputs_opt);
-
- auto embeds_opt = prepare_video_generation_embeds(sd,
- sd_vid_gen_params,
- request,
- latents,
- ref_image_params);
- if (!embeds_opt) {
- return false;
- }
- ImageGenerationEmbeds embeds = std::move(*embeds_opt);
- if (latent_upscale_enabled) {
- LOG_INFO("generate_video %dx%dx%d -> video latent spatial upscale",
- request.width,
- request.height,
- request.frames);
- } else {
- LOG_INFO("generate_video %dx%dx%d",
- request.width,
- request.height,
- request.frames);
- }
-
- int64_t latent_start = ggml_time_ms();
- int W = request.width / request.vae_scale_factor;
- int H = request.height / request.vae_scale_factor;
- int T = static_cast<int>(latents.init_latent.shape()[2]);
-
- sd::Tensor<float> x_t = latents.init_latent;
- sd::Tensor<float> noise = sd::Tensor<float>::randn_like(x_t, sd->rng);
-
- if (plan.high_noise_sample_steps > 0) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before high-noise sampling");
- return false;
- }
- LOG_VERBOSE("sample(high noise) %dx%dx%d", W, H, T);
-
- int64_t sampling_start = ggml_time_ms();
- std::vector<float> high_noise_sigmas(plan.sigmas.begin(), plan.sigmas.begin() + plan.high_noise_sample_steps + 1);
- plan.sigmas = std::vector<float>(plan.sigmas.begin() + plan.high_noise_sample_steps, plan.sigmas.end());
-
- sd::Tensor<float> x_t_sampled = sd->sample(sd->high_noise_diffusion_model,
- false,
- x_t,
- std::move(noise),
- embeds.cond,
- request.use_high_noise_uncond ? embeds.uncond : SDCondition(),
- embeds.img_uncond,
- sd::Tensor<float>(),
- 0.f,
- request.high_noise_guidance,
- plan.high_noise_eta,
- request.shifted_timestep,
- plan.high_noise_sample_method,
- sd->is_flow_denoiser(),
- plan.high_noise_extra_sample_args,
- high_noise_sigmas,
- std::vector<sd::Tensor<float>>{},
- ref_image_params,
- latents.denoise_mask,
- latents.vace_context,
- request.vace_strength,
- latents.audio_length,
- static_cast<float>(request.fps),
- request.cache_params,
- true,
- latents.video_positions);
- int64_t sampling_end = ggml_time_ms();
- if (x_t_sampled.empty()) {
- LOG_ERROR("sampling(high noise) failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
- return false;
- }
-
- x_t = std::move(x_t_sampled);
- noise = {};
- LOG_INFO("sampling(high noise) completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
- }
-
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before sampling");
- return false;
- }
- LOG_VERBOSE("sample %dx%dx%d", W, H, T);
- int64_t sampling_start = ggml_time_ms();
- sd::Tensor<float> final_latent = sd->sample(sd->diffusion_model,
- true,
- x_t,
- std::move(noise),
- embeds.cond,
- request.use_uncond ? embeds.uncond : SDCondition(),
- embeds.img_uncond,
- sd::Tensor<float>(),
- 0.f,
- request.guidance,
- plan.eta,
- request.shifted_timestep,
- plan.sample_method,
- sd->is_flow_denoiser(),
- plan.extra_sample_args,
- plan.sigmas,
- std::vector<sd::Tensor<float>>{},
- ref_image_params,
- latents.denoise_mask,
- latents.vace_context,
- request.vace_strength,
- latents.audio_length,
- static_cast<float>(request.fps),
- request.cache_params,
- plan.high_noise_sample_steps <= 0,
- latents.video_positions);
-
- int64_t sampling_end = ggml_time_ms();
- if (final_latent.empty()) {
- LOG_ERROR("sampling failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
- return false;
- }
- LOG_INFO("sampling completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
-
- if (latent_upscale_enabled) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before latent upscale");
- return false;
- }
- sd->generation_progress.report("upscaling", "hires");
- int64_t upscale_start = ggml_time_ms();
- sd::Tensor<float> upscaled_latent;
- if (sd_version_is_ltxav(sd->version)) {
- upscaled_latent = upscale_ltx_spatial_video_latent(sd,
- request.hires.model_path,
- final_latent,
- latents.audio_length);
- } else if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) {
- upscaled_latent = upscale_h3_learned_video_latent(sd,
- request.hires.model_path,
- final_latent,
- latents.audio_length,
- request.hires.target_width / request.vae_scale_factor,
- request.hires.target_height / request.vae_scale_factor);
- } else {
- upscaled_latent = upscale_spatial_video_latent(sd,
- request,
- final_latent,
- latents.audio_length);
- }
- int64_t upscale_end = ggml_time_ms();
- if (upscaled_latent.empty()) {
- return false;
- }
- LOG_INFO("video latent spatial upscale completed, taking %.2fs",
- (upscale_end - upscale_start) * 1.0f / 1000);
-
- x_t = std::move(upscaled_latent);
- hires_request.width = static_cast<int>(x_t.shape()[0]) * hires_request.vae_scale_factor;
- hires_request.height = static_cast<int>(x_t.shape()[1]) * hires_request.vae_scale_factor;
- int upscaled_latent_frames = static_cast<int>(x_t.shape()[2]);
- int upscaled_frames = sd->latent_frames_to_video_frames(upscaled_latent_frames);
- if (upscaled_frames != hires_request.frames) {
- LOG_INFO("video latent upsampler output latent frames %d, frames %d -> %d",
- upscaled_latent_frames,
- hires_request.frames,
- upscaled_frames);
- hires_request.frames = upscaled_frames;
- }
- if (sd_version_is_ltxav(sd->version) && latents.audio_length > 0) {
- int target_audio_length = get_ltxav_num_audio_latents(hires_request.frames, hires_request.fps);
- if (target_audio_length != latents.audio_length) {
- int latent_channels = sd->get_latent_channel();
- sd::Tensor<float> video_latent = x_t;
- sd::Tensor<float> audio_latent = latents.audio_latent;
- if (x_t.shape()[3] > latent_channels) {
- video_latent = sd::ops::slice(x_t, 3, 0, latent_channels);
- audio_latent = unpack_ltxav_audio_latent(x_t, latents.audio_length, latent_channels);
- }
- audio_latent = resize_ltxav_audio_latent(audio_latent, target_audio_length);
- if (audio_latent.empty()) {
- LOG_ERROR("failed to resize LTX audio latent for latent upscale: %d -> %d",
- latents.audio_length,
- target_audio_length);
- return false;
- }
- x_t = pack_ltxav_audio_and_video_latents(video_latent, audio_latent);
- latents.audio_latent = std::move(audio_latent);
- LOG_INFO("LTX audio latent length adjusted for latent upscale: %d -> %d",
- latents.audio_length,
- target_audio_length);
- latents.audio_length = target_audio_length;
- }
- }
- if (sd_version_is_ltxav(sd->version) &&
- (request.hires.target_width > 0 || request.hires.target_height > 0) &&
- (request.hires.target_width != hires_request.width || request.hires.target_height != hires_request.height)) {
- LOG_WARN("LTX latent spatial upsampler output is %dx%d; ignoring hires target %dx%d",
- hires_request.width,
- hires_request.height,
- request.hires.target_width,
- request.hires.target_height);
- }
- sd::Tensor<float> hires_denoise_mask;
- sd::Tensor<float> hires_video_positions;
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before latent upscale refine");
- return false;
- }
- if (sd_version_is_ltxav(sd->version) &&
- !apply_ltxv_refine_image_conditioning(sd,
- sd_vid_gen_params,
- hires_request,
- latents,
- &x_t,
- &hires_denoise_mask,
- &hires_video_positions)) {
- return false;
- }
- // H3 keyframe conditioning must be re-encoded at the hires resolution.
- if (sd_version_is_minimax_h3(sd->version) &&
- !latents.denoise_mask.empty() &&
- (latents.denoise_mask.shape()[0] != x_t.shape()[0] ||
- latents.denoise_mask.shape()[1] != x_t.shape()[1])) {
- LOG_ERROR("MiniMax-H3 hires cannot rescale an existing video denoise mask");
- return false;
- }
- noise = sd::Tensor<float>::randn_like(x_t, sd->rng);
-
- W = hires_request.width / hires_request.vae_scale_factor;
- H = hires_request.height / hires_request.vae_scale_factor;
- T = static_cast<int>(x_t.shape()[2]);
- sample_method_t hires_sample_method = plan.sample_method;
- int hires_scheduler_steps = 0;
- std::vector<float> hires_sigma_sched =
- make_hires_sigma_schedule(sd,
- request.hires,
- request.sample_params,
- plan.sample_steps,
- sd->get_image_seq_len(hires_request.height, hires_request.width) * T,
- &hires_scheduler_steps);
- float hires_eta = plan.eta;
-
- LOG_VERBOSE("sample(latent upscale) %dx%dx%d", W, H, T);
- LOG_INFO("video latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s",
- hires_scheduler_steps,
- request.hires.denoising_strength,
- sampling_methods_str[hires_sample_method],
- hires_sigma_sched.size(),
- request.hires.custom_sigmas_count > 0 ? ", custom_sigmas=true" : "");
-
- sampling_start = ggml_time_ms();
- final_latent = sd->sample(sd->diffusion_model,
- true,
- x_t,
- std::move(noise),
- embeds.cond,
- hires_request.use_uncond ? embeds.uncond : SDCondition(),
- embeds.img_uncond,
- sd::Tensor<float>(),
- 0.f,
- request.guidance,
- hires_eta,
- request.shifted_timestep,
- hires_sample_method,
- sd->is_flow_denoiser(),
- plan.extra_sample_args,
- hires_sigma_sched,
- std::vector<sd::Tensor<float>>{},
- ref_image_params,
- hires_denoise_mask,
- sd::Tensor<float>(),
- hires_request.vace_strength,
- latents.audio_length,
- static_cast<float>(hires_request.fps),
- hires_request.cache_params,
- false,
- hires_video_positions);
- sampling_end = ggml_time_ms();
- if (final_latent.empty()) {
- LOG_ERROR("sampling(latent upscale) failed after %.2fs",
- (sampling_end - sampling_start) * 1.0f / 1000);
- return false;
- }
- LOG_INFO("sampling(latent upscale) completed, taking %.2fs",
- (sampling_end - sampling_start) * 1.0f / 1000);
- }
-
- int64_t latent_end = ggml_time_ms();
- LOG_INFO("generating latent video completed, taking %.2fs", (latent_end - latent_start) * 1.0f / 1000);
-
- sd::Audio* generated_audio = nullptr;
- if (sd->version == VERSION_WAN2_2_S2V && sd_vid_gen_params->ref_audios_count > 0) {
- // Return the driving track for muxing with the generated video.
- const sd::Audio& driving = sd_vid_gen_params->ref_audios[0];
- generated_audio = (sd::Audio*)malloc(sizeof(sd::Audio));
- if (generated_audio != nullptr) {
- generated_audio->sample_rate = driving.sample_rate;
- generated_audio->channels = driving.channels;
- generated_audio->sample_count = driving.sample_count;
- generated_audio->data = (float*)malloc(sizeof(float) * driving.sample_count * driving.channels);
- if (generated_audio->data == nullptr) {
- free(generated_audio);
- generated_audio = nullptr;
- } else {
- memcpy(generated_audio->data,
- driving.data,
- sizeof(float) * driving.sample_count * driving.channels);
- }
- }
- if (generated_audio != nullptr) {
- LOG_DEBUG("s2v output audio: %u Hz, %u channels, %llu samples",
- generated_audio->sample_rate,
- generated_audio->channels,
- (unsigned long long)generated_audio->sample_count);
- } else {
- LOG_DEBUG("s2v output audio copy failed (out of memory)");
- }
- }
- if ((sd_version_is_ltxav(sd->version) || sd_version_is_minimax_h3(sd->version)) &&
- latents.audio_length > 0 &&
- sd->audio_vae_model != nullptr) {
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before audio decode");
- return false;
- }
- int64_t audio_latent_decode_start = ggml_time_ms();
-
- auto audio_latent = sd_version_is_minimax_h3(sd->version)
- ? unpack_minimax_h3_audio_latent(final_latent,
- latents.audio_length,
- sd->get_latent_channel())
- : unpack_ltxav_audio_latent(final_latent,
- latents.audio_length,
- sd->get_latent_channel());
- if (!audio_latent.empty()) {
- LOG_VERBOSE("decode audio latent %dx%dx%dx%d",
- (int)audio_latent.shape()[0],
- (int)audio_latent.shape()[1],
- (int)audio_latent.shape()[2],
- (int)audio_latent.shape()[3]);
- auto waveform = sd->decode_ltx_audio_latent(audio_latent);
- if (!waveform.empty()) {
- generated_audio = waveform_to_sd_audio(sd, waveform);
- } else {
- LOG_WARN("audio latent decode failed; continuing with silent video output");
- }
- }
- int64_t audio_latent_decode_end = ggml_time_ms();
- LOG_INFO("decoding audio latent completed, taking %.2fs", (audio_latent_decode_end - audio_latent_decode_start) * 1.0f / 1000);
- }
-
- if (latents.video_conditioning_frame_count > 0) {
- int64_t target_frames = latents.video_target_frame_count > 0 ? latents.video_target_frame_count
- : final_latent.shape()[2] - latents.video_conditioning_frame_count;
- final_latent = sd::ops::slice(final_latent, 2, 0, target_frames);
- }
-
- if (latents.ref_image_num > 0) {
- final_latent = sd::ops::slice(final_latent, 2, latents.ref_image_num, final_latent.shape()[2]);
- }
-
- if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
- LOG_ERROR("cancelling generation before video decode");
- sd::free_audio(generated_audio);
- return false;
- }
- auto result = decode_video_outputs(sd, latent_upscale_enabled ? hires_request : request, final_latent, num_frames_out);
- LOG_DEBUG("decode_video_outputs returned %s", result == nullptr ? "nullptr (failed)" : "frames");
- if (result == nullptr) {
- sd::free_audio(generated_audio);
- return false;
- }
-
- sd->lora_stat();
-
- int64_t t1 = ggml_time_ms();
- LOG_INFO("generate_video completed in %.2fs", (t1 - t0) * 1.0f / 1000);
- if (frames_out != nullptr) {
- *frames_out = result;
- }
- if ((sd->version == VERSION_WAN2_2_S2V || sd_version_is_minimax_h3(sd->version)) && generated_audio != nullptr) {
- // Limit the audio track to the generated video's duration.
- int fps = request.fps;
- uint64_t video_frames = num_frames_out != nullptr ? (uint64_t)*num_frames_out : 0;
- uint64_t want_samples = (uint64_t)((double)video_frames / fps * generated_audio->sample_rate);
- LOG_DEBUG("output audio truncate: %llu samples -> %llu (video %llu frames @ %d fps)",
- (unsigned long long)generated_audio->sample_count,
- (unsigned long long)want_samples,
- (unsigned long long)video_frames,
- fps);
- if (want_samples > 0 && want_samples < generated_audio->sample_count) {
- generated_audio->sample_count = want_samples;
- }
- }
- if (info_out) {
- try {
- *info_out = info;
- } catch (...) {
- sd::free_audio(generated_audio);
- throw;
- }
- }
- if (audio_out != nullptr) {
- *audio_out = generated_audio;
- } else {
- sd::free_audio(generated_audio);
- }
- return true;
- }
-
-} // namespace sd::pipeline
diff --git a/src/pipeline/video_upscale.cpp b/src/pipeline/video_upscale.cpp
deleted file mode 100644
index 98837e8..0000000
--- a/src/pipeline/video_upscale.cpp
+++ /dev/null
@@ -1,200 +0,0 @@
-#include "pipeline/video_upscale.h"
-#include "api/options.h"
-#include "api/result.h"
-#include "generation_info.h"
-
-#include <climits>
-
-#include "core/ggml_runner.h"
-#include "core/probe/conv_direct.h"
-#include "core/probe/flash_attn.h"
-#include "core/util.h"
-
-namespace sd {
- namespace pipeline {
-
- bool VideoUpscaleRuntime::flash_attention_for(SDBackendModule module) {
- const auto mode = flash_attn_assignment.get(module);
- if (mode == "on" || mode == "off")
- return mode == "on";
- const auto devices = backends.runtime_backends(module);
- bool enabled = !devices.empty(), any_enabled = false;
- for (auto backend : devices) {
- auto it = flash_attn_by_backend.find(backend);
- if (it == flash_attn_by_backend.end())
- it = flash_attn_by_backend.emplace(backend, sd_probe_flash_attention(backend, n_threads)).first;
- enabled = enabled && it->second;
- any_enabled = any_enabled || it->second;
- }
- if (any_enabled && !enabled)
- LOG_INFO("flash attention auto: %s -> off (compute devices disagree)", sd_backend_module_name(module));
- return enabled;
- }
-
- bool VideoUpscaleRuntime::conv_direct_for(SDBackendModule module, bool conv3d) {
- const auto mode = conv_direct_assignment.get(module);
- if (mode == "on" || mode == "off")
- return mode == "on";
- auto& cache = conv3d ? conv3d_direct_by_backend : conv2d_direct_by_backend;
- const auto devices = backends.runtime_backends(module);
- bool enabled = !devices.empty(), any_enabled = false;
- for (auto backend : devices) {
- auto it = cache.find(backend);
- if (it == cache.end())
- it = cache.emplace(backend, sd_probe_conv_direct(backend, conv3d)).first;
- enabled = enabled && it->second;
- any_enabled = any_enabled || it->second;
- }
- if (any_enabled && !enabled)
- LOG_INFO("conv%dd direct auto: %s -> off (compute devices disagree)", conv3d ? 3 : 2, sd_backend_module_name(module));
- return enabled;
- }
-
- void VideoUpscaleRuntime::configure_runner(GGMLRunner& runner, SDBackendModule module, bool conv2d, bool conv3d) {
- const bool flash = flash_attention_for(module);
- const bool direct2d = conv2d && conv_direct_for(module, false);
- const bool direct3d = conv3d && conv_direct_for(module, true);
- runner.set_flash_attention_enabled(flash);
- runner.set_conv2d_direct_enabled(direct2d);
- runner.set_conv3d_direct_enabled(direct3d);
- LOG_INFO("video upscale %s: flash attention %s, conv2d direct %s, conv3d direct %s",
- sd_backend_module_name(module), flash ? "on" : "off", direct2d ? "on" : "off", direct3d ? "on" : "off");
- }
-
- VideoUpscaler::~VideoUpscaler() {
- if (model_)
- model_->end();
- // ModelManager retains parameter pointers owned by the model runners.
- runtime_.manager.reset();
- model_.reset();
- }
-
- bool VideoUpscaler::init(const sd::VideoUpscalerConfig& params) {
- if (!params.diffusion_model_path || !*params.diffusion_model_path ||
- !params.vae_path || !*params.vae_path || params.upscale_factor <= 0) {
- LOG_ERROR("video upscale requires diffusion and decoder model paths and a positive upscale factor");
- return false;
- }
- upscale_factor_ = params.upscale_factor;
- runtime_.n_threads = params.n_threads > 0 ? params.n_threads : sd_get_num_physical_cores();
- executor_ = std::make_unique<sd::ParallelExecutor>(runtime_.n_threads);
- sd::ParallelScope scope(executor_.get());
- std::string error;
- if (!sd_parse_flash_attention_assignment(params.flash_attn_mode ? params.flash_attn_mode : "", &runtime_.flash_attn_assignment, &error) ||
- !sd_parse_conv_direct_assignment(params.conv_direct_mode ? params.conv_direct_mode : "", &runtime_.conv_direct_assignment, &error)) {
- LOG_ERROR("video upscale operator configuration: %s", error.c_str());
- return false;
- }
- if (!runtime_.backends.init(params.backend, params.params_backend, nullptr, &error)) {
- LOG_ERROR("video upscaler backend: %s", error.c_str());
- return false;
- }
- for (auto module : {SDBackendModule::DIFFUSION, SDBackendModule::VAE}) {
- if (!runtime_.backends.runtime_backend(module) || !runtime_.backends.params_backend(module))
- return false;
- }
- ModelLoader loader;
- if (!loader.init_from_file(params.diffusion_model_path, "diffusion.") ||
- !loader.init_from_file(params.vae_path, "vae."))
- return false;
- const auto& weights = loader.get_tensor_storage_map();
- if (is_relay_vsr_model(weights))
- model_ = create_relay_vsr_upscaler();
- else if (is_seedvr2_model(weights))
- model_ = create_seedvr2_upscaler();
- else {
- LOG_ERROR("unsupported video upscale model architecture");
- return false;
- }
- runtime_.manager = std::make_shared<ModelManager>();
- runtime_.manager->set_n_threads(runtime_.n_threads);
- if (!runtime_.manager->set_loader(loader))
- return false;
- return model_->init(runtime_, weights, upscale_factor_);
- }
-
- bool VideoUpscaler::upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out) {
- if (!result_out)
- return false;
- *result_out = nullptr;
- if (!model_ || !params.frames || params.frame_count <= 0 || params.fps <= 0) {
- LOG_ERROR("video upscale requires nonempty RGB frames and positive FPS");
- return false;
- }
- const auto& first = params.frames[0];
- if (!first.width || !first.height || first.width > uint32_t(INT_MAX / upscale_factor_) ||
- first.height > uint32_t(INT_MAX / upscale_factor_))
- return false;
- for (int i = 0; i < params.frame_count; ++i) {
- const auto& frame = params.frames[i];
- if (!frame.data || frame.channel != 3 || frame.width != first.width || frame.height != first.height) {
- LOG_ERROR("video upscale input frames must have equal dimensions and three RGB channels");
- return false;
- }
- }
- sd::ParallelScope scope(executor_.get());
- auto result = std::unique_ptr<sd::VideoGenerationResult, decltype(&sd::free_video_generation_result)>(
- static_cast<sd::VideoGenerationResult*>(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result);
- if (!result)
- return false;
- result->frames = static_cast<sd::Image*>(calloc(params.frame_count, sizeof(sd::Image)));
- if (!result->frames)
- return false;
- result->frame_count = params.frame_count;
- result->fps = params.fps;
- if (!model_->upscale(params, result.get()))
- return false;
- *result_out = result.release();
- return true;
- }
-
- } // namespace pipeline
-} // namespace sd
-
-struct sd_vid_upscaler {
- sd::pipeline::VideoUpscaler upscaler;
-};
-
-sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::VidUpscaler, config);
- if (status != SD_STATUS_OK)
- return status;
- auto result = std::make_unique<sd_vid_upscaler_t>();
- if (!result->upscaler.init(config.post.video_upscaler))
- return sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not load video upscaler");
- *out = result.release();
- return SD_STATUS_OK;
- },
- SD_STATUS_LOAD_FAILED);
-}
-
-void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler) {
- delete upscaler;
-}
-
-sd_status_t sd_upscale_video(sd_vid_upscaler_t* upscaler, const sd_vid_upscale_opt_t* opt, sd_result_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::VidUpscale, config);
- if (status != SD_STATUS_OK)
- return status;
- if (!upscaler)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
- sd::VideoGenerationResult* video = nullptr;
- if (!upscaler->upscaler.upscale(config.post.video_upscale, &video))
- return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "video upscale failed");
- return sd::make_video_result(video, out) ? SD_STATUS_OK
- : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
- },
- SD_STATUS_GENERATION_FAILED);
-}
diff --git a/src/pipeline/video_upscale.h b/src/pipeline/video_upscale.h
deleted file mode 100644
index 51ab768..0000000
--- a/src/pipeline/video_upscale.h
+++ /dev/null
@@ -1,62 +0,0 @@
-#ifndef __SD_PIPELINE_VIDEO_UPSCALE_H__
-#define __SD_PIPELINE_VIDEO_UPSCALE_H__
-
-#include <map>
-#include <memory>
-
-#include "api/result.h"
-#include "config.h"
-#include "core/ggml_extend_backend.h"
-#include "core/tensor.hpp"
-#include "model_manager.h"
-#include "stable-diffusion.h"
-
-struct GGMLRunner;
-
-namespace sd {
- namespace pipeline {
-
- struct VideoUpscaleRuntime {
- SDBackendManager backends;
- std::shared_ptr<ModelManager> manager;
- int n_threads = 1;
- SDBackendAssignment flash_attn_assignment;
- SDBackendAssignment conv_direct_assignment;
- std::map<ggml_backend_t, bool> flash_attn_by_backend;
- std::map<ggml_backend_t, bool> conv2d_direct_by_backend;
- std::map<ggml_backend_t, bool> conv3d_direct_by_backend;
-
- bool flash_attention_for(SDBackendModule module);
- bool conv_direct_for(SDBackendModule module, bool conv3d);
- void configure_runner(GGMLRunner& runner, SDBackendModule module, bool conv2d, bool conv3d);
- };
-
- class VideoUpscaleModel {
- public:
- virtual ~VideoUpscaleModel() = default;
- virtual bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& weights, int upscale_factor) = 0;
- virtual void end() = 0;
- virtual bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) = 0;
- };
-
- bool is_relay_vsr_model(const String2TensorStorage& weights);
- std::unique_ptr<VideoUpscaleModel> create_relay_vsr_upscaler();
- bool is_seedvr2_model(const String2TensorStorage& weights);
- std::unique_ptr<VideoUpscaleModel> create_seedvr2_upscaler();
-
- class VideoUpscaler {
- VideoUpscaleRuntime runtime_;
- int upscale_factor_ = 0;
- std::unique_ptr<sd::ParallelExecutor> executor_;
- std::unique_ptr<VideoUpscaleModel> model_;
-
- public:
- ~VideoUpscaler();
- bool init(const sd::VideoUpscalerConfig& params);
- bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out);
- };
-
- } // namespace pipeline
-} // namespace sd
-
-#endif // __SD_PIPELINE_VIDEO_UPSCALE_H__
diff --git a/src/runtime/cache_dit.hpp b/src/runtime/cache_dit.hpp
index 666e61c..f0029be 100644
--- a/src/runtime/cache_dit.hpp
+++ b/src/runtime/cache_dit.hpp
@@ -71,7 +71,8 @@ struct TaylorSeerState {
}
bool can_approximate() const {
- return last_computed_step >= n_derivatives && !dY_prev.empty() && !dY_prev[0].empty();
+ return last_computed_step >= 0 && !dY_current.empty() &&
+ std::all_of(dY_current.begin(), dY_current.end(), [](const auto& derivative) { return !derivative.empty(); });
}
void update_derivatives(const float* Y, size_t size, int step) {
@@ -102,7 +103,7 @@ struct TaylorSeerState {
}
void approximate(float* output, size_t size, int target_step) const {
- if (!can_approximate() || dY_prev[0].size() != size) {
+ if (!can_approximate() || dY_current[0].size() != size) {
return;
}
@@ -112,16 +113,16 @@ struct TaylorSeerState {
std::fill(output, output + size, 0.0f);
float factorial = 1.0f;
- int order = static_cast<int>(dY_prev.size());
+ int order = static_cast<int>(dY_current.size());
for (int o = 0; o < order; o++) {
- if (dY_prev[o].empty() || dY_prev[o].size() != size)
+ if (dY_current[o].empty() || dY_current[o].size() != size)
continue;
if (o > 0)
factorial *= static_cast<float>(o);
float coeff = ::powf(static_cast<float>(elapsed), static_cast<float>(o)) / factorial;
for (size_t i = 0; i < size; i++) {
- output[i] += coeff * dY_prev[o][i];
+ output[i] += coeff * dY_current[o][i];
}
}
}
@@ -638,46 +639,37 @@ inline void parse_taylorseer_options(const std::string& opts, TaylorSeerConfig&
struct CacheDitConditionState {
DBCacheConfig config;
TaylorSeerConfig taylor_config;
- bool initialized = false;
-
+ bool initialized = false;
int current_step_index = -1;
bool step_active = false;
+ bool can_read_cache = false;
bool skip_current_step = false;
- bool initial_step = true;
int warmup_remaining = 0;
std::vector<int> cached_steps;
int continuous_cached_steps = 0;
- float accumulated_residual_diff = 0.0f;
+ float accumulated_residual_diff = 0.f;
int total_steps_skipped = 0;
-
- const void* anchor_condition = nullptr;
+ const void* anchor_condition = nullptr;
struct CacheEntry {
std::vector<float> diff;
std::vector<float> prev_input;
- std::vector<float> prev_output;
- bool has_prev = false;
+ std::vector<int64_t> shape;
+ TaylorSeerState taylor;
};
std::unordered_map<const void*, CacheEntry> cache_diffs;
-
- TaylorSeerState taylor_state;
-
float start_sigma = std::numeric_limits<float>::max();
- float end_sigma = 0.0f;
+ float end_sigma = 0.f;
void reset_runtime() {
current_step_index = -1;
- step_active = false;
- skip_current_step = false;
- initial_step = true;
- warmup_remaining = config.max_warmup_steps;
+ step_active = can_read_cache = skip_current_step = false;
+ warmup_remaining = config.max_warmup_steps;
cached_steps.clear();
- continuous_cached_steps = 0;
- accumulated_residual_diff = 0.0f;
- total_steps_skipped = 0;
- anchor_condition = nullptr;
+ continuous_cached_steps = total_steps_skipped = 0;
+ accumulated_residual_diff = 0.f;
+ anchor_condition = nullptr;
cache_diffs.clear();
- taylor_state.reset();
}
void init(const DBCacheConfig& dbcfg, const TaylorSeerConfig& tcfg) {
@@ -685,211 +677,115 @@ struct CacheDitConditionState {
taylor_config = tcfg;
initialized = dbcfg.enabled || tcfg.enabled;
reset_runtime();
+ }
- if (taylor_config.enabled) {
- taylor_state.init(taylor_config.n_derivatives, 0);
+ void invalidate_condition(const void* condition) {
+ cache_diffs.erase(condition);
+ if (anchor_condition == condition) {
+ anchor_condition = nullptr;
+ skip_current_step = can_read_cache = false;
+ continuous_cached_steps = 0;
+ accumulated_residual_diff = 0.f;
+ warmup_remaining = config.max_warmup_steps;
}
}
void set_sigmas(const std::vector<float>& sigmas) {
if (!initialized || sigmas.size() < 2)
return;
-
- float start_percent = 0.15f;
- float end_percent = 0.95f;
-
- size_t n_steps = sigmas.size() - 1;
- size_t start_step = static_cast<size_t>(start_percent * n_steps);
- size_t end_step = static_cast<size_t>(end_percent * n_steps);
-
- if (start_step >= n_steps)
- start_step = n_steps - 1;
- if (end_step >= n_steps)
- end_step = n_steps - 1;
-
- start_sigma = sigmas[start_step];
- end_sigma = sigmas[end_step];
-
- if (start_sigma < end_sigma) {
+ const size_t n_steps = sigmas.size() - 1;
+ start_sigma = sigmas[std::min(n_steps - 1, size_t(.15f * n_steps))];
+ end_sigma = sigmas[std::min(n_steps - 1, size_t(.95f * n_steps))];
+ if (start_sigma < end_sigma)
std::swap(start_sigma, end_sigma);
- }
}
- bool enabled() const {
- return initialized && (config.enabled || taylor_config.enabled);
- }
+ bool enabled() const { return initialized && (config.enabled || taylor_config.enabled); }
+ bool step_is_active() const { return enabled() && step_active; }
+ bool is_step_skipped() const { return step_is_active() && skip_current_step; }
void begin_step(int step_index, float sigma) {
- if (!enabled())
+ if (!enabled() || step_index == current_step_index)
return;
- if (step_index == current_step_index)
- return;
-
current_step_index = step_index;
- skip_current_step = false;
- step_active = false;
-
- if (sigma > start_sigma)
- return;
- if (!(sigma > end_sigma))
+ skip_current_step = can_read_cache = false;
+ step_active = sigma <= start_sigma && sigma > end_sigma;
+ if (!step_active)
return;
-
- step_active = true;
-
if (warmup_remaining > 0) {
- warmup_remaining--;
+ --warmup_remaining;
return;
}
-
- if (!config.steps_computation_mask.empty()) {
- if (step_index < static_cast<int>(config.steps_computation_mask.size())) {
- if (config.steps_computation_mask[step_index] == 1) {
- return;
- }
- }
- }
-
- if (config.max_cached_steps >= 0 &&
- static_cast<int>(cached_steps.size()) >= config.max_cached_steps) {
+ if (step_index < int(config.steps_computation_mask.size()) && config.steps_computation_mask[step_index] == 1)
return;
- }
-
- if (config.max_continuous_cached_steps >= 0 &&
- continuous_cached_steps >= config.max_continuous_cached_steps) {
- return;
- }
- }
-
- bool step_is_active() const {
- return enabled() && step_active;
- }
-
- bool is_step_skipped() const {
- return enabled() && step_active && skip_current_step;
- }
-
- bool has_cache(const void* cond) const {
- auto it = cache_diffs.find(cond);
- return it != cache_diffs.end() && !it->second.diff.empty();
- }
-
- void update_cache(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
- CacheEntry& entry = cache_diffs[cond];
- if (!sd::store_condition_cache_diff(&entry.diff, input, output)) {
- entry.prev_input.clear();
- entry.prev_output.clear();
- entry.has_prev = false;
+ if (config.max_cached_steps >= 0 && int(cached_steps.size()) >= config.max_cached_steps)
return;
- }
-
- size_t size = static_cast<size_t>(output.numel());
- const float* input_data = input.data();
- const float* output_data = output.data();
- entry.prev_input.resize(size);
- entry.prev_output.resize(size);
- for (size_t i = 0; i < size; i++) {
- entry.prev_input[i] = input_data[i];
- entry.prev_output[i] = output_data[i];
- }
- entry.has_prev = true;
- }
-
- void apply_cache(const void* cond,
- const sd::Tensor<float>& input,
- sd::Tensor<float>* output) {
- auto it = cache_diffs.find(cond);
- if (it == cache_diffs.end() || it->second.diff.empty())
+ if (config.max_continuous_cached_steps >= 0 && continuous_cached_steps >= config.max_continuous_cached_steps)
return;
- sd::apply_condition_cache_diff(it->second.diff, input, output);
+ can_read_cache = true;
}
- bool before_condition(const void* cond, const sd::Tensor<float>& input, sd::Tensor<float>* output, float sigma, int step_index) {
- if (!enabled() || step_index < 0)
+ bool before_condition(const void* condition, const sd::Tensor<float>& input, sd::Tensor<float>* output, float sigma, int step_index) {
+ if (!enabled() || step_index < 0 || !condition || !output)
return false;
-
- if (step_index != current_step_index) {
+ if (step_index != current_step_index)
begin_step(step_index, sigma);
- }
-
if (!step_active)
return false;
-
- if (initial_step) {
- anchor_condition = cond;
- initial_step = false;
- }
-
- bool is_anchor = (cond == anchor_condition);
-
- if (skip_current_step) {
- if (has_cache(cond)) {
- apply_cache(cond, input, output);
- return true;
- }
+ if (!anchor_condition)
+ anchor_condition = condition;
+ if (!can_read_cache)
return false;
- }
-
- if (!is_anchor)
+ auto it = cache_diffs.find(condition);
+ if (it == cache_diffs.end() || it->second.shape != input.shape() || it->second.diff.empty())
return false;
-
- auto it = cache_diffs.find(cond);
- if (it == cache_diffs.end() || !it->second.has_prev)
+ auto& entry = it->second;
+ if (taylor_config.enabled &&
+ (!entry.taylor.can_approximate() || taylor_config.skip_interval_steps <= 0 ||
+ current_step_index - entry.taylor.last_computed_step > taylor_config.skip_interval_steps))
return false;
-
- size_t ne = static_cast<size_t>(input.numel());
- if (it->second.prev_input.size() != ne)
+ const bool anchor = condition == anchor_condition;
+ if (!anchor && !skip_current_step)
return false;
-
- const float* input_data = input.data();
- float diff = CacheDitState::calculate_residual_diff(
- it->second.prev_input.data(), input_data, ne);
-
- float effective_threshold = config.residual_diff_threshold;
- if (config.Fn_compute_blocks > 0) {
- float fn_confidence = 1.0f + 0.02f * (config.Fn_compute_blocks - 8);
- fn_confidence = std::max(0.5f, std::min(2.0f, fn_confidence));
- effective_threshold *= fn_confidence;
+ float diff = 0.f;
+ if (anchor && config.enabled) {
+ diff = CacheDitState::calculate_residual_diff(entry.prev_input.data(), input.data(), input.numel());
+ if (!(diff < config.residual_diff_threshold))
+ return false;
}
- if (config.Bn_compute_blocks > 0) {
- float bn_quality = 1.0f - 0.03f * config.Bn_compute_blocks;
- bn_quality = std::max(0.5f, std::min(1.0f, bn_quality));
- effective_threshold *= bn_quality;
+ if (taylor_config.enabled) {
+ *output = sd::Tensor<float>(input.shape());
+ entry.taylor.approximate(output->data(), output->numel(), current_step_index);
+ } else {
+ sd::apply_condition_cache_diff(entry.diff, input, output);
}
-
- if (diff < effective_threshold) {
+ if (anchor && !skip_current_step) {
skip_current_step = true;
- total_steps_skipped++;
+ ++total_steps_skipped;
cached_steps.push_back(current_step_index);
- continuous_cached_steps++;
+ ++continuous_cached_steps;
accumulated_residual_diff += diff;
- apply_cache(cond, input, output);
- return true;
}
-
- continuous_cached_steps = 0;
- return false;
+ return true;
}
- void after_condition(const void* cond, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
+ void after_condition(const void* condition, const sd::Tensor<float>& input, const sd::Tensor<float>& output) {
if (!step_is_active())
return;
-
- update_cache(cond, input, output);
-
- if (cond == anchor_condition && taylor_config.enabled) {
- taylor_state.update_derivatives(output.data(), static_cast<size_t>(output.numel()), current_step_index);
- }
- }
-
- void log_metrics() const {
- if (!enabled())
+ auto& entry = cache_diffs[condition];
+ if (!sd::store_condition_cache_diff(&entry.diff, input, output)) {
+ cache_diffs.erase(condition);
return;
-
- LOG_INFO("CacheDIT: steps_skipped=%d/%d (%.1f%%), accum_residual_diff=%.4f",
- total_steps_skipped,
- current_step_index + 1,
- (current_step_index > 0) ? (100.0f * total_steps_skipped / (current_step_index + 1)) : 0.0f,
- accumulated_residual_diff);
+ }
+ if (entry.shape != input.shape()) {
+ entry.taylor.init(taylor_config.n_derivatives, output.numel());
+ entry.shape = input.shape();
+ }
+ entry.prev_input = input.values();
+ if (taylor_config.enabled)
+ entry.taylor.update_derivatives(output.data(), output.numel(), current_step_index);
+ if (condition == anchor_condition)
+ continuous_cached_steps = 0;
}
};
diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp
index b947074..52d0640 100644
--- a/src/runtime/denoiser.hpp
+++ b/src/runtime/denoiser.hpp
@@ -1385,10 +1385,14 @@ struct MingImageFlowDenoiser : DiscreteFlowDenoiser {
return time_snr_shift(std::isfinite(shift) ? shift : resolution_shift, (t + 1.f) / 1000.f);
}
- std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler, SDVersion version, const char* extra_sample_args = nullptr) override {
+ void set_sequence_length(int image_seq_len) {
const float tokens = static_cast<float>(image_seq_len) / 4.f;
const float mu = tokens >= 4096.f ? 1.35f : 0.5f + (tokens - 256.f) * (1.15f - 0.5f) / (4096.f - 256.f);
resolution_shift = std::exp(mu);
+ }
+
+ std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler, SDVersion version, const char* extra_sample_args = nullptr) override {
+ set_sequence_length(image_seq_len);
if (scheduler != DISCRETE_SCHEDULER) {
return DiscreteFlowDenoiser::get_sigmas(n, image_seq_len, scheduler, version, extra_sample_args);
}
diff --git a/src/runtime/easycache.hpp b/src/runtime/easycache.hpp
index bae9f30..ae5cedc 100644
--- a/src/runtime/easycache.hpp
+++ b/src/runtime/easycache.hpp
@@ -68,6 +68,26 @@ struct EasyCacheState {
current_step_index = -1;
}
+ void invalidate_condition(const void* condition) {
+ cache_diffs.erase(condition);
+ if (anchor_condition != condition)
+ return;
+ anchor_condition = nullptr;
+ initial_step = true;
+ skip_current_step = false;
+ prev_input.clear();
+ prev_output.clear();
+ output_prev_norm = 0.0f;
+ has_prev_input = false;
+ has_prev_output = false;
+ has_output_prev_norm = false;
+ has_relative_transformation_rate = false;
+ relative_transformation_rate = 0.0f;
+ cumulative_change_rate = 0.0f;
+ last_input_change = 0.0f;
+ has_last_input_change = false;
+ }
+
void init(const EasyCacheConfig& cfg, Denoiser* d) {
config = cfg;
denoiser = d;
diff --git a/src/runtime/guidance.h b/src/runtime/guidance.h
index 3de3370..b79bfa2 100644
--- a/src/runtime/guidance.h
+++ b/src/runtime/guidance.h
@@ -72,6 +72,8 @@ namespace sd::guidance {
float image_guidance_scale,
AdaptiveProjectedGuidanceParams params);
+ void set_guidance_scale(float scale) { guidance_scale_ = scale; }
+
GuiderOutput forward(const GuidanceInput& input,
GuiderOutput previous,
std::optional<float> scale_override = std::nullopt) const override;
diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp
deleted file mode 100644
index d5e17d9..0000000
--- a/src/runtime/image_preprocess.cpp
+++ /dev/null
@@ -1,488 +0,0 @@
-#include "image_preprocess.h"
-#include "core/util.h"
-
-#include <climits>
-#include <set>
-
-namespace sd {
-
- std::pair<int, int> select_image_size_bucket(int source_width, int source_height, int area, int factor, int fallback_width, int fallback_height) {
- int width = fallback_width;
- int height = fallback_height;
- double ratio = static_cast<double>(source_width) / source_height;
- double best_crop = INFINITY;
- int64_t best_area_error = INT64_MAX;
- int min_size = 4 * factor;
- for (int w = min_size; w < 2LL * area / min_size; w += factor) {
- for (int h = min_size; h < 2LL * area / w; h += factor) {
- int64_t pixels = int64_t(w) * h;
- if (pixels < 0.9 * area || pixels > 1.1 * area) {
- continue;
- }
- double candidate_ratio = static_cast<double>(w) / h;
- double crop = std::round((1.0 - std::min(candidate_ratio / ratio, ratio / candidate_ratio)) * 1e9);
- int64_t area_error = std::abs(pixels - area);
- if (crop < best_crop || (crop == best_crop && (area_error < best_area_error || (area_error == best_area_error && w > width)))) {
- width = w;
- height = h;
- best_crop = crop;
- best_area_error = area_error;
- }
- }
- }
- return {width, height};
- }
-
- Tensor<float> resize_and_center_crop(const Tensor<float>& image, int width, int height, ops::InterpolateMode filter) {
- double scale = std::max(static_cast<double>(width) / image.shape()[0], static_cast<double>(height) / image.shape()[1]);
- int w = static_cast<int>(std::ceil(image.shape()[0] * scale));
- int h = static_cast<int>(std::ceil(image.shape()[1] * scale));
- auto resized = ops::interpolate(image, {w, h, image.shape()[2], image.shape()[3]}, filter);
- resized = ops::slice(resized, 0, (w - width) / 2, (w - width) / 2 + width);
- return ops::slice(resized, 1, (h - height) / 2, (h - height) / 2 + height);
- }
-
- static constexpr std::pair<const char*, ImageTarget> image_targets[] = {
- {"init", ImageTarget::Init},
- {"end", ImageTarget::End},
- {"mask", ImageTarget::Mask},
- {"control", ImageTarget::Control},
- {"ref", ImageTarget::Ref},
- {"ip-adapter", ImageTarget::IPAdapter},
- {"id", ImageTarget::ID},
- {"control-frame", ImageTarget::ControlFrame},
- };
-
- static constexpr std::pair<const char*, ImageResizeMode> image_resize_modes[] = {
- {"auto", ImageResizeMode::Auto},
- {"none", ImageResizeMode::None},
- {"stretch", ImageResizeMode::Stretch},
- {"crop", ImageResizeMode::Crop},
- {"crop-resize", ImageResizeMode::CropResize},
- {"fit-pad", ImageResizeMode::FitPad},
- };
-
- template <typename T, size_t N>
- static bool parse_enum(const std::string& text, const std::pair<const char*, T> (&names)[N], T& value) {
- for (const auto& entry : names) {
- if (text == entry.first) {
- value = entry.second;
- return true;
- }
- }
- return false;
- }
-
- template <typename T, size_t N>
- static const char* enum_name(T value, const std::pair<const char*, T> (&names)[N]) {
- for (const auto& entry : names) {
- if (value == entry.second)
- return entry.first;
- }
- return "unknown";
- }
-
- template <typename T>
- static bool one_of(T value, std::initializer_list<T> choices) {
- return std::find(choices.begin(), choices.end(), value) != choices.end();
- }
-
- static bool one_of(const std::string& value, std::initializer_list<const char*> choices) {
- for (const char* choice : choices) {
- if (value == choice)
- return true;
- }
- return false;
- }
-
- static ImageResizeMode resolve_mode(const std::map<std::string, std::string>& options, ImageResizeMode default_mode) {
- auto it = options.find("mode");
- ImageResizeMode mode = ImageResizeMode::Auto;
- if (it != options.end())
- parse_enum(it->second, image_resize_modes, mode);
- if (mode != ImageResizeMode::Auto)
- return mode;
- return options.count("width") && default_mode == ImageResizeMode::None ? ImageResizeMode::Stretch : default_mode;
- }
-
- bool ImagePreprocessor::fail(const std::string& message) const {
- LOG_ERROR("image preprocessing: %s", message.c_str());
- valid_ = false;
- return false;
- }
-
- ImagePreprocessor::ImagePreprocessor(const char* text) {
- if (text == nullptr || trim(text).empty())
- return;
- for (const auto& part : split_string(text, ';')) {
- ImagePreprocessRule rule;
- std::set<std::string> keys;
- if (trim(part).empty()) {
- fail("empty rule");
- return;
- }
- for (const auto& entry : split_string(part, ',')) {
- size_t equal = entry.find('=');
- if (equal == std::string::npos) {
- fail("expected key=value: " + entry);
- return;
- }
- std::string key = trim(entry.substr(0, equal));
- std::string value = trim(entry.substr(equal + 1));
- bool ok = !value.empty() && keys.insert(key).second;
- int number = 0;
- if (key == "target") {
- ok &= parse_enum(value, image_targets, rule.target);
- } else if (key == "index") {
- ok &= parse_strict_int(value, rule.index) && rule.index >= 0;
- } else {
- if (key == "mode") {
- ImageResizeMode mode;
- ok &= parse_enum(value, image_resize_modes, mode);
- } else if (key == "filter") {
- ok &= one_of(value, {"auto", "nearest", "nearest-exact", "bilinear", "bicubic", "lanczos"});
- } else if (key == "antialias") {
- ok &= one_of(value, {"auto", "true", "false"});
- } else if (key == "canny") {
- ok &= one_of(value, {"true", "false"});
- } else if (key == "anchor") {
- ok &= one_of(value, {"center", "top", "bottom", "left", "right"});
- } else if (key == "width" || key == "height") {
- ok &= parse_strict_int(value, number) && number > 0;
- } else if (key == "pad_color") {
- ok &= value.size() == 7 || value.size() == 9;
- ok &= !value.empty() && value[0] == '#';
- for (size_t i = 1; i < value.size(); ++i)
- ok &= std::isxdigit(static_cast<unsigned char>(value[i])) != 0;
- } else {
- ok = false;
- }
- rule.options[key] = value;
- }
- if (!ok) {
- fail("invalid or duplicate option: " + entry);
- return;
- }
- }
- if (!keys.count("target") || rule.options.empty() ||
- rule.options.count("width") != rule.options.count("height") ||
- (rule.index >= 0 && !one_of(rule.target, {ImageTarget::Ref, ImageTarget::ID, ImageTarget::ControlFrame}))) {
- fail("invalid target, index, or incomplete dimensions: " + part);
- return;
- }
- rules_.push_back(std::move(rule));
- }
- for (const auto& rule : rules_) {
- const auto options = resolve_options(rule.target, std::max(0, rule.index));
- if (options.count("antialias") && options.at("antialias") == "true" && options.count("filter") &&
- one_of(options.at("filter"), {"nearest", "nearest-exact"})) {
- fail("antialias requires bilinear, bicubic, or lanczos");
- return;
- }
- }
- }
-
- std::map<std::string, std::string> ImagePreprocessor::resolve_options(ImageTarget target, int index) const {
- std::map<std::string, std::string> options;
- for (int specificity = 0; specificity < 2; ++specificity) {
- for (const auto& rule : rules_) {
- if (rule.target == target &&
- rule.index == (specificity == 0 ? -1 : index)) {
- for (const auto& entry : rule.options)
- options[entry.first] = entry.second;
- }
- }
- }
- return options;
- }
-
- bool ImagePreprocessor::validate_inputs(const sd::ImageConfig& params) const {
- const std::map<ImageTarget, int> counts = {
- {ImageTarget::Init, params.init_image.data != nullptr},
- {ImageTarget::Mask, params.mask_image.data != nullptr},
- {ImageTarget::Control, params.control_image.data != nullptr},
- {ImageTarget::IPAdapter, params.ip_adapter_image.data != nullptr},
- {ImageTarget::Ref, params.ref_images != nullptr ? params.ref_images_count : 0},
- {ImageTarget::ID, params.pm_params.id_images != nullptr ? params.pm_params.id_images_count : 0},
- };
- for (const auto& rule : rules_) {
- auto it = counts.find(rule.target);
- int count = it == counts.end() ? 0 : it->second;
- if (count <= 0 || rule.index >= count) {
- return fail(std::string("rule targets an unavailable image: ") + enum_name(rule.target, image_targets));
- }
- }
- return valid_;
- }
-
- bool ImagePreprocessor::validate_inputs(const sd::VideoConfig& params) const {
- const std::map<ImageTarget, int> counts = {
- {ImageTarget::Init, params.init_image.data != nullptr},
- {ImageTarget::End, params.end_image.data != nullptr},
- {ImageTarget::Ref, params.ref_images != nullptr ? params.ref_images_count : 0},
- {ImageTarget::ControlFrame, params.control_frames != nullptr ? params.control_frames_size : 0},
- };
- for (const auto& rule : rules_) {
- auto it = counts.find(rule.target);
- int count = it == counts.end() ? 0 : it->second;
- if (count <= 0 || rule.index >= count)
- return fail(std::string("rule targets an unavailable video input: ") + enum_name(rule.target, image_targets));
- }
- return valid_;
- }
-
- static int anchor_offset(int remaining, const std::string& anchor, bool horizontal) {
- if (anchor == (horizontal ? "left" : "top"))
- return 0;
- if (anchor == (horizontal ? "right" : "bottom"))
- return remaining;
- return remaining / 2;
- }
-
- Tensor<float> ImagePreprocessor::apply_transform(const Tensor<float>& image, const std::map<std::string, std::string>& options, ImageTransform p, const std::string& label, ops::InterpolateMode default_filter) const {
- auto value = [&](const char* key, const char* fallback) {
- auto it = options.find(key);
- return it == options.end() ? std::string(fallback) : it->second;
- };
- std::string filter = value("filter", "auto");
- ops::InterpolateMode mode = default_filter;
- if (filter == "nearest")
- mode = ops::InterpolateMode::Nearest;
- if (filter == "nearest-exact")
- mode = ops::InterpolateMode::NearestExact;
- if (filter == "bilinear")
- mode = ops::InterpolateMode::Bilinear;
- if (filter == "bicubic")
- mode = ops::InterpolateMode::Bicubic;
- if (filter == "lanczos")
- mode = ops::InterpolateMode::Lanczos;
- bool filtered = ops::is_2d_filter_interpolate_mode(mode);
- bool antialias = value("antialias", "auto") == "true" ||
- (value("antialias", "auto") == "auto" && filtered &&
- (p.resize_width < p.crop_width || p.resize_height < p.crop_height));
- if (antialias && !filtered) {
- fail(label + ": antialias requires bilinear, bicubic, or lanczos");
- return {};
- }
- auto cropped = ops::slice(ops::slice(image, 0, p.x, p.x + p.crop_width), 1, p.y, p.y + p.crop_height);
- int channels = static_cast<int>(image.shape()[2]);
- bool resize = p.resize_width != p.crop_width || p.resize_height != p.crop_height;
- if (resize && channels == 4 && filtered) {
- for (int64_t i = 0, pixels = cropped.shape()[0] * cropped.shape()[1]; i < pixels; ++i) {
- for (int c = 0; c < 3; ++c)
- cropped[i + c * pixels] *= cropped[i + 3 * pixels];
- }
- }
- auto resized = ops::interpolate(cropped, {p.resize_width, p.resize_height, channels, 1}, mode, false, antialias);
- if (resize && channels == 4 && filtered) {
- for (int64_t i = 0, pixels = resized.shape()[0] * resized.shape()[1]; i < pixels; ++i) {
- float alpha = std::clamp(resized[i + 3 * pixels], 0.f, 1.f);
- for (int c = 0; c < 3; ++c)
- resized[i + c * pixels] = alpha > 1e-6f ? resized[i + c * pixels] / alpha : 0.f;
- }
- }
- resized = ops::clamp(resized, 0.f, 1.f);
- Tensor<float> output({p.width, p.height, channels, 1});
- std::string color = value("pad_color", "#000000ff");
- if (color.size() == 7)
- color += "ff";
- uint8_t rgba[4];
- for (int c = 0; c < 4; ++c)
- rgba[c] = static_cast<uint8_t>(std::strtoul(color.substr(1 + c * 2, 2).c_str(), nullptr, 16));
- for (int c = 0; c < channels; ++c) {
- float fill = rgba[channels == 1 ? 0 : c] / 255.f;
- for (int y = 0; y < p.height; ++y) {
- for (int x = 0; x < p.width; ++x) {
- output.index(x, y, c, 0) = x >= p.pad_x && x < p.pad_x + p.resize_width && y >= p.pad_y && y < p.pad_y + p.resize_height
- ? resized.index(x - p.pad_x, y - p.pad_y, c, 0)
- : fill;
- }
- }
- }
- LOG_INFO("preprocess %s: %dx%d crop=(%d,%d,%d,%d) resize=%dx%d pad=(%d,%d) output=%dx%d filter=%s(%d) antialias=%s",
- label.c_str(), p.source_width, p.source_height, p.x, p.y, p.crop_width, p.crop_height,
- p.resize_width, p.resize_height, p.pad_x, p.pad_y, p.width, p.height, filter.c_str(), static_cast<int>(mode), BOOL_STR(antialias));
- return output;
- }
-
- Tensor<float> ImagePreprocessor::apply_geometry(const Tensor<float>& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode, ops::InterpolateMode default_filter, ImageTransform* plan_out) const {
- if (!valid_ || image.empty())
- return {};
- const std::string label = std::string(enum_name(target, image_targets)) + "[" + std::to_string(index) + "]";
- auto options = resolve_options(target, index);
- if (image.dim() != 4 || image.shape()[3] != 1 || image.shape()[2] < 1 || image.shape()[2] > 4) {
- fail(label + ": expected one image with 1 to 4 channels");
- return {};
- }
- ImageTransform p;
- p.source_width = p.crop_width = static_cast<int>(image.shape()[0]);
- p.source_height = p.crop_height = static_cast<int>(image.shape()[1]);
- int target_width = width > 0 ? width : p.source_width;
- int target_height = height > 0 ? height : p.source_height;
- if (options.count("width")) {
- parse_strict_int(options.at("width"), target_width);
- parse_strict_int(options.at("height"), target_height);
- }
- ImageResizeMode mode = resolve_mode(options, default_mode);
- std::string anchor = options.count("anchor") ? options.at("anchor") : "center";
- p.width = p.resize_width = target_width;
- p.height = p.resize_height = target_height;
- if (mode == ImageResizeMode::None) {
- if (options.count("width") && (target_width != p.source_width || target_height != p.source_height)) {
- fail(label + ": mode=none conflicts with requested dimensions");
- return {};
- }
- p.width = p.resize_width = p.source_width;
- p.height = p.resize_height = p.source_height;
- } else if (mode == ImageResizeMode::Crop || mode == ImageResizeMode::CropResize) {
- if (mode == ImageResizeMode::Crop) {
- p.crop_width = target_width;
- p.crop_height = target_height;
- } else if (int64_t(p.source_width) * target_height > int64_t(p.source_height) * target_width) {
- p.crop_width = std::max(1, static_cast<int>(int64_t(p.source_height) * target_width / target_height));
- } else {
- p.crop_height = std::max(1, static_cast<int>(int64_t(p.source_width) * target_height / target_width));
- }
- if (p.crop_width > p.source_width || p.crop_height > p.source_height) {
- fail(label + ": crop exceeds source dimensions");
- return {};
- }
- p.x = anchor_offset(p.source_width - p.crop_width, anchor, true);
- p.y = anchor_offset(p.source_height - p.crop_height, anchor, false);
- } else if (mode == ImageResizeMode::FitPad) {
- double scale = std::min(double(target_width) / p.source_width, double(target_height) / p.source_height);
- p.resize_width = std::max(1, std::min(target_width, static_cast<int>(std::round(p.source_width * scale))));
- p.resize_height = std::max(1, std::min(target_height, static_cast<int>(std::round(p.source_height * scale))));
- p.pad_x = anchor_offset(target_width - p.resize_width, anchor, true);
- p.pad_y = anchor_offset(target_height - p.resize_height, anchor, false);
- }
- if (p.width <= 0 || p.height <= 0) {
- fail(label + ": invalid output dimensions");
- return {};
- }
- uint64_t max_pixels = std::min<uint64_t>(INT64_MAX, SIZE_MAX / sizeof(float)) / static_cast<uint64_t>(image.shape()[2]);
- if (uint64_t(p.width) * p.height > max_pixels || uint64_t(p.resize_width) * p.resize_height > max_pixels) {
- fail(label + ": image allocation size overflows");
- return {};
- }
- if (plan_out != nullptr)
- *plan_out = p;
- return apply_transform(image, options, p, label, default_filter);
- }
-
- Tensor<float> ImagePreprocessor::preprocess_input(sd::Image image, ImageTarget target, int index, int width, int height) {
- if (image.data == nullptr || image.width == 0 || image.height == 0 || image.width > INT_MAX || image.height > INT_MAX || image.channel < 1 || image.channel > 4) {
- fail(std::string(enum_name(target, image_targets)) + ": invalid input image");
- return {};
- }
- auto tensor = sd_image_to_tensor(image);
- if (target == ImageTarget::Mask && has_init_transform_) {
- auto options = resolve_options(target, index);
- if (image.width != init_transform_.source_width || image.height != init_transform_.source_height) {
- ImagePreprocessor alignment;
- tensor = alignment.apply_geometry(tensor, target, index, init_transform_.source_width, init_transform_.source_height,
- ImageResizeMode::CropResize, ops::InterpolateMode::NearestExact);
- if (tensor.empty()) {
- fail("could not align mask to init source dimensions");
- return {};
- }
- }
- bool geometry_override = options.count("width") || options.count("anchor") ||
- (options.count("mode") && options.at("mode") != "auto");
- if (geometry_override) {
- ImageTransform p;
- auto init_options = resolve_options(ImageTarget::Init, 0);
- ImageResizeMode default_mode = resolve_mode(init_options, ImageResizeMode::CropResize);
- auto result = apply_geometry(tensor, target, index, init_transform_.width, init_transform_.height, default_mode, ops::InterpolateMode::NearestExact, &p);
- if (result.empty())
- return {};
- const auto& q = init_transform_;
- if (p.x != q.x || p.y != q.y || p.crop_width != q.crop_width || p.crop_height != q.crop_height ||
- p.resize_width != q.resize_width || p.resize_height != q.resize_height || p.pad_x != q.pad_x || p.pad_y != q.pad_y || p.width != q.width || p.height != q.height) {
- fail("mask geometry conflicts with init; configure geometry on init and filter on mask");
- return {};
- }
- return result;
- }
- return apply_transform(tensor, options, init_transform_, "mask[0]", ops::InterpolateMode::NearestExact);
- }
- auto result = apply_geometry(tensor, target, index, width, height, width > 0 ? ImageResizeMode::CropResize : ImageResizeMode::None,
- target == ImageTarget::Mask ? ops::InterpolateMode::NearestExact : ops::InterpolateMode::Nearest,
- target == ImageTarget::Init ? &init_transform_ : nullptr);
- if (target == ImageTarget::Init)
- has_init_transform_ = !result.empty();
- return result;
- }
-
- ImagePreprocessor::~ImagePreprocessor() {
- for (const auto& image : owned_images_)
- std::free(image.data);
- }
-
- bool ImagePreprocessor::prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height) {
- if (image.data == nullptr)
- return true;
- auto options = resolve_options(target, index);
- bool canny = options.count("canny") && options.at("canny") == "true";
- auto tensor = preprocess_input(image, target, index, width, height);
- if (tensor.empty())
- return false;
- auto output = tensor_to_sd_image(tensor);
- if (output.data == nullptr)
- return fail("could not allocate input preprocessing buffer");
- owned_images_.push_back(output);
- if (canny && !sd::preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false))
- return fail("Canny preprocessing failed");
- image = output;
- return true;
- }
-
- bool ImagePreprocessor::prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector<sd::Image>& storage, int width, int height) {
- if (count < 0 || (count > 0 && images == nullptr))
- return fail(std::string("invalid image array: ") + enum_name(target, image_targets));
- if (count == 0)
- return true;
- storage.assign(images, images + count);
- for (int i = 0; i < count; ++i) {
- if (storage[i].data == nullptr)
- return fail(std::string("empty image in array: ") + enum_name(target, image_targets));
- if (!prepare_image(storage[i], target, i, width, height))
- return false;
- }
- images = storage.data();
- return true;
- }
-
- bool ImagePreprocessor::prepare_inputs(sd::ImageConfig& params, int width, int height) {
- if (prepared_)
- return fail("inputs have already been prepared");
- prepared_ = true;
- if (!valid_ || !validate_inputs(params))
- return false;
- if (!prepare_image(params.init_image, ImageTarget::Init, 0, width, height) ||
- !prepare_image(params.mask_image, ImageTarget::Mask, 0, width, height) ||
- !prepare_image(params.control_image, ImageTarget::Control, 0, width, height) ||
- !prepare_image(params.ip_adapter_image, ImageTarget::IPAdapter, 0, -1, -1) ||
- !prepare_array(params.ref_images, params.ref_images_count, ImageTarget::Ref, ref_images_) ||
- !prepare_array(params.pm_params.id_images, params.pm_params.id_images_count, ImageTarget::ID, id_images_))
- return false;
- params.image_preprocess = {};
- return true;
- }
-
- bool ImagePreprocessor::prepare_inputs(sd::VideoConfig& params, int width, int height) {
- if (prepared_)
- return fail("inputs have already been prepared");
- prepared_ = true;
- if (!valid_ || !validate_inputs(params))
- return false;
- if (!prepare_image(params.init_image, ImageTarget::Init, 0, width, height) ||
- !prepare_image(params.end_image, ImageTarget::End, 0, width, height) ||
- !prepare_array(params.ref_images, params.ref_images_count, ImageTarget::Ref, ref_images_) ||
- !prepare_array(params.control_frames, params.control_frames_size, ImageTarget::ControlFrame, control_frames_, width, height))
- return false;
- params.image_preprocess = {};
- return true;
- }
-
-} // namespace sd
diff --git a/src/runtime/image_preprocess.h b/src/runtime/image_preprocess.h
deleted file mode 100644
index 078befd..0000000
--- a/src/runtime/image_preprocess.h
+++ /dev/null
@@ -1,95 +0,0 @@
-#ifndef __SD_RUNTIME_IMAGE_PREPROCESS_H__
-#define __SD_RUNTIME_IMAGE_PREPROCESS_H__
-
-#include "core/media.h"
-
-#include <map>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "config.h"
-#include "core/tensor.hpp"
-#include "stable-diffusion.h"
-
-namespace sd {
-
- std::pair<int, int> select_image_size_bucket(int source_width, int source_height, int area, int factor, int fallback_width, int fallback_height);
- Tensor<float> resize_and_center_crop(const Tensor<float>& image, int width, int height, ops::InterpolateMode filter = ops::InterpolateMode::Lanczos);
-
- enum class ImageTarget {
- Init,
- End,
- Mask,
- Control,
- Ref,
- IPAdapter,
- ID,
- ControlFrame,
- };
-
- enum class ImageResizeMode {
- Auto,
- None,
- Stretch,
- Crop,
- CropResize,
- FitPad,
- };
-
- struct ImageTransform {
- int source_width = 0;
- int source_height = 0;
- int x = 0;
- int y = 0;
- int crop_width = 0;
- int crop_height = 0;
- int resize_width = 0;
- int resize_height = 0;
- int width = 0;
- int height = 0;
- int pad_x = 0;
- int pad_y = 0;
- };
-
- struct ImagePreprocessRule {
- ImageTarget target = ImageTarget::Init;
- int index = -1;
- std::map<std::string, std::string> options;
- };
-
- class ImagePreprocessor {
- std::vector<ImagePreprocessRule> rules_;
- mutable bool valid_ = true;
- ImageTransform init_transform_;
- bool has_init_transform_ = false;
- bool prepared_ = false;
- std::vector<sd::Image> owned_images_;
- std::vector<sd::Image> ref_images_;
- std::vector<sd::Image> id_images_;
- std::vector<sd::Image> control_frames_;
-
- bool fail(const std::string& message) const;
- std::map<std::string, std::string> resolve_options(ImageTarget target, int index) const;
- Tensor<float> apply_transform(const Tensor<float>& image, const std::map<std::string, std::string>& options, ImageTransform plan, const std::string& label, ops::InterpolateMode default_filter) const;
-
- bool prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height);
- bool prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector<sd::Image>& storage, int width = -1, int height = -1);
-
- public:
- explicit ImagePreprocessor(const char* rules = nullptr);
- ~ImagePreprocessor();
- ImagePreprocessor(const ImagePreprocessor&) = delete;
- ImagePreprocessor& operator=(const ImagePreprocessor&) = delete;
- bool prepare_inputs(sd::ImageConfig& params, int width, int height);
- bool prepare_inputs(sd::VideoConfig& params, int width, int height);
- bool is_valid() const { return valid_; }
- bool validate_inputs(const sd::ImageConfig& params) const;
- bool validate_inputs(const sd::VideoConfig& params) const;
- Tensor<float> apply_geometry(const Tensor<float>& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode = ImageResizeMode::Stretch, ops::InterpolateMode default_filter = ops::InterpolateMode::Nearest, ImageTransform* plan_out = nullptr) const;
- Tensor<float> preprocess_input(sd::Image image, ImageTarget target, int index = 0, int width = -1, int height = -1);
- };
-
-} // namespace sd
-
-#endif // __SD_RUNTIME_IMAGE_PREPROCESS_H__
diff --git a/src/runtime/latent-preview.h b/src/runtime/latent-preview.h
index 5baf6f5..419382d 100644
--- a/src/runtime/latent-preview.h
+++ b/src/runtime/latent-preview.h
@@ -71,7 +71,7 @@ const float qwen21_latent_rgb_proj[64][3] = {
{-0.01036656294f, -0.00422146068f, -0.0213499052f},
{0.01788952706f, 0.01191944190f, 0.0397205238f},
};
-float qwen21_latent_rgb_bias[3] = {-0.043293118f, -0.02695978f, -0.11986706f};
+inline float qwen21_latent_rgb_bias[3] = {-0.043293118f, -0.02695978f, -0.11986706f};
const float qwen21_latent_alpha_proj[64] = {
-0.0416241114f, -0.00678954612f, -0.0169095515f, -0.0230551401f, 0.0100882595f, 0.00655586802f, 0.0401166874f, -0.0055510216f,
@@ -109,7 +109,7 @@ const float minimax_latent_rgb_proj[24][3] = {
{0.01495088f, 0.00292306f, 0.00416671f},
{0.00495307f, 0.05066542f, 0.05210543f},
{-0.02154842f, -0.01518524f, 0.00442402f}};
-float minimax_latent_rgb_bias[3] = {0.07776964f, -0.01580954f, -0.06561434f};
+inline float minimax_latent_rgb_bias[3] = {0.07776964f, -0.01580954f, -0.06561434f};
const float ltxav_latent_rgb_proj[128][3] = {
{-0.0293802f, -0.0362516f, -0.0291386f},
@@ -241,7 +241,7 @@ const float ltxav_latent_rgb_proj[128][3] = {
{0.00695946f, 0.00313109f, -0.00379435f},
{0.0188377f, 0.0144059f, 0.0229724f},
};
-float ltxav_latent_rgb_bias[3] = {0.043849f, 0.0201085f, 0.0150286f};
+inline float ltxav_latent_rgb_bias[3] = {0.043849f, 0.0201085f, 0.0150286f};
const float wan_21_latent_rgb_proj[16][3] = {
{0.015123f, -0.148418f, 0.479828f},
@@ -260,7 +260,7 @@ const float wan_21_latent_rgb_proj[16][3] = {
{0.014307f, -0.048647f, -0.007219f},
{0.217150f, 0.053074f, 0.319923f},
{0.155357f, 0.083156f, 0.064780f}};
-float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
+inline float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
const float wan_22_latent_rgb_proj[48][3] = {
{0.017126f, -0.027230f, -0.019257f},
@@ -311,7 +311,7 @@ const float wan_22_latent_rgb_proj[48][3] = {
{0.009424f, 0.013088f, -0.001714f},
{-0.024805f, -0.059378f, -0.033756f},
{-0.078293f, 0.029070f, 0.026129f}};
-float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
+inline float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
const float flux_latent_rgb_proj[16][3] = {
{-0.041168f, 0.019917f, 0.097253f},
@@ -330,7 +330,7 @@ const float flux_latent_rgb_proj[16][3] = {
{-0.143501f, 0.018263f, -0.136138f},
{-0.025767f, -0.082035f, -0.040023f},
{-0.111849f, -0.055589f, -0.032361f}};
-float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
+inline float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
const float flux2_latent_rgb_proj[32][3] = {
{0.000736f, -0.008385f, -0.019710f},
@@ -365,7 +365,7 @@ const float flux2_latent_rgb_proj[32][3] = {
{-0.034560f, -0.008740f, 0.012996f},
{0.000166f, 0.001079f, -0.012153f},
{0.017772f, 0.000937f, -0.011953f}};
-float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
+inline float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
// This one was taken straight from
// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303
@@ -388,23 +388,23 @@ const float sd3_latent_rgb_proj[16][3] = {
{-0.0749f, -0.0634f, -0.0456f},
{-0.1418f, -0.1457f, -0.1259f},
};
-float sd3_latent_rgb_bias[3] = {0, 0, 0};
+inline float sd3_latent_rgb_bias[3] = {0, 0, 0};
const float sdxl_latent_rgb_proj[4][3] = {
{0.258303f, 0.277640f, 0.329699f},
{-0.299701f, 0.105446f, 0.014194f},
{0.050522f, 0.186163f, -0.143257f},
{-0.211938f, -0.149892f, -0.080036f}};
-float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
+inline float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
const float sd_latent_rgb_proj[4][3] = {
{0.337366f, 0.216344f, 0.257386f},
{0.165636f, 0.386828f, 0.046994f},
{-0.267803f, 0.237036f, 0.223517f},
{-0.178022f, -0.200862f, -0.678514f}};
-float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
+inline float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
-void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
+inline void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
size_t buffer_head = 0;
uint32_t latent_width = static_cast<uint32_t>(latents->ne[0]);
@@ -486,7 +486,7 @@ static inline bool preview_latent_tensor_is_video(const sd::Tensor<float>& laten
return latents.dim() == 5;
}
-void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
+inline void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
uint32_t latent_width = static_cast<uint32_t>(latents.shape()[0]);
uint32_t latent_height = static_cast<uint32_t>(latents.shape()[1]);
bool is_video = preview_latent_tensor_is_video(latents);
diff --git a/src/runtime/sample-cache.cpp b/src/runtime/sample-cache.cpp
index c8ea2c1..f4998b2 100644
--- a/src/runtime/sample-cache.cpp
+++ b/src/runtime/sample-cache.cpp
@@ -15,15 +15,15 @@ namespace sd_sample {
return std::max(0.0f, reuse_threshold);
}
- bool SampleCacheRuntime::easycache_enabled() const {
+ bool ConditionCacheRuntime::easycache_enabled() const {
return mode == SampleCacheMode::EASYCACHE;
}
- bool SampleCacheRuntime::ucache_enabled() const {
+ bool ConditionCacheRuntime::ucache_enabled() const {
return mode == SampleCacheMode::UCACHE;
}
- bool SampleCacheRuntime::cachedit_enabled() const {
+ bool ConditionCacheRuntime::cachedit_enabled() const {
return mode == SampleCacheMode::CACHEDIT;
}
@@ -39,7 +39,7 @@ namespace sd_sample {
cache_params.start_percent < cache_params.end_percent;
}
- static void init_easycache_runtime(SampleCacheRuntime& runtime,
+ static void init_easycache_runtime(ConditionCacheRuntime& runtime,
SDVersion version,
const sd::CacheConfig& cache_params,
Denoiser* denoiser) {
@@ -67,7 +67,7 @@ namespace sd_sample {
config.end_percent);
}
- static void init_ucache_runtime(SampleCacheRuntime& runtime,
+ static void init_ucache_runtime(ConditionCacheRuntime& runtime,
SDVersion version,
const sd::CacheConfig& cache_params,
Denoiser* denoiser,
@@ -103,7 +103,7 @@ namespace sd_sample {
config.reset_error_on_compute ? "true" : "false");
}
- static void init_cachedit_runtime(SampleCacheRuntime& runtime,
+ static void init_cachedit_runtime(ConditionCacheRuntime& runtime,
SDVersion version,
const sd::CacheConfig& cache_params,
const std::vector<float>& sigmas) {
@@ -114,8 +114,6 @@ namespace sd_sample {
DBCacheConfig dbcfg;
dbcfg.enabled = (cache_params.mode == SD_CACHE_DBCACHE || cache_params.mode == SD_CACHE_CACHE_DIT);
- dbcfg.Fn_compute_blocks = cache_params.Fn_compute_blocks;
- dbcfg.Bn_compute_blocks = cache_params.Bn_compute_blocks;
dbcfg.residual_diff_threshold = cache_params.residual_diff_threshold;
dbcfg.max_warmup_steps = cache_params.max_warmup_steps;
dbcfg.max_cached_steps = cache_params.max_cached_steps;
@@ -123,7 +121,6 @@ namespace sd_sample {
if (cache_params.scm_mask != nullptr && strlen(cache_params.scm_mask) > 0) {
dbcfg.steps_computation_mask = parse_scm_mask(cache_params.scm_mask);
}
- dbcfg.scm_policy_dynamic = cache_params.scm_policy_dynamic;
TaylorSeerConfig tcfg;
tcfg.enabled = (cache_params.mode == SD_CACHE_TAYLORSEER || cache_params.mode == SD_CACHE_CACHE_DIT);
@@ -138,10 +135,8 @@ namespace sd_sample {
runtime.cachedit.set_sigmas(sigmas);
runtime.mode = SampleCacheMode::CACHEDIT;
- LOG_INFO("CacheDIT enabled - mode: %s, Fn: %d, Bn: %d, threshold: %.3f, warmup: %d",
+ LOG_INFO("Condition cache enabled - mode: %s, threshold: %.3f, warmup: %d",
cache_params.mode == SD_CACHE_CACHE_DIT ? "DBCache+TaylorSeer" : (cache_params.mode == SD_CACHE_DBCACHE ? "DBCache" : "TaylorSeer"),
- dbcfg.Fn_compute_blocks,
- dbcfg.Bn_compute_blocks,
dbcfg.residual_diff_threshold,
dbcfg.max_warmup_steps);
}
@@ -155,6 +150,18 @@ namespace sd_sample {
return;
}
+ auto config = spectrum_config(cache_params);
+ size_t total_steps = sigmas.size() > 0 ? sigmas.size() - 1 : 0;
+ runtime.spectrum.init(config, total_steps);
+ runtime.spectrum_enabled = true;
+
+ LOG_INFO("Spectrum enabled - w: %.2f, m: %d, lam: %.2f, window: %d, flex: %.2f, warmup: %d, stop: %.0f%%",
+ config.w, config.m, config.lam,
+ config.window_size, config.flex_window,
+ config.warmup_steps, config.stop_percent * 100.0f);
+ }
+
+ SpectrumConfig spectrum_config(const sd::CacheConfig& cache_params) {
SpectrumConfig config;
config.w = cache_params.spectrum_w;
config.m = cache_params.spectrum_m;
@@ -164,21 +171,14 @@ namespace sd_sample {
config.warmup_steps = cache_params.spectrum_warmup_steps;
config.stop_percent = cache_params.spectrum_stop_percent;
- size_t total_steps = sigmas.size() > 0 ? sigmas.size() - 1 : 0;
- runtime.spectrum.init(config, total_steps);
- runtime.spectrum_enabled = true;
-
- LOG_INFO("Spectrum enabled - w: %.2f, m: %d, lam: %.2f, window: %d, flex: %.2f, warmup: %d, stop: %.0f%%",
- config.w, config.m, config.lam,
- config.window_size, config.flex_window,
- config.warmup_steps, config.stop_percent * 100.0f);
+ return config;
}
- SampleCacheRuntime init_sample_cache_runtime(SDVersion version,
- const sd::CacheConfig* cache_params,
- Denoiser* denoiser,
- const std::vector<float>& sigmas) {
- SampleCacheRuntime runtime;
+ ConditionCacheRuntime init_condition_cache_runtime(SDVersion version,
+ const sd::CacheConfig* cache_params,
+ Denoiser* denoiser,
+ const std::vector<float>& sigmas) {
+ ConditionCacheRuntime runtime;
if (cache_params == nullptr || cache_params->mode == SD_CACHE_DISABLED) {
return runtime;
}
@@ -202,9 +202,6 @@ namespace sd_sample {
case SD_CACHE_CACHE_DIT:
init_cachedit_runtime(runtime, version, *cache_params, sigmas);
break;
- case SD_CACHE_SPECTRUM:
- init_spectrum_runtime(runtime, version, *cache_params, sigmas);
- break;
default:
break;
}
@@ -212,7 +209,15 @@ namespace sd_sample {
return runtime;
}
- SampleStepCacheDispatcher::SampleStepCacheDispatcher(SampleCacheRuntime& runtime, int step, float sigma)
+ SampleCacheRuntime init_sample_cache_runtime(SDVersion version, const sd::CacheConfig* params, Denoiser* denoiser, const std::vector<float>& sigmas) {
+ SampleCacheRuntime runtime;
+ static_cast<ConditionCacheRuntime&>(runtime) = init_condition_cache_runtime(version, params, denoiser, sigmas);
+ if (params && params->mode == SD_CACHE_SPECTRUM)
+ init_spectrum_runtime(runtime, version, *params, sigmas);
+ return runtime;
+ }
+
+ SampleStepCacheDispatcher::SampleStepCacheDispatcher(ConditionCacheRuntime& runtime, int step, float sigma)
: runtime(runtime), step(step), sigma(sigma), step_index(step > 0 ? (step - 1) : -1) {
if (step_index < 0) {
return;
@@ -277,19 +282,45 @@ namespace sd_sample {
}
void SampleStepCacheDispatcher::invalidate_condition(const void* condition) {
- if (condition == nullptr) {
+ runtime.invalidate_condition(condition);
+ }
+
+ void ConditionCacheRuntime::invalidate_condition(const void* condition) {
+ if (!condition)
return;
+ switch (mode) {
+ case SampleCacheMode::EASYCACHE:
+ easycache.invalidate_condition(condition);
+ break;
+ case SampleCacheMode::UCACHE:
+ ucache.invalidate_condition(condition);
+ break;
+ case SampleCacheMode::CACHEDIT:
+ cachedit.invalidate_condition(condition);
+ break;
+ case SampleCacheMode::NONE:
+ break;
}
+ }
- switch (runtime.mode) {
+ void ConditionCacheRuntime::retain_conditions(const std::vector<const void*>& conditions) {
+ const auto retain = [&](auto& state) {
+ std::vector<const void*> retired;
+ for (const auto& [condition, entry] : state.cache_diffs)
+ if (std::find(conditions.begin(), conditions.end(), condition) == conditions.end())
+ retired.push_back(condition);
+ for (auto* condition : retired)
+ state.invalidate_condition(condition);
+ };
+ switch (mode) {
case SampleCacheMode::EASYCACHE:
- runtime.easycache.cache_diffs.erase(condition);
+ retain(easycache);
break;
case SampleCacheMode::UCACHE:
- runtime.ucache.cache_diffs.erase(condition);
+ retain(ucache);
break;
case SampleCacheMode::CACHEDIT:
- runtime.cachedit.cache_diffs.erase(condition);
+ retain(cachedit);
break;
case SampleCacheMode::NONE:
break;
@@ -311,7 +342,7 @@ namespace sd_sample {
return false;
}
- void log_sample_cache_summary(const SampleCacheRuntime& runtime, size_t total_steps) {
+ void log_condition_cache_summary(const ConditionCacheRuntime& runtime, size_t total_steps) {
if (runtime.easycache_enabled()) {
if (runtime.easycache.total_steps_skipped > 0 && total_steps > 0) {
if (runtime.easycache.total_steps_skipped < static_cast<int>(total_steps)) {
@@ -368,7 +399,10 @@ namespace sd_sample {
LOG_INFO("CacheDIT completed without skipping steps");
}
}
+ }
+ void log_sample_cache_summary(const SampleCacheRuntime& runtime, size_t total_steps) {
+ log_condition_cache_summary(runtime, total_steps);
if (runtime.spectrum_enabled && runtime.spectrum.total_steps_skipped > 0 && total_steps > 0) {
double speedup = static_cast<double>(total_steps) /
static_cast<double>(total_steps - runtime.spectrum.total_steps_skipped);
diff --git a/src/runtime/sample-cache.h b/src/runtime/sample-cache.h
index fd22edc..327300c 100644
--- a/src/runtime/sample-cache.h
+++ b/src/runtime/sample-cache.h
@@ -23,28 +23,32 @@ namespace sd_sample {
CACHEDIT,
};
- struct SampleCacheRuntime {
+ struct ConditionCacheRuntime {
SampleCacheMode mode = SampleCacheMode::NONE;
EasyCacheState easycache;
UCacheState ucache;
CacheDitConditionState cachedit;
- SpectrumState spectrum;
-
- bool spectrum_enabled = false;
-
bool easycache_enabled() const;
bool ucache_enabled() const;
bool cachedit_enabled() const;
+ bool enabled() const { return mode != SampleCacheMode::NONE; }
+ void retain_conditions(const std::vector<const void*>& conditions);
+ void invalidate_condition(const void* condition);
+ };
+
+ struct SampleCacheRuntime : ConditionCacheRuntime {
+ SpectrumState spectrum;
+ bool spectrum_enabled = false;
};
struct SampleStepCacheDispatcher {
- SampleCacheRuntime& runtime;
+ ConditionCacheRuntime& runtime;
int step;
float sigma;
int step_index;
- SampleStepCacheDispatcher(SampleCacheRuntime& runtime, int step, float sigma);
+ SampleStepCacheDispatcher(ConditionCacheRuntime& runtime, int step, float sigma);
bool before_condition(const void* condition, const sd::Tensor<float>& input, sd::Tensor<float>* output);
void after_condition(const void* condition, const sd::Tensor<float>& input, const sd::Tensor<float>& output);
@@ -52,6 +56,10 @@ namespace sd_sample {
bool is_step_skipped() const;
};
+ ConditionCacheRuntime init_condition_cache_runtime(SDVersion version, const sd::CacheConfig* params, Denoiser* denoiser, const std::vector<float>& sigmas);
+ SpectrumConfig spectrum_config(const sd::CacheConfig& params);
+ void log_condition_cache_summary(const ConditionCacheRuntime& runtime, size_t total_steps);
+
SampleCacheRuntime init_sample_cache_runtime(SDVersion version,
const sd::CacheConfig* cache_params,
Denoiser* denoiser,
diff --git a/src/runtime/spectrum.hpp b/src/runtime/spectrum.hpp
index e30f693..e7f7b93 100644
--- a/src/runtime/spectrum.hpp
+++ b/src/runtime/spectrum.hpp
@@ -46,10 +46,19 @@ struct SpectrumState {
return (step_cnt / 50.0f) * 2.0f - 1.0f;
}
- bool should_predict() {
- if (cnt < config.warmup_steps)
+ void reset_history() {
+ H_buf.clear();
+ T_buf.clear();
+ num_cached = 0;
+ curr_ws = float(config.window_size);
+ }
+
+ bool should_predict() const { return should_predict(cnt, stop_step > 0); }
+
+ bool should_predict(int logical_step, bool stop_enabled = true) const {
+ if (logical_step < config.warmup_steps)
return false;
- if (stop_step > 0 && cnt >= stop_step)
+ if (stop_enabled && logical_step >= stop_step)
return false;
if ((int)H_buf.size() < 2)
return false;
@@ -58,28 +67,32 @@ struct SpectrumState {
return (num_cached + 1) % ws != 0;
}
- void update(const sd::Tensor<float>& denoised) {
+ void update(const sd::Tensor<float>& denoised) { update(denoised, cnt); }
+
+ void update(const sd::Tensor<float>& denoised, int logical_step) {
H_buf.emplace_back(denoised.data(), denoised.data() + denoised.numel());
- T_buf.push_back(taus(cnt));
+ T_buf.push_back(taus(logical_step));
while ((int)H_buf.size() > K) {
H_buf.erase(H_buf.begin());
T_buf.erase(T_buf.begin());
}
- if (cnt >= config.warmup_steps)
+ if (logical_step >= config.warmup_steps)
curr_ws += config.flex_window;
num_cached = 0;
- cnt++;
+ cnt = logical_step + 1;
}
- void predict(sd::Tensor<float>* denoised) {
+ void predict(sd::Tensor<float>* denoised) { predict(denoised, cnt); }
+
+ void predict(sd::Tensor<float>* denoised, int logical_step) {
GGML_ASSERT(denoised != nullptr);
int64_t F = (int64_t)H_buf[0].size();
int K_curr = (int)H_buf.size();
int M1 = config.m + 1;
- float tau_at = taus(cnt);
+ float tau_at = taus(logical_step);
std::vector<float> X(K_curr * M1);
for (int i = 0; i < K_curr; i++) {
@@ -143,7 +156,7 @@ struct SpectrumState {
num_cached++;
total_steps_skipped++;
- cnt++;
+ cnt = logical_step + 1;
}
private:
diff --git a/src/runtime/ucache.hpp b/src/runtime/ucache.hpp
index 7f3eac8..1c7d371 100644
--- a/src/runtime/ucache.hpp
+++ b/src/runtime/ucache.hpp
@@ -125,6 +125,31 @@ struct UCacheState {
total_active_steps = 0;
}
+ void invalidate_condition(const void* condition) {
+ cache_diffs.erase(condition);
+ if (anchor_condition != condition)
+ return;
+ anchor_condition = nullptr;
+ initial_step = true;
+ skip_current_step = false;
+ prev_input.clear();
+ prev_output.clear();
+ output_prev_norm = 0.0f;
+ has_prev_input = false;
+ has_prev_output = false;
+ has_output_prev_norm = false;
+ has_relative_transformation_rate = false;
+ relative_transformation_rate = 0.0f;
+ last_input_change = 0.0f;
+ has_last_input_change = false;
+ output_change_ema = 0.0f;
+ has_output_change_ema = false;
+ consecutive_skipped_steps = 0;
+ accumulated_error = 0.f;
+ steps_computed_since_active = 0;
+ block_metrics.reset();
+ }
+
void init(const UCacheConfig& cfg, Denoiser* d) {
config = cfg;
denoiser = d;
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
deleted file mode 100644
index acc3672..0000000
--- a/src/stable-diffusion.cpp
+++ /dev/null
@@ -1,135 +0,0 @@
-#include "api/api.h"
-#include "api/options.h"
-#include "api/result.h"
-#include "pipeline/diffusion_engine.h"
-#include "pipeline/generation.h"
-
-#include <cmath>
-#include <cstdlib>
-#include <memory>
-#include <vector>
-
-static sd_status_t validate_generation(StableDiffusionGGML* engine, const sd::CacheConfig& cache, bool video) {
- if (video ? !sd_version_supports_video_generation(engine->version) &&
- !(engine->config_->animatediff_loaded && sd_version_supports_animatediff(engine->version))
- : !sd_version_supports_image_generation(engine->version))
- return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "model does not support this operation");
- if (cache.mode == SD_CACHE_UCACHE && !sd_version_is_unet(engine->version))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "ucache requires a UNet model");
- if ((cache.mode == SD_CACHE_EASYCACHE || cache.mode == SD_CACHE_DBCACHE ||
- cache.mode == SD_CACHE_TAYLORSEER || cache.mode == SD_CACHE_CACHE_DIT) &&
- !sd_version_is_dit(engine->version))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "selected cache requires a DiT model");
- if (cache.mode == SD_CACHE_SPECTRUM && !sd_version_is_unet(engine->version) && !sd_version_is_dit(engine->version))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "spectrum requires a UNet or DiT model");
- return SD_STATUS_OK;
-}
-
-static sd_status_t generation_failure(StableDiffusionGGML* engine) {
- if (engine->get_cancel_flag() == SD_CANCEL_ALL)
- return sd::api::error(SD_STATUS_CANCELLED, nullptr, "generation cancelled");
- return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "generation failed; see log for details");
-}
-
-static sd_status_t run_image(sd_ctx_t* ctx, const sd::ImageConfig* params, sd::ImageGenerationResult** result_out, const sd_opt_t* keys = nullptr) {
- if (result_out)
- *result_out = nullptr;
- if (!result_out || !ctx || !ctx->sd || !params)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required");
- StableDiffusionGGML::ExecutionScope execution(*ctx->sd);
- if (!execution.ready)
- return execution.operation.acquired
- ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources")
- : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing");
- auto status = validate_generation(ctx->sd, params->cache, false);
- if (status != SD_STATUS_OK)
- return status;
- if (sd::api::is_set(keys, "qwen_image_layers") && ctx->sd->version != VERSION_QWEN_IMAGE_LAYERED)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "qwen_image_layers", "requires Qwen Image Layered");
- const bool native_control = sd_version_is_control(ctx->sd->version) || ctx->sd->version == VERSION_FLEX_2 ||
- (sd_version_is_z_image(ctx->sd->version) && ctx->sd->model_manager &&
- ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0);
- if (params->control_image.data && !ctx->sd->control_net && !native_control)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "control_image", "model has no active control-image component");
- if (params->ip_adapter_image.data && !ctx->sd->ip_adapter)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "ip_adapter_image", "no IP-Adapter is loaded");
- if (params->pm_params.id_images_count > 0 && !ctx->sd->config_->photomaker_source_available)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "photomaker.id_images", "no PhotoMaker component is loaded");
- if (params->pulid_params.id_embedding_path && *params->pulid_params.id_embedding_path &&
- (ctx->sd->version != VERSION_FLUX || !ctx->sd->config_->params.pulid_weights_path ||
- !*ctx->sd->config_->params.pulid_weights_path))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "pulid.id_embedding_path", "requires a Flux model with PuLID weights");
- if (params->sample_params.guidance.img_cfg != INFINITY && params->sample_params.guidance.img_cfg != 1.f &&
- !sd_version_is_inpaint_or_unet_edit(ctx->sd->version) &&
- !(params->ref_images_count > 0 && sd_version_supports_ref_latent_img_cfg(ctx->sd->version)))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "sample.guidance.img_cfg", "model does not support image CFG for these inputs");
- sd::Image* images = nullptr;
- int count = 0;
- std::vector<nlohmann::json> infos;
- try {
- if (!sd::pipeline::generate_image(ctx->sd, params, &images, &count, &infos)) {
- sd::free_images(images, count);
- return generation_failure(ctx->sd);
- }
- } catch (...) {
- sd::free_images(images, count);
- throw;
- }
- return sd::make_image_generation_result(images, count, infos, result_out) ? SD_STATUS_OK
- : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate image result");
-}
-
-static sd_status_t run_video(sd_ctx_t* ctx, const sd::VideoConfig* params, sd::VideoGenerationResult** result_out, const sd_opt_t* keys = nullptr) {
- if (result_out)
- *result_out = nullptr;
- if (!result_out || !ctx || !ctx->sd || !params)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required");
- StableDiffusionGGML::ExecutionScope execution(*ctx->sd);
- if (!execution.ready)
- return execution.operation.acquired
- ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources")
- : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing");
- auto status = validate_generation(ctx->sd, params->cache, true);
- if (status != SD_STATUS_OK)
- return status;
- const bool vace = ctx->sd->diffusion_model &&
- (ctx->sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
- ctx->sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
- if (params->control_frames_size > 0 && !vace)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "control_frames", "requires a VACE model");
- if (params->ref_videos_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version))
- return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_videos", "requires a MiniMax-H3 model");
- if (params->ref_audios_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version) && ctx->sd->version != VERSION_WAN2_2_S2V)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_audios", "requires MiniMax-H3 or Wan2.2 S2V");
- if (ctx->sd->version == VERSION_WAN2_2_S2V) {
- if (params->end_image.data)
- return sd::api::error(SD_STATUS_UNSUPPORTED, "end_image", "Wan2.2 S2V does not support an end image");
- if (params->ref_audios_count > 1 || (params->ref_audios_count && !ctx->sd->audio_encoder))
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "ref_audios", "Wan2.2 S2V requires one driving track and an audio encoder");
- }
- auto result = std::unique_ptr<sd::VideoGenerationResult, decltype(&sd::free_video_generation_result)>(
- static_cast<sd::VideoGenerationResult*>(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result);
- if (!result)
- return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video result");
- nlohmann::json info;
- if (!sd::pipeline::generate_video(ctx->sd, params, &result->frames, &result->frame_count,
- &result->audio, &result->fps, &info))
- return generation_failure(ctx->sd);
- result->generation_info = sd::copy_generation_info(info);
- if (!result->generation_info)
- return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video metadata");
- *result_out = result.release();
- return SD_STATUS_OK;
-}
-
-sd_status_t sd::generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source) noexcept {
- if (result_out)
- *result_out = nullptr;
- return sd::api::call([&] { return run_image(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED);
-}
-
-sd_status_t sd::generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source) noexcept {
- if (result_out)
- *result_out = nullptr;
- return sd::api::call([&] { return run_video(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED);
-}
diff --git a/src/tokenizers/tokenizer_config.cpp b/src/tokenizers/tokenizer_config.cpp
index 98650de..474e9a7 100644
--- a/src/tokenizers/tokenizer_config.cpp
+++ b/src/tokenizers/tokenizer_config.cpp
@@ -2,6 +2,7 @@
#include <stdexcept>
+#include "core/file_access.h"
#include "core/util.h"
#include "hf_tokenizer.h"
@@ -52,6 +53,7 @@ std::shared_ptr<Tokenizer> TokenizerConfig::create(Slot slot, int64_t embedding_
return nullptr;
}
try {
+ sd::require_file_access(paths_[slot], SD_DGL_FILE_READ_MODEL);
auto tokenizer = std::make_shared<HFTokenizer>(paths_[slot]);
tokenizer->set_padding(padding_id, pad_left);
tokenizer->validate_vocab_size(embedding_rows);
diff --git a/src/tokenizers/tokenizer_config.h b/src/tokenizers/tokenizer_config.h
index 3edc4a9..8e62e26 100644
--- a/src/tokenizers/tokenizer_config.h
+++ b/src/tokenizers/tokenizer_config.h
@@ -4,6 +4,7 @@
#include <array>
#include <memory>
#include <string>
+#include <utility>
#include "tokenizer.h"
@@ -20,6 +21,9 @@ private:
public:
TokenizerConfig() = default;
explicit TokenizerConfig(const char* config);
+ explicit TokenizerConfig(std::array<std::string, 3> paths)
+ : paths_(std::move(paths)) {}
+ const std::array<std::string, 3>& paths() const { return paths_; }
bool has(Slot slot) const;
std::shared_ptr<Tokenizer> create(Slot slot, int64_t embedding_rows, int padding_id, bool pad_left = false, bool clip = false) const;
void validate_usage() const;
diff --git a/src/upscaler.cpp b/src/upscaler.cpp
deleted file mode 100644
index 9a16c0a..0000000
--- a/src/upscaler.cpp
+++ /dev/null
@@ -1,183 +0,0 @@
-#include "upscaler.h"
-#include "api/options.h"
-#include "api/result.h"
-#include "core/ggml_extend_backend.h"
-#include "core/util.h"
-#include "generation_info.h"
-#include "model/upscaler/esrgan.hpp"
-#include "model/upscaler/mosr.hpp"
-#include "model_loader.h"
-#include "stable-diffusion.h"
-
-#include <cstdlib>
-#include <utility>
-
-UpscalerGGML::UpscalerGGML(int n_threads,
- bool direct,
- int tile_size,
- std::string backend_spec,
- std::string params_backend_spec)
- : n_threads(n_threads),
- tensor_executor(n_threads > 0 ? n_threads : sd_get_num_physical_cores()),
- direct(direct),
- tile_size(tile_size),
- backend_spec(std::move(backend_spec)),
- params_backend_spec(std::move(params_backend_spec)) {}
-
-UpscalerGGML::~UpscalerGGML() = default;
-
-void UpscalerGGML::set_max_graph_vram_bytes(size_t max_vram_bytes) {
- max_graph_vram_bytes = max_vram_bytes;
- if (model != nullptr) {
- model->runner()->set_max_graph_vram_bytes(max_vram_bytes);
- }
-}
-
-int UpscalerGGML::get_scale() {
- return model != nullptr ? model->scale() : 1;
-}
-
-bool UpscalerGGML::load_from_file(const std::string& path, int n_threads) {
- sd::ParallelScope tensor_scope(&tensor_executor);
- ggml_log_set(sd_ggml_log_callback, nullptr);
- model.reset();
- std::string error;
- if (!backend_manager.init(backend_spec.c_str(), params_backend_spec.c_str(), nullptr, &error)) {
- LOG_ERROR("upscaler backend config failed: %s", error.c_str());
- return false;
- }
- model_manager = std::make_shared<ModelManager>();
- model_manager->set_n_threads(n_threads);
- model_manager->set_enable_mmap(false);
- model = sd::pipeline::load_image_upscale_model(model_manager, backend_manager, path,
- max_graph_vram_bytes, direct, model_data_type);
- return model != nullptr;
-}
-
-sd::Tensor<float> UpscalerGGML::upscale_tensor(const sd::Tensor<float>& input_tensor) {
- sd::ParallelScope tensor_scope(&tensor_executor);
- if (model == nullptr) {
- return {};
- }
- return sd::pipeline::upscale_image_tensor(*model, input_tensor, n_threads, tile_size);
-}
-
-sd::Image UpscalerGGML::upscale(sd::Image input_image) {
- sd::ParallelScope tensor_scope(&tensor_executor);
- sd::Image upscaled_image = {0, 0, 0, nullptr};
- const int scale = get_scale();
- int output_width = (int)input_image.width * scale;
- int output_height = (int)input_image.height * scale;
- LOG_INFO("upscaling from (%i x %i) to (%i x %i)",
- input_image.width, input_image.height, output_width, output_height);
-
- sd::Tensor<float> input_tensor = sd_image_to_tensor(input_image);
- sd::Tensor<float> upscaled;
- int64_t t0 = ggml_time_ms();
- upscaled = upscale_tensor(input_tensor);
- if (upscaled.empty()) {
- return upscaled_image;
- }
- sd::Image upscaled_data = tensor_to_sd_image(upscaled);
- int64_t t3 = ggml_time_ms();
- LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f);
- upscaled_image = upscaled_data;
- return upscaled_image;
-}
-
-struct sd_upscaler {
- std::unique_ptr<UpscalerGGML> upscaler;
-};
-
-sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::Upscaler, config);
- if (status != SD_STATUS_OK)
- return status;
- auto result = std::make_unique<sd_upscaler_t>();
- const auto& ctx = config.context;
- result->upscaler = std::make_unique<UpscalerGGML>(ctx.n_threads, config.post.conv_direct, config.post.tile_size,
- SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend));
- if (!result->upscaler->load_from_file(ctx.model_path, ctx.n_threads))
- return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load upscaler");
- *out = result.release();
- return SD_STATUS_OK;
- },
- SD_STATUS_LOAD_FAILED);
-}
-
-void sd_upscaler_free(sd_upscaler_t* upscaler) {
- delete upscaler;
-}
-
-sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out) {
- if (out)
- *out = nullptr;
- return sd::api::call([&] {
- if (!out)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
- sd::OptConfig config;
- auto status = sd::api::resolve(opt, sd::api::Upscale, config);
- if (status != SD_STATUS_OK)
- return status;
- if (!upscaler || !upscaler->upscaler)
- return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
- const auto& image = config.post.image;
- const auto scale = upscaler->upscaler->get_scale();
- if (scale <= 0 || image.width > uint32_t(INT32_MAX / scale) || image.height > uint32_t(INT32_MAX / scale))
- return sd::api::error(SD_STATUS_OUT_OF_RANGE, "image", "upscaled dimensions exceed the supported range");
- auto release = [](sd::Image* images) { sd::free_images(images, 1); };
- std::unique_ptr<sd::Image, decltype(release)> images(static_cast<sd::Image*>(calloc(1, sizeof(sd::Image))), release);
- if (!images)
- return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
- images.get()[0] = upscaler->upscaler->upscale(image);
- if (!images->data)
- return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "upscale failed");
- const std::vector<nlohmann::json> infos{sd::upscale_generation_info(scale)};
- return sd::make_image_result(images.release(), 1, infos, out) ? SD_STATUS_OK
- : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result metadata");
- },
- SD_STATUS_GENERATION_FAILED);
-}
-
-int sd_upscaler_get_scale(const sd_upscaler_t* upscaler) {
- sd::api::clear_error();
- if (!upscaler || !upscaler->upscaler) {
- sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
- return 0;
- }
- return upscaler->upscaler->get_scale();
-}
-
-int sd_upscaler_probe_scale(const char* model_path) {
- if (model_path == nullptr || model_path[0] == '\0') {
- return 0;
- }
- try {
- ModelLoader loader;
- if (!loader.init_from_file_and_convert_name(model_path, "", VERSION_ESRGAN)) {
- return 0;
- }
- const auto& tensors = loader.get_tensor_storage_map();
- if (tensors.count("gblocks.0.weight") > 0) {
- MoSRConfig config = MoSRConfig::detect_from_weights(tensors);
- return config.is_valid() ? config.scale : 0;
- }
- auto first = tensors.find("conv_first.weight");
- auto last = tensors.find("conv_last.weight");
- if (first == tensors.end() || last == tensors.end() ||
- tensors.count("body.0.rdb1.conv1.weight") == 0 ||
- first->second.n_dims != 4 || last->second.n_dims != 4 ||
- first->second.ne[2] != 3 || last->second.ne[3] != 3) {
- return 0;
- }
- return ESRGANConfig::detect_from_weights(tensors).scale;
- } catch (const std::exception&) {
- return 0;
- }
-}
diff --git a/src/upscaler.h b/src/upscaler.h
deleted file mode 100644
index e3cf09f..0000000
--- a/src/upscaler.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#ifndef __SD_UPSCALER_H__
-#define __SD_UPSCALER_H__
-
-#include "core/media.h"
-
-#include "core/ggml_extend_backend.h"
-#include "core/tensor.hpp"
-#include "model_manager.h"
-#include "pipeline/image_upscale.h"
-#include "stable-diffusion.h"
-
-#include <memory>
-#include <string>
-
-struct UpscalerGGML {
- SDBackendManager backend_manager;
- std::shared_ptr<ModelManager> model_manager;
- ggml_type model_data_type = GGML_TYPE_F16;
- std::unique_ptr<sd::pipeline::ImageUpscaleModel> model;
- int n_threads;
- sd::ParallelExecutor tensor_executor;
- bool direct = false;
- int tile_size = 128;
- size_t max_graph_vram_bytes = 0;
- std::string backend_spec;
- std::string params_backend_spec;
-
- UpscalerGGML(int n_threads,
- bool direct = false,
- int tile_size = 128,
- std::string backend_spec = "",
- std::string params_backend_spec = "");
- ~UpscalerGGML();
-
- bool load_from_file(const std::string& esrgan_path,
- int n_threads);
- void set_max_graph_vram_bytes(size_t max_vram_bytes);
- int get_scale();
- sd::Tensor<float> upscale_tensor(const sd::Tensor<float>& input_tensor);
- sd::Image upscale(sd::Image input_image);
-};
-
-#endif // __SD_UPSCALER_H__