Commit 0d71f96 for stable-diffusion.cpp
commit 0d71f963af9c94941f4a67e8ea145935651dd756
Author: leejet <leejet714@gmail.com>
Date: Mon Nov 23 02:39:34 2026 +0800
fix: restore runtime LoRA tensor counts and unify automatic LoRA mode selection (#2400)
diff --git a/docs/dgl.md b/docs/dgl.md
index 17072e0..163824a 100644
--- a/docs/dgl.md
+++ b/docs/dgl.md
@@ -605,9 +605,9 @@ across both modes. Changing mode or strength can reuse the original weights
while they remain resident. Persistent contexts retain idle buffers subject
to RAM and VRAM pressure. Run-once contexts can release shared buffers after
their last use.
-Automatic mode defaults to runtime LoRA in persistent contexts. For run-once
-DiTs and text encoders, automatic mode also considers quantization and
-backend placement.
+In both persistent and run-once contexts, `auto` selects `immediately` for
+unquantized weights on the compute backend without row splitting. It selects
+`at_runtime` for quantized, offloaded, or row-split weights.
### Prompt encoding
diff --git a/docs/models/lora.md b/docs/models/lora.md
index f5fc1cf..6be0ba7 100644
--- a/docs/models/lora.md
+++ b/docs/models/lora.md
@@ -76,10 +76,10 @@ The example loads `../models/loras/marblesh.safetensors` or
There are two ways to apply LoRA: **immediately** and **at_runtime**. You can specify it using the `--lora-apply-mode` parameter.
-By default, the mode is selected automatically:
-
-* If the model weights contain any quantized parameters, the **at_runtime** mode is used;
-* Otherwise, the **immediately** mode is used.
+The default **auto** mode chooses **immediately** for unquantized weights on
+the compute backend without row splitting. It chooses **at_runtime** for
+quantized, offloaded, or row-split weights. For scripts, see
+[DGL LoRA combinations](../dgl.md#lora-combinations).
The **immediately** mode may have precision and compatibility issues with quantized parameters, but it usually offers faster inference speed and, in some cases, lower memory usage.
In contrast, the **at_runtime** mode provides better compatibility and higher precision, but inference may be slower and memory usage may be higher in some cases.
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 2e2aea0..9beaaa5 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -762,11 +762,9 @@ ArgOptions SDContextParams::get_options() {
on_prediction_arg},
{"",
"--lora-apply-mode",
- "the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. "
- "In auto mode, if the model weights contain any quantized parameters, the at_runtime mode will be used; otherwise, immediately will be used."
- "The immediately mode may have precision and compatibility issues with quantized parameters, "
- "but it usually offers faster inference speed and, in some cases, lower memory usage. "
- "The at_runtime mode, on the other hand, is exactly the opposite.",
+ "Select auto (default), immediately, or at_runtime for LoRA application. "
+ "Auto merges unquantized weights on the compute backend when row splitting is off. "
+ "It applies LoRA at runtime for quantized, offloaded, or row-split weights.",
on_lora_apply_mode_arg},
};
diff --git a/src/dgl/model_value.cpp b/src/dgl/model_value.cpp
index a57cd07..2f45e96 100644
--- a/src/dgl/model_value.cpp
+++ b/src/dgl/model_value.cpp
@@ -69,7 +69,7 @@ namespace sd::dgl {
return visit_model(base, [&](const auto& model) {
using T = typename std::decay_t<decltype(model)>::element_type;
auto variant = context.make_model<T>(base.model_source->key);
- if (!variant->init_variant(*model, loras, context.keep_resources))
+ if (!variant->init_variant(*model, loras))
throw std::runtime_error("cannot build LoRA variant of " + type_name(base.type()));
if constexpr (std::is_same_v<T, models::DiT>)
return Value::object(std::make_shared<models::DiffusionModel>(models::DiffusionModel{std::move(variant)}));
diff --git a/src/dgl/models/detail_detector.cpp b/src/dgl/models/detail_detector.cpp
index 3968f97..e6b7753 100644
--- a/src/dgl/models/detail_detector.cpp
+++ b/src/dgl/models/detail_detector.cpp
@@ -325,8 +325,9 @@ namespace sd::dgl::models {
}
bool DetailDetector::attach_loras() {
+ const auto mode = resolve_lora_mode(lora_mode(), model_weights->loader(), backend_manager, SDBackendModule::DETECTOR);
return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::DETECTOR,
- n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+ n_threads, mode == LORA_APPLY_IMMEDIATELY);
}
void DetailDetector::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
diff --git a/src/dgl/models/lora.cpp b/src/dgl/models/lora.cpp
index 9aa8320..5355095 100644
--- a/src/dgl/models/lora.cpp
+++ b/src/dgl/models/lora.cpp
@@ -1,5 +1,6 @@
#include "dgl/models/lora.h"
+#include <algorithm>
#include <string_view>
#include "core/ggml_extend_backend.h"
@@ -7,6 +8,25 @@
namespace sd::dgl::models {
+ lora_apply_mode_t resolve_lora_mode(lora_apply_mode_t mode, const ModelLoader& loader,
+ SDBackendManager& backends, SDBackendModule module, bool row_split) {
+ if (mode != LORA_APPLY_AUTO)
+ return mode;
+ // Offloading can signal limited compute memory, so avoid merged copies in auto mode.
+ if (row_split || backends.params_backend_is_disk(module) ||
+ backends.params_backend(module) != backends.runtime_backend(module))
+ return LORA_APPLY_AT_RUNTIME;
+
+ const auto& tensors = loader.get_tensor_storage_map();
+ const bool quantized = std::any_of(tensors.begin(), tensors.end(), [](const auto& entry) {
+ const auto& tensor = entry.second;
+ return tensor.is_int8_tensorwise || tensor.w4_convrot_kind ||
+ ggml_is_quantized(tensor.type) ||
+ (tensor.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor.expected_type));
+ });
+ return quantized ? LORA_APPLY_AT_RUNTIME : LORA_APPLY_IMMEDIATELY;
+ }
+
static bool is_patch_tensor(std::string_view suffix) {
const auto index = suffix.find('.');
if (index != std::string_view::npos && index != 0 &&
@@ -17,8 +37,14 @@ namespace sd::dgl::models {
}
void LoraExecution::end() {
- for (const auto& model : models_)
+ for (const auto& model : models_) {
+ if (adapter_ && (!model->applied_lora_tensors.empty() || !model->skipped_incompatible_lora_tensors.empty())) {
+ model->stat(true, sd_backend_module_name(module_));
+ model->applied_lora_tensors.clear();
+ model->skipped_incompatible_lora_tensors.clear();
+ }
model->runner_end();
+ }
}
void LoraExecution::reset() {
@@ -85,6 +111,7 @@ namespace sd::dgl::models {
adapter_ = std::make_shared<MultiLoraAdapter>(models);
if (!models.empty())
LOG_INFO("apply lora %s", immediately ? "immediately" : "at runtime");
+ module_ = module;
models_ = std::move(models);
ready_ = true;
return true;
diff --git a/src/dgl/models/lora.h b/src/dgl/models/lora.h
index 2a6ed2e..40b5a86 100644
--- a/src/dgl/models/lora.h
+++ b/src/dgl/models/lora.h
@@ -29,9 +29,13 @@ namespace sd::dgl::models {
lora_apply_mode_t mode = LORA_APPLY_AUTO;
};
+ lora_apply_mode_t resolve_lora_mode(lora_apply_mode_t mode, const ModelLoader& loader,
+ SDBackendManager& backends, SDBackendModule module, bool row_split = false);
+
class LoraExecution {
std::vector<std::shared_ptr<LoraModel>> models_;
std::shared_ptr<WeightAdapter> adapter_;
+ SDBackendModule module_{};
bool ready_ = false;
public:
diff --git a/src/dgl/models/model_base.cpp b/src/dgl/models/model_base.cpp
index e9e851b..4f5e486 100644
--- a/src/dgl/models/model_base.cpp
+++ b/src/dgl/models/model_base.cpp
@@ -102,11 +102,10 @@ namespace sd::dgl::models {
return !params_placement_ || params_placement_(name);
}
- bool Model::init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent) {
+ bool Model::init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras) {
if (typeid(*this) != typeid(base))
throw std::invalid_argument("LoRA variant must have the same model type as its base");
loras_ = std::move(loras);
- persistent_loras_ = persistent;
family_dependencies_ = base.family_dependencies_;
return build_variant(base, base.base_loader()) && apply_loras();
}
@@ -122,8 +121,7 @@ namespace sd::dgl::models {
}
lora_apply_mode_t Model::lora_mode() const {
- const auto mode = loras_ ? loras_->mode : LORA_APPLY_AUTO;
- return persistent_loras_ && mode == LORA_APPLY_AUTO ? LORA_APPLY_AT_RUNTIME : mode;
+ return loras_ ? loras_->mode : LORA_APPLY_AUTO;
}
bool Model::apply_loras() {
diff --git a/src/dgl/models/model_base.h b/src/dgl/models/model_base.h
index 0c98983..ea09485 100644
--- a/src/dgl/models/model_base.h
+++ b/src/dgl/models/model_base.h
@@ -40,7 +40,6 @@ namespace sd::dgl::models {
std::shared_ptr<const LoraConfig> loras_;
LoraExecution lora_execution_;
bool executing_ = false;
- bool persistent_loras_ = false;
bool weights_released_ = false;
bool eager_load_ = false;
ModelLoader::Dependencies family_dependencies_;
@@ -99,7 +98,7 @@ namespace sd::dgl::models {
bool finish_initialization();
bool place_params(const std::string& name, ggml_backend_t backend, bool disk);
void set_shared_weights(std::string key) { model_weights->set_shared_weights(std::move(key)); }
- bool init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent);
+ bool init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras);
ModelLoader::Dependencies dependencies() const;
};
diff --git a/src/dgl/models/model_runtime.cpp b/src/dgl/models/model_runtime.cpp
index 3bed540..1773085 100644
--- a/src/dgl/models/model_runtime.cpp
+++ b/src/dgl/models/model_runtime.cpp
@@ -409,45 +409,17 @@ namespace sd::dgl::models {
model_.model_weights->prepare_file_io();
if (!model_.has_loras())
return;
- const auto mode = model_.lora_mode();
- const auto& model_loader = model_.model_weights->loader();
- const auto wtype_stat = model_loader.get_wtype_stat();
- bool have_int8_tensorwise = false;
- bool have_quant_type_rule = false;
- for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) {
- if (tensor_storage.is_int8_tensorwise || tensor_storage.w4_convrot_kind) {
- have_int8_tensorwise = true;
- }
- if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) {
- have_quant_type_rule = true;
- }
- if (have_int8_tensorwise && have_quant_type_rule) {
- break;
- }
- }
-
- if (mode == LORA_APPLY_AUTO) {
- bool have_quantized_weight = have_int8_tensorwise;
- for (const auto& [type, _] : wtype_stat) {
- if (ggml_is_quantized(type)) {
- have_quantized_weight = true;
- break;
- }
- }
- // Avoid full-model LoRA merge buffers on constrained setups.
- const bool params_offloaded = params_backend_for() != backend_for();
- const bool streaming_constrained = params_offloaded ||
- backend_manager.params_backend_is_disk(module);
-
- if (have_quant_type_rule || have_quantized_weight || streaming_constrained || split_active(SDSplitMode::ROW)) {
- apply_lora_immediately_ = false;
- } else {
- apply_lora_immediately_ = true;
- }
- } else if (mode == LORA_APPLY_IMMEDIATELY) {
- apply_lora_immediately_ = !have_int8_tensorwise && !split_active(SDSplitMode::ROW);
+ const auto mode = model_.lora_mode();
+ const auto& loader = model_.model_weights->loader();
+ const bool row_split = mode != LORA_APPLY_AT_RUNTIME && split_active(SDSplitMode::ROW);
+ if (mode == LORA_APPLY_IMMEDIATELY) {
+ const auto& tensors = loader.get_tensor_storage_map();
+ apply_lora_immediately_ = !row_split &&
+ std::none_of(tensors.begin(), tensors.end(), [](const auto& entry) {
+ return entry.second.is_int8_tensorwise || entry.second.w4_convrot_kind;
+ });
} else {
- apply_lora_immediately_ = false;
+ apply_lora_immediately_ = resolve_lora_mode(mode, loader, backend_manager, module, row_split) == LORA_APPLY_IMMEDIATELY;
}
}
diff --git a/src/dgl/models/upscaler.cpp b/src/dgl/models/upscaler.cpp
index 39ba9a6..e8b11a6 100644
--- a/src/dgl/models/upscaler.cpp
+++ b/src/dgl/models/upscaler.cpp
@@ -77,8 +77,10 @@ namespace sd::dgl::models {
}
bool ImageUpscaler::attach_loras() {
+ const auto& loader = model_weights->model_source(model->source_file());
+ const auto mode = resolve_lora_mode(lora_mode(), loader, backend_manager, SDBackendModule::UPSCALER);
return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::UPSCALER,
- n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+ n_threads, mode == LORA_APPLY_IMMEDIATELY);
}
void ImageUpscaler::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index ddb2f24..2541f19 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -1068,7 +1068,7 @@ struct LoraModel : public GGMLRunner {
return apply(model_tensors, tensor_names(model_tensors), version, n_threads, warn_unused);
}
- void stat(bool at_runntime = false) {
+ void stat(bool at_runtime, const char* component = nullptr) {
size_t total_lora_tensors_count = 0;
size_t applied_lora_tensors_count = 0;
size_t skipped_lora_tensors_count = 0;
@@ -1078,7 +1078,7 @@ struct LoraModel : public GGMLRunner {
if (skipped_incompatible_lora_tensors.find(kv.first) != skipped_incompatible_lora_tensors.end()) {
skipped_lora_tensors_count++;
} else if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) {
- if (!at_runntime) {
+ if (!at_runtime) {
LOG_WARN("unused lora tensor |%s|", kv.first.c_str());
print_ggml_tensor(kv.second, true);
}
@@ -1086,20 +1086,18 @@ struct LoraModel : public GGMLRunner {
applied_lora_tensors_count++;
}
}
- /* Don't worry if this message shows up twice in the logs per LoRA,
- * this function is called once to calculate the required buffer size
- * and then again to actually generate a graph to be used */
+ const std::string prefix = component ? std::string(component) + ": " : "";
size_t compatible_lora_tensors_count = total_lora_tensors_count - skipped_lora_tensors_count;
- if (!at_runntime && applied_lora_tensors_count != compatible_lora_tensors_count) {
- LOG_WARN("Only (%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
+ if (!at_runtime && applied_lora_tensors_count != compatible_lora_tensors_count) {
+ LOG_WARN("Only (%zu / %zu) LoRA tensors have been applied, lora_file_path = %s",
applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
} else {
- LOG_INFO("(%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
- applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
+ LOG_INFO("%s(%zu / %zu) LoRA tensors have been applied, lora_file_path = %s",
+ prefix.c_str(), applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
}
if (skipped_lora_tensors_count > 0) {
- LOG_WARN("(%lu / %lu) incompatible LoRA tensors have been skipped, lora_file_path = %s",
- skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
+ LOG_WARN("%s(%zu / %zu) incompatible LoRA tensors have been skipped, lora_file_path = %s",
+ prefix.c_str(), skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
}
}
};