Commit 0d71f96 for stable-diffusion.cpp

commit 0d71f963af9c94941f4a67e8ea145935651dd756
Author: leejet <leejet714@gmail.com>
Date:   Mon Nov 23 02:39:34 2026 +0800

    fix: restore runtime LoRA tensor counts and unify automatic LoRA mode selection (#2400)

diff --git a/docs/dgl.md b/docs/dgl.md
index 17072e0..163824a 100644
--- a/docs/dgl.md
+++ b/docs/dgl.md
@@ -605,9 +605,9 @@ across both modes. Changing mode or strength can reuse the original weights
 while they remain resident. Persistent contexts retain idle buffers subject
 to RAM and VRAM pressure. Run-once contexts can release shared buffers after
 their last use.
-Automatic mode defaults to runtime LoRA in persistent contexts. For run-once
-DiTs and text encoders, automatic mode also considers quantization and
-backend placement.
+In both persistent and run-once contexts, `auto` selects `immediately` for
+unquantized weights on the compute backend without row splitting. It selects
+`at_runtime` for quantized, offloaded, or row-split weights.

 ### Prompt encoding

diff --git a/docs/models/lora.md b/docs/models/lora.md
index f5fc1cf..6be0ba7 100644
--- a/docs/models/lora.md
+++ b/docs/models/lora.md
@@ -76,10 +76,10 @@ The example loads `../models/loras/marblesh.safetensors` or

 There are two ways to apply LoRA: **immediately** and **at_runtime**. You can specify it using the `--lora-apply-mode` parameter.

-By default, the mode is selected automatically:
-
-* If the model weights contain any quantized parameters, the **at_runtime** mode is used;
-* Otherwise, the **immediately** mode is used.
+The default **auto** mode chooses **immediately** for unquantized weights on
+the compute backend without row splitting. It chooses **at_runtime** for
+quantized, offloaded, or row-split weights. For scripts, see
+[DGL LoRA combinations](../dgl.md#lora-combinations).

 The **immediately** mode may have precision and compatibility issues with quantized parameters, but it usually offers faster inference speed and, in some cases, lower memory usage.
 In contrast, the **at_runtime** mode provides better compatibility and higher precision, but inference may be slower and memory usage may be higher in some cases.
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 2e2aea0..9beaaa5 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -762,11 +762,9 @@ ArgOptions SDContextParams::get_options() {
          on_prediction_arg},
         {"",
          "--lora-apply-mode",
-         "the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. "
-         "In auto mode, if the model weights contain any quantized parameters, the at_runtime mode will be used; otherwise, immediately will be used."
-         "The immediately mode may have precision and compatibility issues with quantized parameters, "
-         "but it usually offers faster inference speed and, in some cases, lower memory usage. "
-         "The at_runtime mode, on the other hand, is exactly the opposite.",
+         "Select auto (default), immediately, or at_runtime for LoRA application. "
+         "Auto merges unquantized weights on the compute backend when row splitting is off. "
+         "It applies LoRA at runtime for quantized, offloaded, or row-split weights.",
          on_lora_apply_mode_arg},
     };

diff --git a/src/dgl/model_value.cpp b/src/dgl/model_value.cpp
index a57cd07..2f45e96 100644
--- a/src/dgl/model_value.cpp
+++ b/src/dgl/model_value.cpp
@@ -69,7 +69,7 @@ namespace sd::dgl {
         return visit_model(base, [&](const auto& model) {
             using T      = typename std::decay_t<decltype(model)>::element_type;
             auto variant = context.make_model<T>(base.model_source->key);
-            if (!variant->init_variant(*model, loras, context.keep_resources))
+            if (!variant->init_variant(*model, loras))
                 throw std::runtime_error("cannot build LoRA variant of " + type_name(base.type()));
             if constexpr (std::is_same_v<T, models::DiT>)
                 return Value::object(std::make_shared<models::DiffusionModel>(models::DiffusionModel{std::move(variant)}));
diff --git a/src/dgl/models/detail_detector.cpp b/src/dgl/models/detail_detector.cpp
index 3968f97..e6b7753 100644
--- a/src/dgl/models/detail_detector.cpp
+++ b/src/dgl/models/detail_detector.cpp
@@ -325,8 +325,9 @@ namespace sd::dgl::models {
     }

     bool DetailDetector::attach_loras() {
+        const auto mode = resolve_lora_mode(lora_mode(), model_weights->loader(), backend_manager, SDBackendModule::DETECTOR);
         return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::DETECTOR,
-                             n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+                             n_threads, mode == LORA_APPLY_IMMEDIATELY);
     }

     void DetailDetector::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
diff --git a/src/dgl/models/lora.cpp b/src/dgl/models/lora.cpp
index 9aa8320..5355095 100644
--- a/src/dgl/models/lora.cpp
+++ b/src/dgl/models/lora.cpp
@@ -1,5 +1,6 @@
 #include "dgl/models/lora.h"

+#include <algorithm>
 #include <string_view>

 #include "core/ggml_extend_backend.h"
@@ -7,6 +8,25 @@

 namespace sd::dgl::models {

+    lora_apply_mode_t resolve_lora_mode(lora_apply_mode_t mode, const ModelLoader& loader,
+                                       SDBackendManager& backends, SDBackendModule module, bool row_split) {
+        if (mode != LORA_APPLY_AUTO)
+            return mode;
+        // Offloading can signal limited compute memory, so avoid merged copies in auto mode.
+        if (row_split || backends.params_backend_is_disk(module) ||
+            backends.params_backend(module) != backends.runtime_backend(module))
+            return LORA_APPLY_AT_RUNTIME;
+
+        const auto& tensors  = loader.get_tensor_storage_map();
+        const bool quantized = std::any_of(tensors.begin(), tensors.end(), [](const auto& entry) {
+            const auto& tensor = entry.second;
+            return tensor.is_int8_tensorwise || tensor.w4_convrot_kind ||
+                   ggml_is_quantized(tensor.type) ||
+                   (tensor.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor.expected_type));
+        });
+        return quantized ? LORA_APPLY_AT_RUNTIME : LORA_APPLY_IMMEDIATELY;
+    }
+
     static bool is_patch_tensor(std::string_view suffix) {
         const auto index = suffix.find('.');
         if (index != std::string_view::npos && index != 0 &&
@@ -17,8 +37,14 @@ namespace sd::dgl::models {
     }

     void LoraExecution::end() {
-        for (const auto& model : models_)
+        for (const auto& model : models_) {
+            if (adapter_ && (!model->applied_lora_tensors.empty() || !model->skipped_incompatible_lora_tensors.empty())) {
+                model->stat(true, sd_backend_module_name(module_));
+                model->applied_lora_tensors.clear();
+                model->skipped_incompatible_lora_tensors.clear();
+            }
             model->runner_end();
+        }
     }

     void LoraExecution::reset() {
@@ -85,6 +111,7 @@ namespace sd::dgl::models {
             adapter_ = std::make_shared<MultiLoraAdapter>(models);
         if (!models.empty())
             LOG_INFO("apply lora %s", immediately ? "immediately" : "at runtime");
+        module_ = module;
         models_ = std::move(models);
         ready_  = true;
         return true;
diff --git a/src/dgl/models/lora.h b/src/dgl/models/lora.h
index 2a6ed2e..40b5a86 100644
--- a/src/dgl/models/lora.h
+++ b/src/dgl/models/lora.h
@@ -29,9 +29,13 @@ namespace sd::dgl::models {
         lora_apply_mode_t mode = LORA_APPLY_AUTO;
     };

+    lora_apply_mode_t resolve_lora_mode(lora_apply_mode_t mode, const ModelLoader& loader,
+                                       SDBackendManager& backends, SDBackendModule module, bool row_split = false);
+
     class LoraExecution {
         std::vector<std::shared_ptr<LoraModel>> models_;
         std::shared_ptr<WeightAdapter> adapter_;
+        SDBackendModule module_{};
         bool ready_ = false;

     public:
diff --git a/src/dgl/models/model_base.cpp b/src/dgl/models/model_base.cpp
index e9e851b..4f5e486 100644
--- a/src/dgl/models/model_base.cpp
+++ b/src/dgl/models/model_base.cpp
@@ -102,11 +102,10 @@ namespace sd::dgl::models {
         return !params_placement_ || params_placement_(name);
     }

-    bool Model::init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent) {
+    bool Model::init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras) {
         if (typeid(*this) != typeid(base))
             throw std::invalid_argument("LoRA variant must have the same model type as its base");
         loras_               = std::move(loras);
-        persistent_loras_    = persistent;
         family_dependencies_ = base.family_dependencies_;
         return build_variant(base, base.base_loader()) && apply_loras();
     }
@@ -122,8 +121,7 @@ namespace sd::dgl::models {
     }

     lora_apply_mode_t Model::lora_mode() const {
-        const auto mode = loras_ ? loras_->mode : LORA_APPLY_AUTO;
-        return persistent_loras_ && mode == LORA_APPLY_AUTO ? LORA_APPLY_AT_RUNTIME : mode;
+        return loras_ ? loras_->mode : LORA_APPLY_AUTO;
     }

     bool Model::apply_loras() {
diff --git a/src/dgl/models/model_base.h b/src/dgl/models/model_base.h
index 0c98983..ea09485 100644
--- a/src/dgl/models/model_base.h
+++ b/src/dgl/models/model_base.h
@@ -40,7 +40,6 @@ namespace sd::dgl::models {
         std::shared_ptr<const LoraConfig> loras_;
         LoraExecution lora_execution_;
         bool executing_        = false;
-        bool persistent_loras_ = false;
         bool weights_released_ = false;
         bool eager_load_       = false;
         ModelLoader::Dependencies family_dependencies_;
@@ -99,7 +98,7 @@ namespace sd::dgl::models {
         bool finish_initialization();
         bool place_params(const std::string& name, ggml_backend_t backend, bool disk);
         void set_shared_weights(std::string key) { model_weights->set_shared_weights(std::move(key)); }
-        bool init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras, bool persistent);
+        bool init_variant(const Model& base, std::shared_ptr<const LoraConfig> loras);
         ModelLoader::Dependencies dependencies() const;
     };

diff --git a/src/dgl/models/model_runtime.cpp b/src/dgl/models/model_runtime.cpp
index 3bed540..1773085 100644
--- a/src/dgl/models/model_runtime.cpp
+++ b/src/dgl/models/model_runtime.cpp
@@ -409,45 +409,17 @@ namespace sd::dgl::models {
         model_.model_weights->prepare_file_io();
         if (!model_.has_loras())
             return;
-        const auto mode           = model_.lora_mode();
-        const auto& model_loader  = model_.model_weights->loader();
-        const auto wtype_stat     = model_loader.get_wtype_stat();
-        bool have_int8_tensorwise = false;
-        bool have_quant_type_rule = false;
-        for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) {
-            if (tensor_storage.is_int8_tensorwise || tensor_storage.w4_convrot_kind) {
-                have_int8_tensorwise = true;
-            }
-            if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) {
-                have_quant_type_rule = true;
-            }
-            if (have_int8_tensorwise && have_quant_type_rule) {
-                break;
-            }
-        }
-
-        if (mode == LORA_APPLY_AUTO) {
-            bool have_quantized_weight = have_int8_tensorwise;
-            for (const auto& [type, _] : wtype_stat) {
-                if (ggml_is_quantized(type)) {
-                    have_quantized_weight = true;
-                    break;
-                }
-            }
-            // Avoid full-model LoRA merge buffers on constrained setups.
-            const bool params_offloaded      = params_backend_for() != backend_for();
-            const bool streaming_constrained = params_offloaded ||
-                                               backend_manager.params_backend_is_disk(module);
-
-            if (have_quant_type_rule || have_quantized_weight || streaming_constrained || split_active(SDSplitMode::ROW)) {
-                apply_lora_immediately_ = false;
-            } else {
-                apply_lora_immediately_ = true;
-            }
-        } else if (mode == LORA_APPLY_IMMEDIATELY) {
-            apply_lora_immediately_ = !have_int8_tensorwise && !split_active(SDSplitMode::ROW);
+        const auto mode      = model_.lora_mode();
+        const auto& loader   = model_.model_weights->loader();
+        const bool row_split = mode != LORA_APPLY_AT_RUNTIME && split_active(SDSplitMode::ROW);
+        if (mode == LORA_APPLY_IMMEDIATELY) {
+            const auto& tensors = loader.get_tensor_storage_map();
+            apply_lora_immediately_ = !row_split &&
+                                      std::none_of(tensors.begin(), tensors.end(), [](const auto& entry) {
+                                          return entry.second.is_int8_tensorwise || entry.second.w4_convrot_kind;
+                                      });
         } else {
-            apply_lora_immediately_ = false;
+            apply_lora_immediately_ = resolve_lora_mode(mode, loader, backend_manager, module, row_split) == LORA_APPLY_IMMEDIATELY;
         }
     }

diff --git a/src/dgl/models/upscaler.cpp b/src/dgl/models/upscaler.cpp
index 39ba9a6..e8b11a6 100644
--- a/src/dgl/models/upscaler.cpp
+++ b/src/dgl/models/upscaler.cpp
@@ -77,8 +77,10 @@ namespace sd::dgl::models {
     }

     bool ImageUpscaler::attach_loras() {
+        const auto& loader = model_weights->model_source(model->source_file());
+        const auto mode    = resolve_lora_mode(lora_mode(), loader, backend_manager, SDBackendModule::UPSCALER);
         return prepare_loras(VERSION_COUNT, backend_manager, SDBackendModule::UPSCALER,
-                             n_threads, lora_mode() == LORA_APPLY_IMMEDIATELY);
+                             n_threads, mode == LORA_APPLY_IMMEDIATELY);
     }

     void ImageUpscaler::set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index ddb2f24..2541f19 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -1068,7 +1068,7 @@ struct LoraModel : public GGMLRunner {
         return apply(model_tensors, tensor_names(model_tensors), version, n_threads, warn_unused);
     }

-    void stat(bool at_runntime = false) {
+    void stat(bool at_runtime, const char* component = nullptr) {
         size_t total_lora_tensors_count   = 0;
         size_t applied_lora_tensors_count = 0;
         size_t skipped_lora_tensors_count = 0;
@@ -1078,7 +1078,7 @@ struct LoraModel : public GGMLRunner {
             if (skipped_incompatible_lora_tensors.find(kv.first) != skipped_incompatible_lora_tensors.end()) {
                 skipped_lora_tensors_count++;
             } else if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) {
-                if (!at_runntime) {
+                if (!at_runtime) {
                     LOG_WARN("unused lora tensor |%s|", kv.first.c_str());
                     print_ggml_tensor(kv.second, true);
                 }
@@ -1086,20 +1086,18 @@ struct LoraModel : public GGMLRunner {
                 applied_lora_tensors_count++;
             }
         }
-        /* Don't worry if this message shows up twice in the logs per LoRA,
-         * this function is called once to calculate the required buffer size
-         * and then again to actually generate a graph to be used */
+        const std::string prefix             = component ? std::string(component) + ": " : "";
         size_t compatible_lora_tensors_count = total_lora_tensors_count - skipped_lora_tensors_count;
-        if (!at_runntime && applied_lora_tensors_count != compatible_lora_tensors_count) {
-            LOG_WARN("Only (%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
+        if (!at_runtime && applied_lora_tensors_count != compatible_lora_tensors_count) {
+            LOG_WARN("Only (%zu / %zu) LoRA tensors have been applied, lora_file_path = %s",
                      applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
         } else {
-            LOG_INFO("(%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
-                     applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
+            LOG_INFO("%s(%zu / %zu) LoRA tensors have been applied, lora_file_path = %s",
+                     prefix.c_str(), applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
         }
         if (skipped_lora_tensors_count > 0) {
-            LOG_WARN("(%lu / %lu) incompatible LoRA tensors have been skipped, lora_file_path = %s",
-                     skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
+            LOG_WARN("%s(%zu / %zu) incompatible LoRA tensors have been skipped, lora_file_path = %s",
+                     prefix.c_str(), skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
         }
     }
 };