Commit 412a363c6 for llama.cpp

commit 412a363c655b479279d3050d0d1c67a5733697b4
Author: fzowl <160063452+fzowl@users.noreply.github.com>
Date:   Mon Oct 19 08:29:02 2026 +0200

    qwen3 : bidirectional attention and dense_2 output projection (#30607)

    * qwen3 : bidirectional attention and dense_2 output projection

    Assisted-by: Grok

    * convert : read use_bidirectional_attention in base

    Assisted-by: Grok

    * Update conversion/base.py

    Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

    ---------

    Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

diff --git a/conversion/base.py b/conversion/base.py
index 72c1256ba..14a52f01d 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):
             self.gguf_writer.add_head_count_kv(n_head_kv)
             logger.info(f"gguf: key-value head count = {n_head_kv}")

-        if self.hparams.get("is_causal") is False:
+        if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:
             self.gguf_writer.add_causal_attention(False)
             logger.info("gguf: causal attention = False")

diff --git a/conversion/qwen.py b/conversion/qwen.py
index 43d2fb466..0c0d0c1ed 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):
                     yield from super().modify_tensors(data_torch, name, bid)
                 return

+        # output embedding projection (voyage-4-nano)
+        if name == "linear.weight":
+            self.gguf_writer.add_embedding_length_out(data_torch.shape[0])
+            name = "dense_2_out.weight"
+
         yield from super().modify_tensors(data_torch, name, bid)


diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 32e088d88..e098efae4 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_GATE,
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano
     ],
     MODEL_ARCH.QWEN3MOE: [
         MODEL_TENSOR.TOKEN_EMBD,
diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index bf7e622fb..c01f83829 100644
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {
     // output rerank head
     cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);

+    // output embedding projection
+    dense_2_out_layers   = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);
+    dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"),   {hparams.n_embd_out()        }, TENSOR_NOT_REQUIRED);
+
     for (int i = 0; i < n_layer; ++i) {
         auto & layer = layers[i];