Commit 62a6d3a30 for llama.cpp

commit 62a6d3a302165fc06509b5250260677ef8cd28c2
Author: Niklas Wenzel <dev@nikwen.de>
Date:   Mon Nov 16 15:16:12 2026 +0100

    llama : migrate to `llama_batch_ext` (#32356)

    * llama : migrate to llama_batch_ext

    * mtmd : migrate zonos3-mini code

    Added in https://github.com/ggml-org/llama.cpp/pull/32020

diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift
index fd90bbec5..d8761f265 100644
--- a/examples/batched.swift/Sources/main.swift
+++ b/examples/batched.swift/Sources/main.swift
@@ -87,36 +87,32 @@ for id: llama_token in tokens {

 print("\n")

-var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
+let batch = llama_batch_ext_init(context)
 defer {
-    llama_batch_free(batch)
+    llama_batch_ext_free(batch)
 }

 // evaluate the initial prompt
-batch.n_tokens = Int32(tokens.count)
-
 for (i, token) in tokens.enumerated() {
-    batch.token[i] = token
-    batch.pos[i] = Int32(i)
-    batch.n_seq_id[i] = 1
-    // batch.seq_id[i][0] = 0
-    // TODO: is this the proper way to do this?
-    if let seq_id = batch.seq_id[i] {
-        seq_id[0] = 0
+    let idx = llama_batch_ext_add_token(batch, 0, token)
+    if idx < 0 {
+        print("llama_batch_ext_add_token() failed: \(idx)")
+        exit(1)
     }
-    batch.logits[i] = 0
+    var pos = llama_pos(i)
+    llama_batch_ext_set_pos(batch, idx, &pos)
 }

-// llama_decode will output logits only for the last token of the prompt
-batch.logits[Int(batch.n_tokens) - 1] = 1
+// llama_process will output logits only for the last token of the prompt
+llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)

-if llama_decode(context, batch) != 0 {
-    print("llama_decode() failed")
+if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+    print("llama_process() failed")
     exit(1)
 }

 for i in 1 ..< n_parallel {
-    llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
+    llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
 }

 if n_parallel > 1 {
@@ -125,16 +121,17 @@ if n_parallel > 1 {

 var streams: [String] = .init(repeating: "", count: n_parallel)
 var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
-var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
+var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)

-var n_cur = batch.n_tokens
+var n_cur = Int32(tokens.count)
 var n_decode = 0

 let t_main_start = ggml_time_us()

 while n_cur <= n_len {
     // prepare the next batch
-    batch.n_tokens = 0
+    llama_batch_ext_clear(batch)
+    var n_tokens: Int32 = 0

     // sample the next token for each parallel sequence / stream
     for i in 0 ..< n_parallel {
@@ -165,31 +162,32 @@ while n_cur <= n_len {
         streams[i] += nextStringPiece

         // push this new token for next evaluation
-        batch.token[Int(batch.n_tokens)] = new_token_id
-        batch.pos[Int(batch.n_tokens)] = n_cur
-        batch.n_seq_id[Int(batch.n_tokens)] = 1
-        if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
-            seq_id[0] = Int32(i)
+        let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
+        if idx < 0 {
+            print("llama_batch_ext_add_token() failed: \(idx)")
+            exit(1)
         }
-        batch.logits[Int(batch.n_tokens)] = 1
+        var pos = n_cur
+        llama_batch_ext_set_pos(batch, idx, &pos)
+        llama_batch_ext_set_output_logits(batch, idx, true)

-        i_batch[i] = batch.n_tokens
+        i_batch[i] = idx

-        batch.n_tokens += 1
+        n_tokens += 1

         n_decode += 1
     }

     // all streams are finished
-    if batch.n_tokens == 0 {
+    if n_tokens == 0 {
         break
     }

     n_cur += 1

     // evaluate the current batch with the transformer model
-    if llama_decode(context, batch) != 0 {
-        print("llama_decode() failed")
+    if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+        print("llama_process() failed")
         exit(1)
     }
 }
diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
index dc2bafc88..e00938300 100644
--- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
+++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
@@ -5,20 +5,16 @@ enum LlamaError: Error {
     case couldNotInitializeContext
 }

-func llama_batch_clear(_ batch: inout llama_batch) {
-    batch.n_tokens = 0
-}
-
-func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
-    batch.token   [Int(batch.n_tokens)] = id
-    batch.pos     [Int(batch.n_tokens)] = pos
-    batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
-    for i in 0..<seq_ids.count {
-        batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
+@discardableResult
+func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
+    let idx = llama_batch_ext_add_token(batch, seq_id, id)
+    if idx < 0 {
+        return idx
     }
-    batch.logits  [Int(batch.n_tokens)] = logits ? 1 : 0
-
-    batch.n_tokens += 1
+    var pos = pos
+    llama_batch_ext_set_pos(batch, idx, &pos)
+    llama_batch_ext_set_output_logits(batch, idx, logits)
+    return idx
 }

 actor LlamaContext {
@@ -26,7 +22,7 @@ actor LlamaContext {
     private var context: OpaquePointer
     private var vocab: OpaquePointer
     private var sampling: UnsafeMutablePointer<llama_sampler>
-    private var batch: llama_batch
+    private var batch: OpaquePointer
     private var tokens_list: [llama_token]
     var is_done: Bool = false

@@ -42,7 +38,7 @@ actor LlamaContext {
         self.model = model
         self.context = context
         self.tokens_list = []
-        self.batch = llama_batch_init(512, 0, 1)
+        self.batch = llama_batch_ext_init(context)
         self.temporary_invalid_cchars = []
         let sparams = llama_sampler_chain_default_params()
         self.sampling = llama_sampler_chain_init(sparams)
@@ -53,7 +49,7 @@ actor LlamaContext {

     deinit {
         llama_sampler_free(sampling)
-        llama_batch_free(batch)
+        llama_batch_ext_free(batch)
         llama_model_free(model)
         llama_free(context)
         llama_backend_free()
@@ -110,10 +106,6 @@ actor LlamaContext {
         return SwiftString
     }

-    func get_n_tokens() -> Int32 {
-        return batch.n_tokens;
-    }
-
     func completion_init(text: String) {
         print("attempting to complete \"\(text)\"")

@@ -133,25 +125,27 @@ actor LlamaContext {
             print(String(cString: token_to_piece(token: id) + [0]))
         }

-        llama_batch_clear(&batch)
+        llama_batch_ext_clear(batch)

         for i1 in 0..<tokens_list.count {
             let i = Int(i1)
-            llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
+            let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
+            if idx < 0 {
+                print("llama_batch_add() failed: \(idx)")
+            }
         }
-        batch.logits[Int(batch.n_tokens) - 1] = 1 // true

-        if llama_decode(context, batch) != 0 {
-            print("llama_decode() failed")
+        if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+            print("llama_process() failed")
         }

-        n_cur = batch.n_tokens
+        n_cur = Int32(tokens_list.count)
     }

     func completion_loop() -> String {
         var new_token_id: llama_token = 0

-        new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
+        new_token_id = llama_sampler_sample(sampling, context, -1)

         if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
             print("\n")
@@ -178,13 +172,13 @@ actor LlamaContext {
         print(new_token_str)
         // tokens_list.append(new_token_id)

-        llama_batch_clear(&batch)
-        llama_batch_add(&batch, new_token_id, n_cur, [0], true)
+        llama_batch_ext_clear(batch)
+        llama_batch_add(batch, new_token_id, n_cur, 0, true)

         n_decode += 1
         n_cur    += 1

-        if llama_decode(context, batch) != 0 {
+        if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
             print("failed to evaluate llama!")
         }

@@ -201,21 +195,20 @@ actor LlamaContext {
         for _ in 0..<nr {
             // bench prompt processing

-            llama_batch_clear(&batch)
+            llama_batch_ext_clear(batch)

             let n_tokens = pp

             for i in 0..<n_tokens {
-                llama_batch_add(&batch, 0, Int32(i), [0], false)
+                llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
             }
-            batch.logits[Int(batch.n_tokens) - 1] = 1 // true

             llama_memory_clear(llama_get_memory(context), false)

             let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;

-            if llama_decode(context, batch) != 0 {
-                print("llama_decode() failed during prompt")
+            if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+                print("llama_process() failed during prompt")
             }
             llama_synchronize(context)

@@ -228,14 +221,14 @@ actor LlamaContext {
             let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;

             for i in 0..<tg {
-                llama_batch_clear(&batch)
+                llama_batch_ext_clear(batch)

                 for j in 0..<pl {
-                    llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
+                    llama_batch_add(batch, 0, Int32(i), Int32(j), true)
                 }

-                if llama_decode(context, batch) != 0 {
-                    print("llama_decode() failed during text generation")
+                if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+                    print("llama_process() failed during text generation")
                 }
                 llama_synchronize(context)
             }
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index b66092b17..969989ab5 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -4254,7 +4254,7 @@ void llama_context::opt_epoch_iter(
         ggml_opt_result_t                result,
         const std::vector<llama_token> & tokens,
         const std::vector<llama_token> & labels_sparse,
-        llama_batch                    & batch,
+        llama_batch_ext                & batch,
         ggml_opt_epoch_callback          callback,
         bool                             train,
         int64_t                          idata_in_loop,
@@ -4268,22 +4268,21 @@ void llama_context::opt_epoch_iter(
     memory->clear(true);

     for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
-        batch.n_tokens = n_batch;
+        batch.clear();
         for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
-            batch.token   [pos_batch]    = tokens[pos_ctx + pos_batch];
-            batch.pos     [pos_batch]    = pos_ctx + pos_batch;
-            batch.n_seq_id[pos_batch]    = 1;
-            batch.seq_id  [pos_batch][0] = 0;
-            batch.logits  [pos_batch]    = true;
-        }
-
-        // TODO: use llama_batch_ext here
-        {
-            llama_batch_compat compat(this, batch);
-            if (!balloc->init(*compat.batch_ext, model.vocab, true)) {
+            const int32_t   idx = batch.add_token(0);
+            const llama_pos pos = pos_ctx + pos_batch;
+            if (idx < 0 || !batch.set_token_id(idx, tokens[pos_ctx + pos_batch])) {
                 LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
                 return;
             }
+            batch.set_token_pos(idx, &pos);
+            batch.set_output(idx, true);
+        }
+
+        if (!balloc->init(batch, model.vocab, true)) {
+            LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
+            return;
         }

         const uint32_t n_tokens_all = balloc->get_n_tokens();
@@ -4382,7 +4381,7 @@ void llama_context::opt_epoch(

     const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;

-    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
+    llama_batch_ext batch(this);
     std::vector<llama_token>        tokens(n_ctx);
     std::vector<llama_token> labels_sparse(n_ctx);

@@ -4409,8 +4408,6 @@ void llama_context::opt_epoch(
         opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
             callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
     }
-
-    llama_batch_free(batch);
 }

 //
diff --git a/src/llama-context.h b/src/llama-context.h
index 45eca4566..3e3921c7d 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -233,7 +233,7 @@ struct llama_context {
             ggml_opt_result_t                result,
             const std::vector<llama_token> & tokens,
             const std::vector<llama_token> & labels_sparse,
-            llama_batch                    & batch,
+            llama_batch_ext                & batch,
             ggml_opt_epoch_callback          callback,
             bool                             train,
             int64_t                          idata_in_loop,
diff --git a/tests/llama-context/cases-kv.cpp b/tests/llama-context/cases-kv.cpp
index b970ea940..4d2970e9d 100644
--- a/tests/llama-context/cases-kv.cpp
+++ b/tests/llama-context/cases-kv.cpp
@@ -557,7 +557,8 @@ struct case_state_rotation : ctx_test {
             }

             llama_token token = 0;
-            if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
+            common_batch batch = common_batch_get_one(src.get(), &token, 1);
+            if (llama_process(src.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
                 TLOG_ERR("%s: failed to decode a token\n", __func__);
                 success = false;
                 break;
diff --git a/tests/test-batch-alloc.cpp b/tests/test-batch-alloc.cpp
index c41e91d61..42c4c5249 100644
--- a/tests/test-batch-alloc.cpp
+++ b/tests/test-batch-alloc.cpp
@@ -978,7 +978,9 @@ static void test_compat(testing & t) {
         mem.ranges[0] = {0, 9};

         llama_token token[3] = { 5, 6, 7 };
-        llama_batch lb = llama_batch_get_one(token, 3);
+        llama_batch lb = {};
+        lb.n_tokens = 3;
+        lb.token    = token;

         batch_builder bb(2, &mem, 4, 1, /*n_vocab*/ 100);
         llama_batch_compat::init(bb.b, lb);
@@ -999,7 +1001,9 @@ static void test_compat(testing & t) {

     t.test("auto_pos_starts_at_zero_without_memory", [&](testing & t) {
         llama_token token[2] = { 5, 6 };
-        llama_batch lb = llama_batch_get_one(token, 2);
+        llama_batch lb = {};
+        lb.n_tokens = 2;
+        lb.token    = token;

         batch_builder bb(2, nullptr, 4, 1, /*n_vocab*/ 100);
         llama_batch_compat::init(bb.b, lb);
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 8a5757c1e..2148ec526 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -52,6 +52,20 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
     return LLAMA_TOKEN_NULL;
 }

+static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) {
+    llama_batch_ext_clear(batch);
+    for (int32_t i = 0; i < n; i++) {
+        const int32_t   idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]);
+        const llama_pos pos = pos_0 + i;
+        if (idx < 0) {
+            LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx);
+            return false;
+        }
+        llama_batch_ext_set_pos(batch, idx, &pos);
+    }
+    return true;
+}
+
 static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
     const char * c = (const char *) p;
     buf.insert(buf.end(), c, c + n);
@@ -1206,18 +1220,12 @@ private:
     }

     bool decode(llama_token * tokens, int n) {
-        llama_batch batch = llama_batch_init(n, 0, 1);
-        batch.n_tokens = n;
-        for (int i = 0; i < n; ++i) {
-            batch.token[i] = tokens[i];
-            batch.pos[i] = pos + i;
-            batch.n_seq_id[i] = 1;
-            batch.seq_id[i][0] = seq_id;
-            batch.logits[i] = i == n - 1;
-        }
-        const int ret = llama_decode(lctx, batch);
-        llama_batch_free(batch);
-        if (ret != 0) {
+        llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+        if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) {
+            return false;
+        }
+        llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
             LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
             return false;
         }
@@ -1330,18 +1338,14 @@ public:
             return 0;
         }
         const int n = std::min(n_batch, n_prompt - prompt_pos);
-        llama_batch batch = llama_batch_init(n, 0, 1);
-        for (int i = 0; i < n; i++) {
-            batch.token   [i]    = prompt_tokens[(size_t) (prompt_pos + i)];
-            batch.pos     [i]    = pos + i;
-            batch.n_seq_id[i]    = 1;
-            batch.seq_id  [i][0] = seq_id;
-            batch.logits  [i]    = prompt_pos + i == n_prompt - 1;
-        }
-        batch.n_tokens = n;
-        const int ret = llama_decode(lctx, batch);
-        llama_batch_free(batch);
-        if (ret != 0) {
+        llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+        if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+            return -1;
+        }
+        if (prompt_pos + n == n_prompt) {
+            llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+        }
+        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
             LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
             return -1;
         }
@@ -1382,8 +1386,7 @@ public:
                 text_done = true;
                 speaking  = true;
             }
-            llama_batch batch = llama_batch_get_one(&sampled, 1);
-            if (!decode(batch)) {
+            if (!decode(sampled)) {
                 return 1;
             }
         } else {
@@ -1626,25 +1629,17 @@ private:
         return true;
     }

-    bool decode(llama_batch batch) {
-        std::vector<llama_pos>      positions(batch.n_tokens);
-        std::vector<int32_t>        n_seq(batch.n_tokens, 1);
-        std::vector<llama_seq_id>   seq(batch.n_tokens, seq_id);
-        std::vector<llama_seq_id *> seq_ptr(batch.n_tokens);
-        std::vector<int8_t>         logits(batch.n_tokens, 1);
-        for (int i = 0; i < batch.n_tokens; i++) {
-            positions[i] = pos + i;
-            seq_ptr[i]   = &seq[i];
-        }
-        batch.pos      = positions.data();
-        batch.n_seq_id = n_seq.data();
-        batch.seq_id   = seq_ptr.data();
-        batch.logits   = logits.data();
-        if (llama_decode(lctx, batch) != 0) {
+    bool decode(llama_token token) {
+        llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+        if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) {
+            return false;
+        }
+        llama_batch_ext_set_output_logits(batch.get(), 0, true);
+        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
             LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
             return false;
         }
-        pos += batch.n_tokens;
+        pos += 1;
         return true;
     }

@@ -2160,18 +2155,14 @@ public:
         }
         // small batches: every row also outputs logits
         const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
-        llama_batch batch = llama_batch_init(n, 0, 1);
+        llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+        if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+            return -1;
+        }
         for (int i = 0; i < n; i++) {
-            batch.token   [i]    = prompt_tokens[(size_t) (prompt_pos + i)];
-            batch.pos     [i]    = pos + i;
-            batch.n_seq_id[i]    = 1;
-            batch.seq_id  [i][0] = seq_id;
-            batch.logits  [i]    = 1;
-        }
-        batch.n_tokens = n;
-        const int ret = llama_decode(lctx, batch);
-        llama_batch_free(batch);
-        if (ret != 0) {
+            llama_batch_ext_set_output_logits(batch.get(), i, true);
+        }
+        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
             LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
             return -1;
         }
@@ -2546,17 +2537,14 @@ public:

         int ret;
         if (!seg.tokens.empty()) {
-            llama_batch batch = llama_batch_init(n, 0, 1);
-            for (int i = 0; i < n; i++) {
-                batch.token   [i]    = seg.tokens[(size_t) (seg_pos + i)];
-                batch.pos     [i]    = pos + i;
-                batch.n_seq_id[i]    = 1;
-                batch.seq_id  [i][0] = seq_id;
-                batch.logits  [i]    = last && i == n - 1;
-            }
-            batch.n_tokens = n;
-            ret = llama_decode(lctx, batch);
-            llama_batch_free(batch);
+            llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+            if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) {
+                return -1;
+            }
+            if (last) {
+                llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+            }
+            ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
         } else {
             decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd);
             batch.set_position_normal(pos, seq_id);
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index e747d32d7..201740b2e 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -1032,24 +1032,20 @@ bool server_realtime_slot::decode(const llama_token * tokens, const float * rows
     if (n_past + n >= n_ctx) {
         return fail("the conversation does not fit in the context", ERROR_TYPE_EXCEED_CONTEXT_SIZE);
     }
-    for (int32_t i0 = 0; i0 < n; i0 += n_batch) {
-        const int32_t n_cur = std::min(n_batch, n - i0);
-        llama_batch b = llama_batch_init(n_cur, rows ? n_embd : 0, 1);
-        for (int32_t i = 0; i < n_cur; i++) {
-            if (rows) {
-                memcpy(b.embd + (size_t) i * n_embd, rows + (size_t) (i0 + i) * n_embd, n_embd * sizeof(float));
-            } else {
-                b.token[i] = tokens[i0 + i];
-            }
-            b.pos[i]       = n_past + i0 + i;
-            b.n_seq_id[i]  = 1;
-            b.seq_id[i][0] = seq_id;
-            b.logits[i]    = logits_last && i0 + i == n - 1;
-        }
-        b.n_tokens = n_cur;
-        const int ret = llama_decode(ctx, b);
-        llama_batch_free(b);
-        if (ret != 0) {
+    common_batch batch(ctx);
+    for (int32_t i = 0; i < n; i++) {
+        const llama_pos pos    = n_past + i;
+        const bool      output = logits_last && i == n - 1;
+        if (rows) {
+            const llama_embd e = { rows + (size_t) i * n_embd, 1, (size_t) n_embd };
+            batch.add_embd(e, &pos, seq_id, output);
+        } else {
+            batch.add(tokens[i], pos, seq_id, output);
+        }
+    }
+    for (int32_t i = 0; i < n; i += n_batch) {
+        const int32_t n_cur = std::min(n_batch, n - i);
+        if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get_sub_batch(i, n_cur)) != 0) {
             return fail("failed to decode the conversation");
         }
     }