Commit 62a6d3a30 for llama.cpp
commit 62a6d3a302165fc06509b5250260677ef8cd28c2
Author: Niklas Wenzel <dev@nikwen.de>
Date: Mon Nov 16 15:16:12 2026 +0100
llama : migrate to `llama_batch_ext` (#32356)
* llama : migrate to llama_batch_ext
* mtmd : migrate zonos3-mini code
Added in https://github.com/ggml-org/llama.cpp/pull/32020
diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift
index fd90bbec5..d8761f265 100644
--- a/examples/batched.swift/Sources/main.swift
+++ b/examples/batched.swift/Sources/main.swift
@@ -87,36 +87,32 @@ for id: llama_token in tokens {
print("\n")
-var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
+let batch = llama_batch_ext_init(context)
defer {
- llama_batch_free(batch)
+ llama_batch_ext_free(batch)
}
// evaluate the initial prompt
-batch.n_tokens = Int32(tokens.count)
-
for (i, token) in tokens.enumerated() {
- batch.token[i] = token
- batch.pos[i] = Int32(i)
- batch.n_seq_id[i] = 1
- // batch.seq_id[i][0] = 0
- // TODO: is this the proper way to do this?
- if let seq_id = batch.seq_id[i] {
- seq_id[0] = 0
+ let idx = llama_batch_ext_add_token(batch, 0, token)
+ if idx < 0 {
+ print("llama_batch_ext_add_token() failed: \(idx)")
+ exit(1)
}
- batch.logits[i] = 0
+ var pos = llama_pos(i)
+ llama_batch_ext_set_pos(batch, idx, &pos)
}
-// llama_decode will output logits only for the last token of the prompt
-batch.logits[Int(batch.n_tokens) - 1] = 1
+// llama_process will output logits only for the last token of the prompt
+llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)
-if llama_decode(context, batch) != 0 {
- print("llama_decode() failed")
+if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+ print("llama_process() failed")
exit(1)
}
for i in 1 ..< n_parallel {
- llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
+ llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
}
if n_parallel > 1 {
@@ -125,16 +121,17 @@ if n_parallel > 1 {
var streams: [String] = .init(repeating: "", count: n_parallel)
var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
-var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
+var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)
-var n_cur = batch.n_tokens
+var n_cur = Int32(tokens.count)
var n_decode = 0
let t_main_start = ggml_time_us()
while n_cur <= n_len {
// prepare the next batch
- batch.n_tokens = 0
+ llama_batch_ext_clear(batch)
+ var n_tokens: Int32 = 0
// sample the next token for each parallel sequence / stream
for i in 0 ..< n_parallel {
@@ -165,31 +162,32 @@ while n_cur <= n_len {
streams[i] += nextStringPiece
// push this new token for next evaluation
- batch.token[Int(batch.n_tokens)] = new_token_id
- batch.pos[Int(batch.n_tokens)] = n_cur
- batch.n_seq_id[Int(batch.n_tokens)] = 1
- if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
- seq_id[0] = Int32(i)
+ let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
+ if idx < 0 {
+ print("llama_batch_ext_add_token() failed: \(idx)")
+ exit(1)
}
- batch.logits[Int(batch.n_tokens)] = 1
+ var pos = n_cur
+ llama_batch_ext_set_pos(batch, idx, &pos)
+ llama_batch_ext_set_output_logits(batch, idx, true)
- i_batch[i] = batch.n_tokens
+ i_batch[i] = idx
- batch.n_tokens += 1
+ n_tokens += 1
n_decode += 1
}
// all streams are finished
- if batch.n_tokens == 0 {
+ if n_tokens == 0 {
break
}
n_cur += 1
// evaluate the current batch with the transformer model
- if llama_decode(context, batch) != 0 {
- print("llama_decode() failed")
+ if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+ print("llama_process() failed")
exit(1)
}
}
diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
index dc2bafc88..e00938300 100644
--- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
+++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
@@ -5,20 +5,16 @@ enum LlamaError: Error {
case couldNotInitializeContext
}
-func llama_batch_clear(_ batch: inout llama_batch) {
- batch.n_tokens = 0
-}
-
-func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
- batch.token [Int(batch.n_tokens)] = id
- batch.pos [Int(batch.n_tokens)] = pos
- batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
- for i in 0..<seq_ids.count {
- batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
+@discardableResult
+func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
+ let idx = llama_batch_ext_add_token(batch, seq_id, id)
+ if idx < 0 {
+ return idx
}
- batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
-
- batch.n_tokens += 1
+ var pos = pos
+ llama_batch_ext_set_pos(batch, idx, &pos)
+ llama_batch_ext_set_output_logits(batch, idx, logits)
+ return idx
}
actor LlamaContext {
@@ -26,7 +22,7 @@ actor LlamaContext {
private var context: OpaquePointer
private var vocab: OpaquePointer
private var sampling: UnsafeMutablePointer<llama_sampler>
- private var batch: llama_batch
+ private var batch: OpaquePointer
private var tokens_list: [llama_token]
var is_done: Bool = false
@@ -42,7 +38,7 @@ actor LlamaContext {
self.model = model
self.context = context
self.tokens_list = []
- self.batch = llama_batch_init(512, 0, 1)
+ self.batch = llama_batch_ext_init(context)
self.temporary_invalid_cchars = []
let sparams = llama_sampler_chain_default_params()
self.sampling = llama_sampler_chain_init(sparams)
@@ -53,7 +49,7 @@ actor LlamaContext {
deinit {
llama_sampler_free(sampling)
- llama_batch_free(batch)
+ llama_batch_ext_free(batch)
llama_model_free(model)
llama_free(context)
llama_backend_free()
@@ -110,10 +106,6 @@ actor LlamaContext {
return SwiftString
}
- func get_n_tokens() -> Int32 {
- return batch.n_tokens;
- }
-
func completion_init(text: String) {
print("attempting to complete \"\(text)\"")
@@ -133,25 +125,27 @@ actor LlamaContext {
print(String(cString: token_to_piece(token: id) + [0]))
}
- llama_batch_clear(&batch)
+ llama_batch_ext_clear(batch)
for i1 in 0..<tokens_list.count {
let i = Int(i1)
- llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
+ let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
+ if idx < 0 {
+ print("llama_batch_add() failed: \(idx)")
+ }
}
- batch.logits[Int(batch.n_tokens) - 1] = 1 // true
- if llama_decode(context, batch) != 0 {
- print("llama_decode() failed")
+ if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+ print("llama_process() failed")
}
- n_cur = batch.n_tokens
+ n_cur = Int32(tokens_list.count)
}
func completion_loop() -> String {
var new_token_id: llama_token = 0
- new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
+ new_token_id = llama_sampler_sample(sampling, context, -1)
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
print("\n")
@@ -178,13 +172,13 @@ actor LlamaContext {
print(new_token_str)
// tokens_list.append(new_token_id)
- llama_batch_clear(&batch)
- llama_batch_add(&batch, new_token_id, n_cur, [0], true)
+ llama_batch_ext_clear(batch)
+ llama_batch_add(batch, new_token_id, n_cur, 0, true)
n_decode += 1
n_cur += 1
- if llama_decode(context, batch) != 0 {
+ if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
print("failed to evaluate llama!")
}
@@ -201,21 +195,20 @@ actor LlamaContext {
for _ in 0..<nr {
// bench prompt processing
- llama_batch_clear(&batch)
+ llama_batch_ext_clear(batch)
let n_tokens = pp
for i in 0..<n_tokens {
- llama_batch_add(&batch, 0, Int32(i), [0], false)
+ llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
}
- batch.logits[Int(batch.n_tokens) - 1] = 1 // true
llama_memory_clear(llama_get_memory(context), false)
let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;
- if llama_decode(context, batch) != 0 {
- print("llama_decode() failed during prompt")
+ if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+ print("llama_process() failed during prompt")
}
llama_synchronize(context)
@@ -228,14 +221,14 @@ actor LlamaContext {
let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;
for i in 0..<tg {
- llama_batch_clear(&batch)
+ llama_batch_ext_clear(batch)
for j in 0..<pl {
- llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
+ llama_batch_add(batch, 0, Int32(i), Int32(j), true)
}
- if llama_decode(context, batch) != 0 {
- print("llama_decode() failed during text generation")
+ if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
+ print("llama_process() failed during text generation")
}
llama_synchronize(context)
}
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index b66092b17..969989ab5 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -4254,7 +4254,7 @@ void llama_context::opt_epoch_iter(
ggml_opt_result_t result,
const std::vector<llama_token> & tokens,
const std::vector<llama_token> & labels_sparse,
- llama_batch & batch,
+ llama_batch_ext & batch,
ggml_opt_epoch_callback callback,
bool train,
int64_t idata_in_loop,
@@ -4268,22 +4268,21 @@ void llama_context::opt_epoch_iter(
memory->clear(true);
for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
- batch.n_tokens = n_batch;
+ batch.clear();
for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
- batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
- batch.pos [pos_batch] = pos_ctx + pos_batch;
- batch.n_seq_id[pos_batch] = 1;
- batch.seq_id [pos_batch][0] = 0;
- batch.logits [pos_batch] = true;
- }
-
- // TODO: use llama_batch_ext here
- {
- llama_batch_compat compat(this, batch);
- if (!balloc->init(*compat.batch_ext, model.vocab, true)) {
+ const int32_t idx = batch.add_token(0);
+ const llama_pos pos = pos_ctx + pos_batch;
+ if (idx < 0 || !batch.set_token_id(idx, tokens[pos_ctx + pos_batch])) {
LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
return;
}
+ batch.set_token_pos(idx, &pos);
+ batch.set_output(idx, true);
+ }
+
+ if (!balloc->init(batch, model.vocab, true)) {
+ LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
+ return;
}
const uint32_t n_tokens_all = balloc->get_n_tokens();
@@ -4382,7 +4381,7 @@ void llama_context::opt_epoch(
const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
- struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
+ llama_batch_ext batch(this);
std::vector<llama_token> tokens(n_ctx);
std::vector<llama_token> labels_sparse(n_ctx);
@@ -4409,8 +4408,6 @@ void llama_context::opt_epoch(
opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
}
-
- llama_batch_free(batch);
}
//
diff --git a/src/llama-context.h b/src/llama-context.h
index 45eca4566..3e3921c7d 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -233,7 +233,7 @@ struct llama_context {
ggml_opt_result_t result,
const std::vector<llama_token> & tokens,
const std::vector<llama_token> & labels_sparse,
- llama_batch & batch,
+ llama_batch_ext & batch,
ggml_opt_epoch_callback callback,
bool train,
int64_t idata_in_loop,
diff --git a/tests/llama-context/cases-kv.cpp b/tests/llama-context/cases-kv.cpp
index b970ea940..4d2970e9d 100644
--- a/tests/llama-context/cases-kv.cpp
+++ b/tests/llama-context/cases-kv.cpp
@@ -557,7 +557,8 @@ struct case_state_rotation : ctx_test {
}
llama_token token = 0;
- if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
+ common_batch batch = common_batch_get_one(src.get(), &token, 1);
+ if (llama_process(src.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
TLOG_ERR("%s: failed to decode a token\n", __func__);
success = false;
break;
diff --git a/tests/test-batch-alloc.cpp b/tests/test-batch-alloc.cpp
index c41e91d61..42c4c5249 100644
--- a/tests/test-batch-alloc.cpp
+++ b/tests/test-batch-alloc.cpp
@@ -978,7 +978,9 @@ static void test_compat(testing & t) {
mem.ranges[0] = {0, 9};
llama_token token[3] = { 5, 6, 7 };
- llama_batch lb = llama_batch_get_one(token, 3);
+ llama_batch lb = {};
+ lb.n_tokens = 3;
+ lb.token = token;
batch_builder bb(2, &mem, 4, 1, /*n_vocab*/ 100);
llama_batch_compat::init(bb.b, lb);
@@ -999,7 +1001,9 @@ static void test_compat(testing & t) {
t.test("auto_pos_starts_at_zero_without_memory", [&](testing & t) {
llama_token token[2] = { 5, 6 };
- llama_batch lb = llama_batch_get_one(token, 2);
+ llama_batch lb = {};
+ lb.n_tokens = 2;
+ lb.token = token;
batch_builder bb(2, nullptr, 4, 1, /*n_vocab*/ 100);
llama_batch_compat::init(bb.b, lb);
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 8a5757c1e..2148ec526 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -52,6 +52,20 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
return LLAMA_TOKEN_NULL;
}
+static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) {
+ llama_batch_ext_clear(batch);
+ for (int32_t i = 0; i < n; i++) {
+ const int32_t idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]);
+ const llama_pos pos = pos_0 + i;
+ if (idx < 0) {
+ LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx);
+ return false;
+ }
+ llama_batch_ext_set_pos(batch, idx, &pos);
+ }
+ return true;
+}
+
static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
const char * c = (const char *) p;
buf.insert(buf.end(), c, c + n);
@@ -1206,18 +1220,12 @@ private:
}
bool decode(llama_token * tokens, int n) {
- llama_batch batch = llama_batch_init(n, 0, 1);
- batch.n_tokens = n;
- for (int i = 0; i < n; ++i) {
- batch.token[i] = tokens[i];
- batch.pos[i] = pos + i;
- batch.n_seq_id[i] = 1;
- batch.seq_id[i][0] = seq_id;
- batch.logits[i] = i == n - 1;
- }
- const int ret = llama_decode(lctx, batch);
- llama_batch_free(batch);
- if (ret != 0) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) {
+ return false;
+ }
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
return false;
}
@@ -1330,18 +1338,14 @@ public:
return 0;
}
const int n = std::min(n_batch, n_prompt - prompt_pos);
- llama_batch batch = llama_batch_init(n, 0, 1);
- for (int i = 0; i < n; i++) {
- batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)];
- batch.pos [i] = pos + i;
- batch.n_seq_id[i] = 1;
- batch.seq_id [i][0] = seq_id;
- batch.logits [i] = prompt_pos + i == n_prompt - 1;
- }
- batch.n_tokens = n;
- const int ret = llama_decode(lctx, batch);
- llama_batch_free(batch);
- if (ret != 0) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+ return -1;
+ }
+ if (prompt_pos + n == n_prompt) {
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
@@ -1382,8 +1386,7 @@ public:
text_done = true;
speaking = true;
}
- llama_batch batch = llama_batch_get_one(&sampled, 1);
- if (!decode(batch)) {
+ if (!decode(sampled)) {
return 1;
}
} else {
@@ -1626,25 +1629,17 @@ private:
return true;
}
- bool decode(llama_batch batch) {
- std::vector<llama_pos> positions(batch.n_tokens);
- std::vector<int32_t> n_seq(batch.n_tokens, 1);
- std::vector<llama_seq_id> seq(batch.n_tokens, seq_id);
- std::vector<llama_seq_id *> seq_ptr(batch.n_tokens);
- std::vector<int8_t> logits(batch.n_tokens, 1);
- for (int i = 0; i < batch.n_tokens; i++) {
- positions[i] = pos + i;
- seq_ptr[i] = &seq[i];
- }
- batch.pos = positions.data();
- batch.n_seq_id = n_seq.data();
- batch.seq_id = seq_ptr.data();
- batch.logits = logits.data();
- if (llama_decode(lctx, batch) != 0) {
+ bool decode(llama_token token) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) {
+ return false;
+ }
+ llama_batch_ext_set_output_logits(batch.get(), 0, true);
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return false;
}
- pos += batch.n_tokens;
+ pos += 1;
return true;
}
@@ -2160,18 +2155,14 @@ public:
}
// small batches: every row also outputs logits
const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
- llama_batch batch = llama_batch_init(n, 0, 1);
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+ return -1;
+ }
for (int i = 0; i < n; i++) {
- batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)];
- batch.pos [i] = pos + i;
- batch.n_seq_id[i] = 1;
- batch.seq_id [i][0] = seq_id;
- batch.logits [i] = 1;
- }
- batch.n_tokens = n;
- const int ret = llama_decode(lctx, batch);
- llama_batch_free(batch);
- if (ret != 0) {
+ llama_batch_ext_set_output_logits(batch.get(), i, true);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
@@ -2546,17 +2537,14 @@ public:
int ret;
if (!seg.tokens.empty()) {
- llama_batch batch = llama_batch_init(n, 0, 1);
- for (int i = 0; i < n; i++) {
- batch.token [i] = seg.tokens[(size_t) (seg_pos + i)];
- batch.pos [i] = pos + i;
- batch.n_seq_id[i] = 1;
- batch.seq_id [i][0] = seq_id;
- batch.logits [i] = last && i == n - 1;
- }
- batch.n_tokens = n;
- ret = llama_decode(lctx, batch);
- llama_batch_free(batch);
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) {
+ return -1;
+ }
+ if (last) {
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ }
+ ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
} else {
decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd);
batch.set_position_normal(pos, seq_id);
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index e747d32d7..201740b2e 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -1032,24 +1032,20 @@ bool server_realtime_slot::decode(const llama_token * tokens, const float * rows
if (n_past + n >= n_ctx) {
return fail("the conversation does not fit in the context", ERROR_TYPE_EXCEED_CONTEXT_SIZE);
}
- for (int32_t i0 = 0; i0 < n; i0 += n_batch) {
- const int32_t n_cur = std::min(n_batch, n - i0);
- llama_batch b = llama_batch_init(n_cur, rows ? n_embd : 0, 1);
- for (int32_t i = 0; i < n_cur; i++) {
- if (rows) {
- memcpy(b.embd + (size_t) i * n_embd, rows + (size_t) (i0 + i) * n_embd, n_embd * sizeof(float));
- } else {
- b.token[i] = tokens[i0 + i];
- }
- b.pos[i] = n_past + i0 + i;
- b.n_seq_id[i] = 1;
- b.seq_id[i][0] = seq_id;
- b.logits[i] = logits_last && i0 + i == n - 1;
- }
- b.n_tokens = n_cur;
- const int ret = llama_decode(ctx, b);
- llama_batch_free(b);
- if (ret != 0) {
+ common_batch batch(ctx);
+ for (int32_t i = 0; i < n; i++) {
+ const llama_pos pos = n_past + i;
+ const bool output = logits_last && i == n - 1;
+ if (rows) {
+ const llama_embd e = { rows + (size_t) i * n_embd, 1, (size_t) n_embd };
+ batch.add_embd(e, &pos, seq_id, output);
+ } else {
+ batch.add(tokens[i], pos, seq_id, output);
+ }
+ }
+ for (int32_t i = 0; i < n; i += n_batch) {
+ const int32_t n_cur = std::min(n_batch, n - i);
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get_sub_batch(i, n_cur)) != 0) {
return fail("failed to decode the conversation");
}
}