Commit 1268edac5 for llama.cpp
commit 1268edac579110aeb1407f7928f53193ed4e28d4
Author: Gabe Goodhart <ghart@us.ibm.com>
Date: Sat Oct 17 01:14:32 2026 -0600
model : Granite switch decision 256 options (#30693)
* feat(convert): Attempt to come up with 256 single-token, semantic option markers
Each switch model needs to identify a collection of single-token option
markers that can be mapped to their respective logits for the output
probability distribution. The simple upper/lower/numbers set only yields 66
unique markers, but the model can support up to 255. The challenge is that
which combinations of characters result in single-token markers is model-
specific, so we do a discovery exercise at conversion time by looking
through the specific model's vocab for single-token sequences of 1+ plain
letter strings (eg `A`, `AA`, `BB`, etc).
Branch: GraniteSwitchDecisionFullOptions
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* docs: Note granite-switch with 256 option markers
Branch: GraniteSwitchDecisionFullOptions
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* style: Remove extra newline
Branch: GraniteSwitchDecisionFullOptions
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(convert): Use the computed labels in the template so c++ side is source of truth
Branch: GraniteSwitchDecisionFullOptions
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat: Use LEV case for GRANITE_SWITCH w/ computed labels
Branch: GraniteSwitchDecisionFullOptions
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* docs: Consolidate g-s into 255 bucket
Branch: GraniteSwitchDecisionFullOptions
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
diff --git a/conversion/granite.py b/conversion/granite.py
index d2ef7c836..4c6c5a0e4 100644
--- a/conversion/granite.py
+++ b/conversion/granite.py
@@ -296,19 +296,20 @@ class GraniteSwitchModel(GraniteMoeModel):
def set_vocab(self):
super().set_vocab()
if self._has_decision_adapter():
- self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+ systemone_template = self._systemone_template()
+ logger.info("System One Template: %s", systemone_template)
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}])
def _systemone_template(self) -> str:
# prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
# (it replaces the first token of "### DECISION"), the answer is the label token that comes next
description = jinja_str_or_json("o.description")
option = (
- "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"
+ "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}"
"{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
"{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
)
return (
- "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"
"<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
+ "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
diff --git a/tools/server/README.md b/tools/server/README.md
index e4a0c418b..c42336c7f 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1723,7 +1723,7 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index 3905c81b3..be9bc8ef4 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {
bias[level] = std::strtof(val, nullptr);
}
- if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
+ if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
// one letter per option, each must be a single token
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
for (const char c : letters) {
@@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) {
}
n_options_max = labels.size();
noul_true_first = true;
- } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
+ } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
// label codes are A..Z then AA..ZZ, only the ones that are a single token are used
std::vector<std::string> codes;
for (char a = 'A'; a <= 'Z'; a++) {
@@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) {
if (model_type == COMMON_DECISION_TYPE_VLLM_D3) {
canonical_legend = true;
}
+ if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
+ noul_true_first = true;
+ }
} else if (model_type == COMMON_DECISION_TYPE_KEV) {
// the hidden state of an option is read at the token that ends it
const auto toks = common_tokenize(vocab, "<|box_end|>", false, true);