Commit 359dc5858 for llama.cpp
commit 359dc5858ec0cb47841b1248f403ee6f45c14749
Author: nv-vankit <162085734+nv-vankit@users.noreply.github.com>
Date: Mon Nov 9 23:38:20 2026 +0530
server : functionality test for new spec dec (#31027)
* server : functionality test to make sure every time when we add new spec dec method the existing ones are not broken.
* server: use compact models for speculative decoding CI
* ci: avoid GHA cache for speculative models
* ci: run speculative tests on DGX Spark
* server: address speculative test review
* server: use consolidated speculative models
* server: refine speculative test calibration
* docs: record speculative Spark validation
* ci : run the server-speculative steps with bash
* server: recalibrate speculative test floors on GB10
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/.github/workflows/server-speculative.yml b/.github/workflows/server-speculative.yml
new file mode 100644
index 000000000..de247afb6
--- /dev/null
+++ b/.github/workflows/server-speculative.yml
@@ -0,0 +1,65 @@
+name: Server (speculative decoding)
+
+on:
+ workflow_dispatch:
+ inputs:
+ sha:
+ description: 'Commit SHA1 to build'
+ required: false
+ type: string
+
+cache-mode: none
+permissions:
+ contents: read
+
+env:
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ server-speculative:
+ runs-on: [self-hosted, Linux, gb10, perf]
+ container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
+ timeout-minutes: 60
+ defaults:
+ run:
+ shell: bash
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Install dependencies
+ run: |
+ apt update
+ apt install -y cmake libssl-dev python3 python3-venv python3-pip
+
+ - name: Build
+ run: |
+ cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+ cmake --build build --config Release -j $(nproc) --target llama-server
+
+ - name: Python setup
+ run: |
+ cd tools/server/tests
+ python3 -m venv venv
+ source venv/bin/activate
+ pip install -r requirements.txt
+
+ - name: Test speculative decoding
+ env:
+ SPEC_METHODS: compact
+ SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
+ N_GPU_LAYERS: 99
+ run: |
+ cd tools/server/tests
+ source venv/bin/activate
+ PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
diff --git a/tools/server/tests/README.md b/tools/server/tests/README.md
index f566b4364..7ed02d22e 100644
--- a/tools/server/tests/README.md
+++ b/tools/server/tests/README.md
@@ -39,6 +39,47 @@ To run slow tests (will download many models, make sure to set `LLAMA_CACHE` if
SLOW_TESTS=1 ./tests.sh
```
+The speculative-decoding method tests are opt-in and marked `heavy`. The compact suite covers MTP,
+DFlash, DSpark, and EAGLE-3 with five pinned files totaling 6.271 GiB:
+
+```shell
+SPEC_METHODS=compact SPEC_MODEL_DIR=/path/to/model-cache PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
+```
+
+- MTP target: `Qwen3.5-4B-MTP-Q4_K_M.gguf`, 2.904 GiB.
+- Shared DFlash/DSpark/EAGLE-3 target: `Qwen3-4B-Q4_K_M.gguf`, 2.326 GiB.
+- DFlash draft: `dflash-Qwen3-4B-Q4_K_M.gguf`, 294.065 MiB.
+- DSpark draft: `dspark-Qwen3-4B-Q4_K_M.gguf`, 544.453 MiB.
+- EAGLE-3 draft: `eagle3-Qwen3-4B-Q8_0.gguf`, 227.075 MiB.
+
+All files come from `ggml-org/spec-for-testing-GGUF`, pinned to revision
+`a0736c8e98d3fe580352dacffa3fbd5e9c22a437` in `unit/test_spec_methods.py`.
+The consolidated repository retains the licenses of its source models; review its model card and
+the source model cards before use.
+
+Use `SPEC_METHODS=nightly` for DFlash2 and LiLiCorr, or a comma-separated subset such as
+`SPEC_METHODS=mtp,dflash`. The nightly pair requires 41.422 GiB and remains separate because no
+verified compact DFlash2 GGUF or compact LiLiCorr checkpoint is available. `SPEC_METHODS=all` runs both
+tiers and downloads about 47.693 GiB.
+
+The `Server (speculative decoding)` workflow runs only by explicit manual dispatch on the dedicated
+DGX Spark runner. It does not use the GitHub Actions cache. Set the `SPEC_METHOD_MODEL_DIR` repository
+variable to the runner's persistent model directory when its mount path is confirmed. Heavyweight
+DFlash2 and LiLiCorr measurements belong in the internal nightly performance/quality runs.
+
+The acceptance floors use one-sided regression checks: improvements do not fail. Three identical
+runs on the GB10 runner against the PR merged with current master produced MTP `0.6640 / 2.9766`,
+DFlash `0.2004 / 2.3687`, DSpark `0.3367 / 3.3246`, and EAGLE-3 `0.2606 / 1.7757` for acceptance
+rate and length. The compact floors subtract six percentage points and 0.2 accepted tokens per
+verification step. The six-point rate margin is wider than the largest observed cross-environment
+shift of 0.0506. Warm compact runs completed in about 40 seconds. The DFlash2 and LiLiCorr nightly
+floors retain their earlier three-point rate and 0.2 length margins.
+
+DFlash's token-acceptance rate is lower because the denominator counts every token proposed in each
+seven-token draft block. It accepted 219 of 1,093 proposed tokens over 160 verification steps,
+equivalent to 1.369 accepted draft tokens per step and a mean acceptance length of 2.3687. The floor
+is intended to detect regressions from that deterministic baseline, not to define a quality target.
+
To run with stdout/stderr display in real time (verbose output, but useful for debugging):
```shell
diff --git a/tools/server/tests/pytest.ini b/tools/server/tests/pytest.ini
index 6df308df7..42d1ab9cc 100644
--- a/tools/server/tests/pytest.ini
+++ b/tools/server/tests/pytest.ini
@@ -1,4 +1,5 @@
[pytest]
markers =
+ heavy: requires a dedicated runner and large model downloads
slow: marks tests as slow (deselect with '-m "not slow"')
serial
diff --git a/tools/server/tests/unit/test_spec_methods.py b/tools/server/tests/unit/test_spec_methods.py
new file mode 100644
index 000000000..eb8e879b7
--- /dev/null
+++ b/tools/server/tests/unit/test_spec_methods.py
@@ -0,0 +1,213 @@
+import os
+import re
+from dataclasses import dataclass
+
+import pytest
+from utils import ServerProcess, download_file
+
+# Runs each speculative method greedily and checks that it still drafts and accepts as expected.
+# accept rate = accepted / drafted, accept len = 1 + accepted / verification steps (from /metrics).
+# DFlash2 and LiLiCorr use --spec-type draft-dflash; the draft GGUF metadata selects the variant.
+# Needs real models and a GPU. Use SPEC_METHODS=compact, nightly, all, or a comma-separated method list.
+
+HF = "https://huggingface.co"
+MODEL_REPO = "ggml-org/spec-for-testing-GGUF"
+MODEL_REVISION = "a0736c8e98d3fe580352dacffa3fbd5e9c22a437"
+
+
+@dataclass(frozen=True)
+class SpecCase:
+ name: str
+ spec_type: str
+ target: str
+ draft: str | None # None when the heads are part of the target (MTP)
+ n_max: int
+ min_accept_rate: float
+ min_accept_len: float
+
+
+CASES = [
+ # Compact floors are the minimum of three identical GB10 calibration runs
+ # against current master minus six acceptance-rate percentage points and
+ # 0.2 accepted tokens per verification step.
+ SpecCase(
+ name="mtp",
+ spec_type="draft-mtp",
+ target="Qwen3.5-4B-MTP-Q4_K_M.gguf",
+ draft=None,
+ n_max=3,
+ min_accept_rate=0.6040,
+ min_accept_len=2.7766,
+ ),
+ SpecCase(
+ name="dflash",
+ spec_type="draft-dflash",
+ target="Qwen3-4B-Q4_K_M.gguf",
+ draft="dflash-Qwen3-4B-Q4_K_M.gguf",
+ n_max=7,
+ min_accept_rate=0.1404,
+ min_accept_len=2.1687,
+ ),
+ SpecCase(
+ name="dspark",
+ spec_type="draft-dspark",
+ target="Qwen3-4B-Q4_K_M.gguf",
+ draft="dspark-Qwen3-4B-Q4_K_M.gguf",
+ n_max=7,
+ min_accept_rate=0.2767,
+ min_accept_len=3.1246,
+ ),
+ SpecCase(
+ name="eagle3",
+ spec_type="draft-eagle3",
+ target="Qwen3-4B-Q4_K_M.gguf",
+ draft="eagle3-Qwen3-4B-Q8_0.gguf",
+ n_max=3,
+ min_accept_rate=0.2006,
+ min_accept_len=1.5757,
+ ),
+ # Nightly floors retain the earlier three-point rate and 0.2 length margins.
+ SpecCase(
+ name="dflash2",
+ spec_type="draft-dflash",
+ target="Qwen3.8-27B-Q4_K_M.gguf",
+ draft="dflash2-Qwen3.8-27B-Q8_0.gguf",
+ n_max=7,
+ min_accept_rate=0.5180,
+ min_accept_len=4.5414,
+ ),
+ SpecCase(
+ name="lilicorr",
+ spec_type="draft-dflash",
+ target="NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf",
+ draft="lilicorr-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-NVFP4.gguf",
+ n_max=7,
+ min_accept_rate=0.6399,
+ min_accept_len=5.4444,
+ ),
+]
+
+COMPACT_METHODS = {"mtp", "dflash", "dspark", "eagle3"}
+NIGHTLY_METHODS = {"dflash2", "lilicorr"}
+
+PROMPTS = [
+ "Write a Python function that returns the n-th Fibonacci number using memoization.\n\n```python\n",
+ "The three main differences between TCP and UDP are:\n1.",
+ "Question: What is the capital of France, and what river flows through it?\nAnswer:",
+]
+
+N_PREDICT = 128
+
+
+def model_path(file: str) -> str:
+ """Download a pinned GGUF from the consolidated test repository."""
+ model_dir = os.environ.get("SPEC_MODEL_DIR", "./tmp")
+ os.makedirs(model_dir, exist_ok=True)
+ return download_file(
+ f"{HF}/{MODEL_REPO}/resolve/{MODEL_REVISION}/{file}",
+ os.path.join(model_dir, file),
+ )
+
+
+def make_server(case: SpecCase) -> ServerProcess:
+ server = ServerProcess()
+ server.model_hf_repo = None
+ server.model_hf_file = None
+ server.model_file = model_path(case.target)
+ server.n_ctx = 1024
+ server.n_slots = 1
+ server.n_gpu_layer = int(os.environ.get("N_GPU_LAYERS", 99))
+ server.fa = "on"
+ server.temperature = 0.0
+ server.server_metrics = True
+ server.spec_type = case.spec_type
+ server.spec_draft_n_gpu_layer = server.n_gpu_layer
+ if case.draft:
+ server.model_draft = model_path(case.draft)
+ if case.n_max:
+ server.spec_draft_n_max = case.n_max
+ return server
+
+
+def generate(server: ServerProcess) -> list[list[int]]:
+ outputs = []
+ for prompt in PROMPTS:
+ res = server.make_request(
+ "POST",
+ "/completion",
+ data={
+ "prompt": prompt,
+ "temperature": 0.0,
+ "top_k": 1,
+ "seed": 42,
+ "n_predict": N_PREDICT,
+ "cache_prompt": False,
+ "return_tokens": True,
+ },
+ )
+ assert res.status_code == 200
+ outputs.append(res.body["tokens"])
+ return outputs
+
+
+def spec_metrics(server: ServerProcess) -> dict[str, float]:
+ # Use exact cumulative counters: the equivalent server log values are rounded
+ # and emitted separately for each request.
+ res = server.make_request("GET", "/metrics")
+ assert res.status_code == 200
+ values = {}
+ for name in ("draft_tokens", "accepted_tokens", "drafts"):
+ m = re.search(
+ rf"^llamacpp:spec_decode_num_{name}_total (\S+)$", res.body, re.MULTILINE
+ )
+ assert m is not None, f"missing spec_decode_num_{name}_total in /metrics"
+ values[name] = float(m.group(1))
+ return values
+
+
+def selected_cases():
+ wanted = os.environ.get("SPEC_METHODS")
+ if not wanted:
+ return []
+ aliases = {
+ "all": {case.name for case in CASES},
+ "compact": COMPACT_METHODS,
+ "nightly": NIGHTLY_METHODS,
+ }
+ selected = aliases.get(
+ wanted, {name.strip() for name in wanted.split(",") if name.strip()}
+ )
+ unknown = selected - {c.name for c in CASES}
+ if unknown:
+ raise ValueError(f"unknown SPEC_METHODS: {', '.join(sorted(unknown))}")
+ return [case for case in CASES if case.name in selected]
+
+
+@pytest.mark.heavy
+@pytest.mark.slow
+@pytest.mark.parametrize("case", selected_cases(), ids=lambda c: c.name)
+def test_spec_method(case: SpecCase):
+ server = make_server(case)
+ try:
+ server.start(timeout_seconds=1800)
+ outputs = generate(server)
+ m = spec_metrics(server)
+ finally:
+ server.stop()
+
+ accept_rate = m["accepted_tokens"] / m["draft_tokens"] if m["draft_tokens"] else 0.0
+ accept_len = 1.0 + m["accepted_tokens"] / m["drafts"] if m["drafts"] else 1.0
+ print(
+ f"SPEC_RESULT {case.name}: drafted={m['draft_tokens']:.0f} accepted={m['accepted_tokens']:.0f} "
+ f"steps={m['drafts']:.0f} accept_rate={accept_rate:.4f} accept_len={accept_len:.4f}"
+ )
+
+ assert all(outputs), f"{case.name}: at least one request returned no tokens"
+ assert m["draft_tokens"] > 0, f"{case.name}: no tokens were drafted"
+ assert m["drafts"] > 0, f"{case.name}: no speculative verification steps ran"
+ assert (
+ accept_rate >= case.min_accept_rate
+ ), f"{case.name}: accept rate {accept_rate:.3f} < {case.min_accept_rate}"
+ assert (
+ accept_len >= case.min_accept_len
+ ), f"{case.name}: accept length {accept_len:.3f} < {case.min_accept_len}"
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index 0ca49192a..c1534fd72 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -103,6 +103,7 @@ class ServerProcess:
spec_draft_n_min: int | None = None
spec_draft_n_max: int | None = None
spec_draft_n_ubatch: int | None = None
+ spec_draft_n_gpu_layer: int | None = None
spec_synth_len: float | None = None
spec_synth_rates: List[float] | None = None
no_ui: bool | None = None
@@ -262,6 +263,8 @@ class ServerProcess:
server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
if self.spec_draft_n_ubatch:
server_args.extend(["--spec-draft-n-ubatch", self.spec_draft_n_ubatch])
+ if self.spec_draft_n_gpu_layer:
+ server_args.extend(["--spec-draft-ngl", self.spec_draft_n_gpu_layer])
if self.spec_synth_len is not None:
server_args.extend(["--spec-synth-len", self.spec_synth_len])
if self.spec_synth_rates is not None: