Commit 5f837145d for llama.cpp
commit 5f837145d86ecc62333f3b83835c64275fdaa90e
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Fri Nov 20 14:21:58 2026 +0100
server: add more audio tests (#32579)
diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txt
index 1a490b5cc..52c3f6249 100644
--- a/tools/server/tests/requirements.txt
+++ b/tools/server/tests/requirements.txt
@@ -6,4 +6,5 @@ numpy~=2.5.4; python_version >= "3.14"
openai~=2.14.0
prometheus-client~=0.20.0
requests~=2.32.3
+websockets~=15.0
wget~=3.2
diff --git a/tools/server/tests/unit/test_compat_oai_speech.py b/tools/server/tests/unit/test_compat_oai_speech.py
index 888942f02..e5b578c89 100644
--- a/tools/server/tests/unit/test_compat_oai_speech.py
+++ b/tools/server/tests/unit/test_compat_oai_speech.py
@@ -1,3 +1,4 @@
+import base64
import pytest
import requests
import struct
@@ -135,3 +136,78 @@ def test_speech_router():
server = ServerPreset.router()
server.start()
assert_wav(post_audio("/v1/audio/speech", model=model))
+
+
+#
+# tts only model, with voice cloning
+#
+
+def pocket_tts(path: str = "/v1/audio/speech", **extra) -> requests.Response:
+ body = {"input": "Hello there.", "seed": 42, "n_predict": 8, **extra}
+ return requests.post(server.make_url(path), json=body, timeout=DEFAULT_REQUEST_TIMEOUT)
+
+
+def test_tts_only():
+ global server
+ server = ServerPreset.tinypockettts()
+ server.start()
+ res = server.make_request("GET", "/props")
+ assert not res.body.get("voices") # this slice has no predefined voices
+ res = server.make_request("GET", "/v1/models")
+ assert res.body["data"][0]["architecture"]["output_modalities"] == ["audio"]
+ res = pocket_tts()
+ assert_wav(res)
+ assert res.content == pocket_tts().content
+
+
+def test_tts_only_speaker_ref():
+ global server
+ server = ServerPreset.tinypockettts()
+ server.start()
+ ref_a = make_wav(seconds=2.0, rate=24000, freq=220)
+ ref_b = make_wav(seconds=2.0, rate=24000, freq=440)
+ res = pocket_tts("/tts", speaker_ref_b64=base64.b64encode(ref_a).decode())
+ assert_wav(res)
+ assert res.content != pocket_tts("/tts").content
+ assert res.content != pocket_tts("/tts", speaker_ref_b64=base64.b64encode(ref_b).decode()).content
+ # the same reference as an uploaded file
+ fields = {"input": (None, "Hello there."), "seed": (None, "42"), "n_predict": (None, "8"), "speaker_ref": ("ref.wav", ref_a)}
+ res_file = requests.post(server.make_url("/tts"), files=fields, timeout=DEFAULT_REQUEST_TIMEOUT)
+ assert res_file.content == res.content
+ # /v1/audio/speech passes it through
+ assert pocket_tts(speaker_ref_b64=base64.b64encode(ref_a).decode()).content == res.content
+
+
+@pytest.mark.parametrize("speaker_ref_b64,message", [
+ ("!!!", "not valid base64"),
+ (base64.b64encode(b"not an audio file").decode(), "failed to decode"),
+])
+def test_tts_only_invalid_speaker_ref(speaker_ref_b64: str, message: str):
+ global server
+ server = ServerPreset.tinypockettts()
+ server.start()
+ res = pocket_tts("/tts", speaker_ref_b64=speaker_ref_b64)
+ assert res.status_code == 400
+ assert message in res.json()["error"]["message"]
+
+
+def test_tts_only_end_of_speech():
+ global server
+ server = ServerPreset.tinypockettts()
+ server.start()
+ # the end-of-speech head stops the generation before n_predict
+ res = pocket_tts(n_predict=16)
+ assert_wav(res)
+ assert res.content == pocket_tts(n_predict=1000).content
+
+
+@pytest.mark.parametrize("path", ["/v1/chat/completions", "/v1/audio/transcriptions"])
+def test_tts_only_no_text_output(path: str):
+ global server
+ server = ServerPreset.tinypockettts()
+ server.start()
+ if path == "/v1/audio/transcriptions":
+ res = requests.post(server.make_url(path), files={"file": ("audio.wav", make_wav())}, timeout=DEFAULT_REQUEST_TIMEOUT)
+ else:
+ res = requests.post(server.make_url(path), json={"messages": [{"role": "user", "content": "hello"}]}, timeout=DEFAULT_REQUEST_TIMEOUT)
+ assert res.status_code == 501
diff --git a/tools/server/tests/unit/test_compat_oai_transcription.py b/tools/server/tests/unit/test_compat_oai_transcription.py
new file mode 100644
index 000000000..eb3954282
--- /dev/null
+++ b/tools/server/tests/unit/test_compat_oai_transcription.py
@@ -0,0 +1,144 @@
+import base64
+import json
+import pytest
+import requests
+from utils import *
+
+server: ServerProcess
+
+MP3_PATH = os.path.join(os.path.dirname(__file__), "../../../mtmd/test-2.mp3")
+
+# full random sampling, so that the output depends on the audio
+SAMPLING_RANDOM = {"temperature": 1.0, "top_k": 0, "top_p": 1.0, "min_p": 0.0, "seed": 42}
+
+
+@pytest.fixture(autouse=True)
+def create_server():
+ global server
+ server = ServerPreset.tinylfm3omni()
+ server.jinja = True
+ server.reasoning = "off"
+
+
+def post_transcription(path: str = "/v1/audio/transcriptions", audio: bytes | None = None, filename: str = "audio.wav", **fields) -> requests.Response:
+ data = {"max_tokens": "8", **fields}
+ files = {"file": (filename, make_wav() if audio is None else audio)}
+ return requests.post(server.make_url(path), data=data, files=files, timeout=DEFAULT_REQUEST_TIMEOUT)
+
+
+def chat_with_audio(audio: bytes, fmt: str = "wav", extra: dict | None = None) -> str:
+ res = server.make_request("POST", "/v1/chat/completions", data={
+ "max_tokens": 8,
+ "messages": [{"role": "user", "content": [
+ {"type": "input_audio", "input_audio": {"data": base64.b64encode(audio).decode(), "format": fmt}},
+ ]}],
+ **(extra or {}),
+ })
+ assert res.status_code == 200, res.body
+ return res.body["choices"][0]["message"]["content"]
+
+
+@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
+def test_transcription(path: str):
+ server.start()
+ res = post_transcription(path)
+ assert res.status_code == 200
+ body = res.json()
+ assert body["type"] == "transcript.text.done"
+ assert len(body["text"]) > 0
+ assert 0 < body["usage"]["output_tokens"] <= 8
+ assert body["usage"]["input_tokens"] > 0
+
+
+def test_transcription_mp3():
+ server.start()
+ with open(MP3_PATH, "rb") as f:
+ res = post_transcription(audio=f.read(), filename="audio.mp3")
+ assert res.status_code == 200
+ assert len(res.json()["text"]) > 0
+
+
+def test_transcription_stream():
+ server.start()
+ res = post_transcription(stream="true")
+ assert res.status_code == 200
+ lines = [line[len("data: "):] for line in res.text.splitlines() if line.startswith("data: ")]
+ assert lines[-1] == "[DONE]"
+ events = [json.loads(line) for line in lines[:-1]]
+ assert len(events) > 1
+ assert all(ev["type"] == "transcript.text.delta" for ev in events[:-1])
+ assert events[-1]["type"] == "transcript.text.done"
+ text = "".join(ev["delta"] for ev in events[:-1])
+ assert text == events[-1]["text"]
+ assert text == post_transcription().json()["text"]
+
+
+def test_transcription_language():
+ server.start()
+ # the LFM3 template puts the language in the system prompt
+ n_default = post_transcription().json()["usage"]["input_tokens"]
+ n_fr = post_transcription(language="fr").json()["usage"]["input_tokens"]
+ assert n_fr > n_default
+
+
+@pytest.mark.parametrize("fields,files,message", [
+ ({"response_format": "text"}, None, "response_format"),
+ ({}, {"file": ("audio.wav", b"not an audio file")}, None),
+ ({}, {"other": ("audio.wav", b"")}, None),
+])
+def test_transcription_invalid_request(fields: dict, files: dict | None, message: str | None):
+ server.start()
+ res = requests.post(server.make_url("/v1/audio/transcriptions"), data={"max_tokens": "8", **fields},
+ files=files or {"file": ("audio.wav", make_wav())}, timeout=DEFAULT_REQUEST_TIMEOUT)
+ assert res.status_code == 400
+ if message:
+ assert message in res.json()["error"]["message"]
+
+
+@pytest.mark.parametrize("preset", [
+ "tinyllama2", # no audio input
+ "tinyvoxtralrt", # live transcription only
+])
+def test_transcription_not_supported(preset: str):
+ global server
+ server = getattr(ServerPreset, preset)()
+ server.start()
+ assert post_transcription().status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
+
+
+def test_transcription_router():
+ global server
+ model = server.model_hf_repo
+ server = ServerPreset.router()
+ server.start()
+ res = post_transcription(model=model)
+ assert res.status_code == 200
+ assert res.json()["type"] == "transcript.text.done"
+
+
+def test_chat_input_audio():
+ server.start()
+ a = make_wav(freq=220)
+ b = make_wav(freq=880)
+ assert chat_with_audio(a, extra=SAMPLING_RANDOM) == chat_with_audio(a, extra=SAMPLING_RANDOM)
+ assert chat_with_audio(a, extra=SAMPLING_RANDOM) != chat_with_audio(b, extra=SAMPLING_RANDOM)
+
+
+def test_chat_input_audio_mp3():
+ server.start()
+ with open(MP3_PATH, "rb") as f:
+ assert len(chat_with_audio(f.read(), fmt="mp3")) > 0
+
+
+def test_chat_input_audio_no_audio_model():
+ global server
+ server = ServerPreset.tinyllama2()
+ server.start()
+ res = server.make_request("POST", "/v1/chat/completions", data={
+ "max_tokens": 8,
+ "messages": [{"role": "user", "content": [
+ {"type": "input_audio", "input_audio": {"data": base64.b64encode(make_wav()).decode(), "format": "wav"}},
+ ]}],
+ })
+ assert res.status_code != 200
+ assert "audio input is not supported" in res.body["error"]["message"]
diff --git a/tools/server/tests/unit/test_realtime.py b/tools/server/tests/unit/test_realtime.py
new file mode 100644
index 000000000..5af63a98e
--- /dev/null
+++ b/tools/server/tests/unit/test_realtime.py
@@ -0,0 +1,410 @@
+import base64
+import json
+import time
+import pytest
+from websockets.exceptions import ConnectionClosed
+from websockets.sync.client import connect, ClientConnection
+from websockets.sync.connection import Connection
+from utils import *
+
+server: ServerProcess
+
+
+@pytest.fixture(autouse=True)
+def create_server():
+ global server
+ server = ServerPreset.tinylfm3omni()
+ server.jinja = True
+ server.reasoning = "off"
+
+
+def ws_connect(path: str = "/v1/realtime") -> ClientConnection:
+ return connect(server.make_url(path).replace("http://", "ws://", 1), max_size=None, open_timeout=DEFAULT_REQUEST_TIMEOUT)
+
+
+def send(ws: Connection, type: str, **fields):
+ ws.send(json.dumps({"type": type, **fields}))
+
+
+def send_audio(ws: Connection, pcm: bytes):
+ send(ws, "input_audio_buffer.append", audio=base64.b64encode(pcm).decode())
+
+
+def send_text(ws: Connection, text: str):
+ send(ws, "conversation.item.create", item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]})
+
+
+def recv(ws: Connection) -> dict:
+ return json.loads(ws.recv(timeout=DEFAULT_REQUEST_TIMEOUT))
+
+
+def recv_until(ws: Connection, type: str) -> list[dict]:
+ events = []
+ while True:
+ ev = recv(ws)
+ events.append(ev)
+ if ev["type"] == type:
+ return events
+ assert ev["type"] != "error", ev
+
+
+def recv_all(ws: Connection) -> list[dict]:
+ """ read events until the server closes the socket """
+ events = []
+ try:
+ while True:
+ events.append(recv(ws))
+ except ConnectionClosed:
+ return events
+
+
+def get_transcripts(events: list[dict]) -> list[tuple[str, str]]:
+ """ (item_id, transcript) of each completed item, checked against its deltas """
+ deltas: dict[str, str] = {}
+ out = []
+ for ev in events:
+ assert ev["type"] != "error", ev
+ if ev["type"] == "conversation.item.input_audio_transcription.delta":
+ deltas[ev["item_id"]] = deltas.get(ev["item_id"], "") + ev["delta"]
+ elif ev["type"] == "conversation.item.input_audio_transcription.completed":
+ assert ev["transcript"] == deltas.get(ev["item_id"], "")
+ out.append((ev["item_id"], ev["transcript"]))
+ return out
+
+
+def session_update(ws: Connection, **session) -> dict:
+ send(ws, "session.update", session=session)
+ return recv_until(ws, "session.updated")[-1]["session"]
+
+
+#
+# live transcription
+#
+
+LIVE_ASR = [
+ # a chat model with an "asr" chat template, live only in sessions opened with ?intent=transcription
+ ("tinylfm3omni", "/v1/realtime?intent=transcription"),
+ # no chat, the text follows the audio by a few tokens
+ ("tinyvoxtralrt", "/v1/realtime"),
+]
+
+
+def start_live_asr(preset: str):
+ global server
+ if preset != "tinylfm3omni":
+ server = getattr(ServerPreset, preset)()
+ server.start()
+
+
+@pytest.mark.parametrize("preset,path", LIVE_ASR)
+def test_live_asr(preset: str, path: str):
+ start_live_asr(preset)
+ with ws_connect(path) as ws:
+ session = recv(ws)
+ assert session["type"] == "session.created"
+ assert session["session"]["object"] == "realtime.transcription_session"
+ assert session["session"]["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 16000}
+ send_audio(ws, make_pcm16())
+ send(ws, "input_audio_buffer.commit")
+ send_audio(ws, make_pcm16(freq=880))
+ send(ws, "input_audio_buffer.commit", final=True)
+ events = recv_all(ws)
+ committed = [ev for ev in events if ev["type"] == "input_audio_buffer.committed"]
+ assert len(committed) == 1
+ assert committed[0]["previous_item_id"] is None
+ transcripts = get_transcripts(events)
+ assert len(transcripts) == 2
+ assert transcripts[0][0] == committed[0]["item_id"]
+ assert transcripts[1][0] != transcripts[0][0]
+ assert all(len(text) > 0 for _, text in transcripts)
+
+
+@pytest.mark.parametrize("preset,path", LIVE_ASR)
+def test_live_asr_chunked_audio(preset: str, path: str):
+ start_live_asr(preset)
+ pcm = make_pcm16()
+ results = []
+ for n_chunks in [1, 7]:
+ size = len(pcm) // n_chunks // 2 * 2
+ with ws_connect(path) as ws:
+ recv(ws)
+ for i in range(0, len(pcm), size):
+ send_audio(ws, pcm[i:i + size])
+ send(ws, "input_audio_buffer.commit", final=True)
+ results.append([text for _, text in get_transcripts(recv_all(ws))])
+ assert results[0] == results[1]
+
+
+@pytest.mark.parametrize("preset,path", LIVE_ASR)
+def test_live_asr_input_rate(preset: str, path: str):
+ start_live_asr(preset)
+ with ws_connect(path) as ws:
+ recv(ws)
+ send(ws, "session.update", session={"audio": {"input": {"format": {"type": "audio/pcm", "rate": 4000}}}})
+ assert "between 8000 and 192000" in recv_until(ws, "error")[-1]["error"]["message"]
+ send(ws, "session.update", session={"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}})
+ assert recv_until(ws, "session.updated")[-1]["session"]["audio"]["input"]["format"]["rate"] == 24000
+ send_audio(ws, make_pcm16(rate=24000))
+ send(ws, "input_audio_buffer.commit", final=True)
+ transcripts = get_transcripts(recv_all(ws))
+ assert len(transcripts) == 1
+ assert len(transcripts[0][1]) > 0
+
+
+@pytest.mark.parametrize("preset,path", LIVE_ASR)
+def test_live_asr_invalid_events(preset: str, path: str):
+ start_live_asr(preset)
+ with ws_connect(path) as ws:
+ recv(ws)
+ ws.send("not json")
+ assert recv_until(ws, "error")[-1]["error"]["message"] == "invalid JSON"
+ send(ws, "response.create")
+ assert "unknown event type" in recv_until(ws, "error")[-1]["error"]["message"]
+ send(ws, "input_audio_buffer.clear")
+ recv_until(ws, "input_audio_buffer.cleared")
+ # the session still works
+ send_audio(ws, make_pcm16())
+ send(ws, "input_audio_buffer.commit", final=True)
+ assert len(get_transcripts(recv_all(ws))) == 1
+
+
+def test_live_asr_depends_on_audio():
+ # the random weights of this model give a different text for each input
+ start_live_asr("tinyvoxtralrt")
+ results = []
+ for freq in [220, 880, 220]:
+ with ws_connect() as ws:
+ recv(ws)
+ send_audio(ws, make_pcm16(freq=freq))
+ send(ws, "input_audio_buffer.commit", final=True)
+ results.append(get_transcripts(recv_all(ws))[0][1])
+ assert results[0] != results[1]
+ assert results[0] == results[2]
+
+
+@pytest.mark.parametrize("preset", ["tinypockettts", "tinyllama2"])
+def test_realtime_not_supported(preset: str):
+ global server
+ server = getattr(ServerPreset, preset)()
+ server.start()
+ with ws_connect() as ws:
+ events = recv_all(ws)
+ assert len(events) == 1
+ assert events[0]["type"] == "error"
+ assert "does not support realtime" in events[0]["error"]["message"]
+
+
+#
+# realtime conversation
+#
+
+def test_conversation_text():
+ server.start()
+ with ws_connect() as ws:
+ session = recv(ws)
+ assert session["type"] == "session.created"
+ assert session["session"]["object"] == "realtime.session"
+ assert session["session"]["audio"]["output"]["format"]["rate"] > 0
+ session_update(ws, output_modalities=["text"], max_output_tokens=8)
+ send_text(ws, "hello")
+ user_item = recv_until(ws, "conversation.item.done")[-1]["item"]
+ assert user_item["role"] == "user"
+ assert user_item["content"] == [{"type": "input_text", "text": "hello"}]
+ send(ws, "response.create")
+ events = recv_until(ws, "response.done")
+
+ types = [ev["type"] for ev in events]
+ n_deltas = types.count("response.output_text.delta")
+ assert n_deltas == 8
+ assert types == [
+ "response.created",
+ "response.output_item.added",
+ "response.content_part.added",
+ *["response.output_text.delta"] * n_deltas,
+ "response.output_text.done",
+ "response.content_part.done",
+ "response.output_item.done",
+ "response.done",
+ ]
+ response = events[-1]["response"]
+ assert response["status"] == "completed"
+ assert response["id"] == events[0]["response"]["id"]
+ assert response["usage"]["output_tokens"] == 8
+ text = "".join(ev["delta"] for ev in events if ev["type"] == "response.output_text.delta")
+ assert response["output"][0]["role"] == "assistant"
+ assert response["output"][0]["content"] == [{"type": "output_text", "text": text}]
+
+
+def test_conversation_audio():
+ server.start()
+ with ws_connect() as ws:
+ recv(ws)
+ session_update(ws, output_modalities=["audio"], max_output_tokens=8)
+ send_audio(ws, make_pcm16())
+ # response.create commits the pending audio
+ send(ws, "response.create")
+ events = recv_until(ws, "response.done")
+ types = [ev["type"] for ev in events]
+ assert types[:3] == ["input_audio_buffer.committed", "conversation.item.added", "conversation.item.done"]
+ assert events[2]["item"]["content"] == [{"type": "input_audio"}]
+ assert "response.output_text.delta" not in types
+ audio = b"".join(base64.b64decode(ev["delta"]) for ev in events if ev["type"] == "response.output_audio.delta")
+ assert len(audio) > 0 and len(audio) % 2 == 0
+ response = events[-1]["response"]
+ assert response["status"] == "completed"
+ assert response["output"][0]["content"][0]["type"] == "output_audio"
+
+
+def test_conversation_multi_turn():
+ server.start()
+ with ws_connect() as ws:
+ recv(ws)
+ session_update(ws, output_modalities=["text"], max_output_tokens=4)
+ send_text(ws, "hello")
+ recv_until(ws, "conversation.item.done")
+ send(ws, "response.create")
+ first = recv_until(ws, "response.done")[-1]["response"]
+ send_audio(ws, make_pcm16())
+ send(ws, "input_audio_buffer.commit")
+ committed = recv_until(ws, "input_audio_buffer.committed")[-1]
+ assert committed["previous_item_id"] == first["output"][0]["id"]
+ send(ws, "response.create")
+ second = recv_until(ws, "response.done")[-1]["response"]
+ assert second["status"] == "completed"
+ assert second["id"] != first["id"]
+
+
+def test_conversation_cancel():
+ server.start()
+ with ws_connect() as ws:
+ recv(ws)
+ session_update(ws, output_modalities=["text"])
+ send_text(ws, "hello")
+ send(ws, "response.create")
+ recv_until(ws, "response.output_text.delta")
+ send(ws, "response.cancel")
+ assert recv_until(ws, "response.done")[-1]["response"]["status"] == "cancelled"
+ # the session still works
+ send_text(ws, "hello again")
+ send(ws, "response.create", response={"max_output_tokens": 4})
+ response = recv_until(ws, "response.done")[-1]["response"]
+ assert response["status"] == "completed"
+ assert response["usage"]["output_tokens"] == 4
+
+
+def test_conversation_session_update():
+ server.start()
+ tools = [{"type": "function", "name": "get_weather", "description": "Get the weather",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}]
+ with ws_connect() as ws:
+ recv(ws)
+ session = session_update(ws, instructions="Be brief.", tools=tools, tool_choice="none", max_output_tokens=16,
+ audio={"output": {"voice": {"id": "male"}}})
+ assert session["instructions"] == "Be brief."
+ assert session["tools"] == tools
+ assert session["tool_choice"] == "none"
+ assert session["max_output_tokens"] == 16
+ assert session["audio"]["output"]["voice"] == "male"
+ session = session_update(ws, max_output_tokens="inf", output_modalities=["text", "audio"])
+ assert session["max_output_tokens"] == "inf"
+ assert session["output_modalities"] == ["text", "audio"]
+
+
+@pytest.mark.parametrize("events,message", [
+ ([{"type": "response.create"}], "commit audio or create a message first"),
+ ([{"type": "input_audio_buffer.commit"}], "the buffer is empty"),
+ ([{"type": "conversation.item.create", "item": {"type": "message", "role": "assistant", "content": []}}], "only user messages"),
+ ([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_image"}]}}], "unsupported content type"),
+ ([{"type": "conversation.item.create", "item": {"type": "function_call_output", "call_id": "x", "output": "{}"}}], "must follow the response"),
+ ([{"type": "session.update", "session": {"type": "foo"}}], "unknown session type"),
+ ([{"type": "session.update", "session": {"output_modalities": []}}], "non-empty array"),
+ ([{"type": "session.update", "session": {"output_modalities": ["video"]}}], "unknown modality"),
+ ([{"type": "session.update", "session": {"max_output_tokens": 0}}], "max_output_tokens"),
+ ([{"type": "session.update", "session": {"tool_choice": "maybe"}}], "tool_choice"),
+ ([{"type": "session.update", "session": {"audio": {"output": {"voice": "alloy"}}}}], "unknown voice"),
+ ([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}},
+ {"type": "session.update", "session": {"instructions": "Be brief."}}], "cannot change after"),
+ ([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}},
+ {"type": "session.update", "session": {"type": "transcription"}}], "cannot change after"),
+])
+def test_conversation_invalid_events(events: list[dict], message: str):
+ server.start()
+ with ws_connect() as ws:
+ recv(ws)
+ for ev in events:
+ ws.send(json.dumps(ev))
+ err = recv_until(ws, "error")[-1]
+ assert message in err["error"]["message"]
+
+
+def test_transcription_session():
+ server.start()
+ with ws_connect() as ws:
+ recv(ws)
+ session = session_update(ws, type="transcription", max_output_tokens=8,
+ audio={"input": {"transcription": {"language": "fr"}}})
+ assert session["object"] == "realtime.transcription_session"
+ assert session["audio"]["input"]["transcription"]["language"] == "fr"
+ send_text(ws, "hello")
+ assert "not supported in a transcription session" in recv_until(ws, "error")[-1]["error"]["message"]
+ transcripts = []
+ for freq in [440, 880]:
+ send_audio(ws, make_pcm16(freq=freq))
+ send(ws, "input_audio_buffer.commit")
+ committed = recv_until(ws, "input_audio_buffer.committed")[-1]
+ events = recv_until(ws, "conversation.item.input_audio_transcription.completed")
+ transcripts += get_transcripts(events)
+ assert transcripts[-1][0] == committed["item_id"]
+ assert len(transcripts) == 2
+ assert all(len(text) > 0 for _, text in transcripts)
+
+
+#
+# slot and server state
+#
+
+@pytest.mark.parametrize("path", ["/v1/realtime", "/v1/realtime?intent=transcription"])
+def test_realtime_disconnect_frees_slot(path: str):
+ server.n_slots = 1
+ server.start()
+ with ws_connect(path) as ws:
+ recv(ws)
+ send_audio(ws, make_pcm16())
+ if path == "/v1/realtime":
+ send(ws, "response.create")
+ recv_until(ws, "response.output_item.added")
+ # the socket is closed in the middle of the session
+ res = server.make_request("POST", "/v1/chat/completions", data={
+ "max_tokens": 4,
+ "messages": [{"role": "user", "content": "hello"}],
+ })
+ assert res.status_code == 200
+
+
+def test_realtime_router():
+ global server
+ model = server.model_hf_repo
+ server = ServerPreset.router()
+ server.start()
+ with ws_connect(f"/v1/realtime?model={model}") as ws:
+ assert recv(ws)["type"] == "session.created"
+ session_update(ws, output_modalities=["text"], max_output_tokens=4)
+ send_text(ws, "hello")
+ send(ws, "response.create")
+ assert recv_until(ws, "response.done")[-1]["response"]["status"] == "completed"
+
+
+def test_realtime_sleep():
+ server.sleep_idle_seconds = 1
+ server.start()
+ start = time.time()
+ while not server.make_request("GET", "/props").body["is_sleeping"]:
+ assert time.time() - start < 10, "server did not go to sleep"
+ time.sleep(0.1)
+ # the session wakes the server up
+ with ws_connect() as ws:
+ assert recv(ws)["type"] == "session.created"
+ session_update(ws, output_modalities=["text"], max_output_tokens=4)
+ send_text(ws, "hello")
+ send(ws, "response.create")
+ assert recv_until(ws, "response.done")[-1]["response"]["status"] == "completed"
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index 6a30e02d1..4d936aa2f 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -5,6 +5,10 @@
import subprocess
import os
+import io
+import math
+import struct
+import wave
TMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tmp")
import re
@@ -726,6 +730,36 @@ class ServerPreset:
server.seed = 42
return server
+ @staticmethod
+ def tinypockettts() -> ServerProcess:
+ # tts only, with voice cloning
+ server = ServerProcess()
+ server.offline = True # will be downloaded by load_all()
+ # mmproj is already provided by HF registry API
+ server.model_hf_file = None
+ server.model_hf_repo = "ggml-org-dev/tinyPocketTTS-for-testing-gguf:Q8_0"
+ server.model_alias = "tinypockettts"
+ server.n_ctx = 1024
+ server.n_batch = 512
+ server.n_slots = 1
+ server.seed = 42
+ return server
+
+ @staticmethod
+ def tinyvoxtralrt() -> ServerProcess:
+ # live transcription only, no chat
+ server = ServerProcess()
+ server.offline = True # will be downloaded by load_all()
+ # mmproj is already provided by HF registry API
+ server.model_hf_file = None
+ server.model_hf_repo = "ggml-org-dev/tinyVoxtral-Realtime-for-testing-gguf:Q8_0"
+ server.model_alias = "tinyvoxtralrt"
+ server.n_ctx = 1024
+ server.n_batch = 512
+ server.n_slots = 1
+ server.seed = 42
+ return server
+
@staticmethod
def router() -> ServerProcess:
server = ServerProcess()
@@ -832,3 +866,19 @@ def get_lan_ip(server: ServerProcess) -> str | None:
def is_slow_test_allowed():
return os.environ.get("SLOW_TESTS") == "1" or os.environ.get("SLOW_TESTS") == "ON"
+
+
+def make_pcm16(seconds: float = 1.0, rate: int = 16000, freq: float = 440.0) -> bytes:
+ """ sine tone as raw PCM16 mono samples """
+ n = int(seconds * rate)
+ return struct.pack(f"<{n}h", *(int(0.3 * 32767 * math.sin(2 * math.pi * freq * i / rate)) for i in range(n)))
+
+
+def make_wav(seconds: float = 1.0, rate: int = 16000, freq: float = 440.0) -> bytes:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(rate)
+ w.writeframes(make_pcm16(seconds, rate, freq))
+ return buf.getvalue()