Commit 2b66bfbf7 for llama.cpp
commit 2b66bfbf7107e43f5cd2fe99802ff6e872e4c4d0
Author: Aleksander Grygier <aleksander.grygier@gmail.com>
Date: Tue Nov 10 22:34:46 2026 +0100
ui: show audio in Discover for models whose mmproj has an audio encoder (#31837)
* ui: show audio in Discover for models whose mmproj has an audio encoder
Discover took audio only from the pipeline tag, so omni models tagged
any-to-any, such as Gemma 4 and LFM 3 Omni, showed vision but never audio,
though their mmproj carries an audio encoder.
Read the encoders from the mmproj's GGUF metadata instead: a ranged read of
its first 64 KB, once per repo, for clip.has_vision_encoder and
clip.has_audio_encoder. The Discover list and details now show audio when
either the pipeline tag or the mmproj says so.
Assisted by: Claude (Opus 5.5 Medium)
* ui: take Discover modalities from the mmproj's encoders in one place
Review fixes for the mmproj header read:
- drop vision for audio-only mmprojs: the converter writes
clip.has_vision_encoder only when true, so a header read in full without
it has no vision encoder; a header cut short leaves vision as it was
- accept only a 206 to the ranged read, so a server that ignores Range
never streams the whole mmproj
- encode the mmproj path in the resolve URL
- derive the list and details modalities with one HuggingFaceService
helper, so both views agree
- point the key comment at tools/mtmd/mtmd-impl.h, where the keys live now
Assisted-by: Claude (Opus 5.5 Medium)
* ui: keep a gated repo's mmproj from being read again on every mount
A gated, private or missing mmproj answers 401, 403 or 404 to every read,
so caching nothing made each mount of its Discover row request it again and
log an error. Cache it as unknown instead: no audio, and vision from the
mmproj being there, as before the header was read.
Assisted-by: Claude (Opus 5.5 Medium)
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte
index 8f7e50da6..7d957e983 100644
--- a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte
+++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte
@@ -35,17 +35,23 @@
return tags.find((t) => t.startsWith('license:'))?.replace('license:', '') ?? null;
});
- // Capabilities derived from HF metadata. Vision comes from an mmproj sidecar
- // or a multimodal pipeline tag; tool use / reasoning from the chat template.
- let hasMmproj = $derived(
- files.some(
+ // Capabilities derived from HF metadata. Vision and audio come from the
+ // pipeline tag and the mmproj; tool use / reasoning from the chat template.
+ let mmprojPath = $derived(
+ files.find(
(f) => HuggingFaceService.extractQuantMeta(f.path)?.sidecar === ModelAuxSidecar.MMPROJ
- )
+ )?.path
+ );
+ let modalities = $derived(
+ HuggingFaceService.getModalities(modelId, details?.pipeline_tag, mmprojPath)
);
- let hasVision = $derived(hasMmproj || details?.pipeline_tag === 'image-text-to-text');
let hasTools = $derived(detectToolUseSupport(gguf?.chat_template ?? ''));
let hasReasoning = $derived(detectThinkingSupport(gguf?.chat_template ?? ''));
+ $effect(() => {
+ if (mmprojPath) void HuggingFaceService.getMmprojEncoders(modelId, mmprojPath);
+ });
+
let bitDepthRows = $derived.by<ModelBitDepthRow[]>(() => {
const rows = new SvelteMap<number, HfModelSibling[]>();
@@ -84,8 +90,8 @@
{gguf}
{hasReasoning}
{hasTools}
- {hasVision}
{licenseTag}
+ {modalities}
{modelId}
/>
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte
index 868765079..b2020fc31 100644
--- a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte
+++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte
@@ -8,6 +8,7 @@
import { Button } from '$lib/components/ui/button';
import { HuggingFaceService } from '$lib/services';
import type { HfModelDetailInfo, HfModelGguf } from '$lib/types/huggingface';
+ import type { ModelModalities } from '$lib/types/models';
import { orgOf } from '$lib/utils';
interface Props {
@@ -16,13 +17,21 @@
gguf?: HfModelGguf;
baseModels: string[];
licenseTag: string | null;
- hasVision: boolean;
+ modalities: ModelModalities;
hasTools: boolean;
hasReasoning: boolean;
}
- let { baseModels, details, gguf, hasReasoning, hasTools, hasVision, licenseTag, modelId }: Props =
- $props();
+ let {
+ baseModels,
+ details,
+ gguf,
+ hasReasoning,
+ hasTools,
+ licenseTag,
+ modalities,
+ modelId
+ }: Props = $props();
// Avatar shows the base model's org (e.g. the Qwen logo for a ggml-org GGUF)
// with the quant org as a corner badge when they differ.
@@ -50,7 +59,7 @@
<ModelCapabilityIcons
gapClass="gap-2"
iconSize="h-4 w-4"
- modalities={{ audio: false, video: false, vision: hasVision }}
+ {modalities}
supportsThinking={hasReasoning}
supportsToolUse={hasTools}
/>
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte
index 1eee85ea0..97c4b04ba 100644
--- a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte
+++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte
@@ -5,7 +5,7 @@
import ModelOrgAvatar from '../../ModelOrgAvatar.svelte';
import { Pause, Play, X } from '@lucide/svelte';
import { ActionIcon } from '$lib/components/app';
- import { HF_MMPROJ_FILENAME_TOKEN, HF_MODALITY_PIPELINE_TAGS, MODEL_ID } from '$lib/constants';
+ import { HF_MMPROJ_FILENAME_TOKEN, MODEL_ID } from '$lib/constants';
import { ModelDraftSidecar, ModelRowDownloadState } from '$lib/enums';
import { HuggingFaceService } from '$lib/services';
import { modelsDiscoverStore, modelsStore } from '$lib/stores';
@@ -96,19 +96,19 @@
let supportsToolUse = $derived(detectToolUseSupport(model.gguf?.chat_template ?? ''));
- // Modalities derived from HF metadata: vision from an mmproj sidecar or a
- // multimodal pipeline tag, audio/video from their pipeline tags.
- let modalities = $derived.by<ModelModalities>(() => {
- const tag = model.pipeline_tag ?? '';
- const vision =
- HF_MODALITY_PIPELINE_TAGS.vision.includes(tag) ||
- Boolean(
- model.siblings?.some((s) => s.rfilename.toLowerCase().includes(HF_MMPROJ_FILENAME_TOKEN))
- );
- const audio = HF_MODALITY_PIPELINE_TAGS.audio.includes(tag);
- const video = HF_MODALITY_PIPELINE_TAGS.video.includes(tag);
-
- return { audio, video, vision };
+ let mmprojPath = $derived(
+ model.siblings?.find((s) => s.rfilename.toLowerCase().includes(HF_MMPROJ_FILENAME_TOKEN))
+ ?.rfilename
+ );
+
+ // Modalities derived from HF metadata: the pipeline tag and the mmproj.
+ let modalities = $derived<ModelModalities>(
+ HuggingFaceService.getModalities(model.id, model.pipeline_tag, mmprojPath)
+ );
+
+ // the mmproj's header says whether it also hears; read once per repo
+ $effect(() => {
+ if (mmprojPath) void HuggingFaceService.getMmprojEncoders(model.id, mmprojPath);
});
// Draft sidecars (mtp, dflash, dspark, eagle3) the repo offers, badged with no
diff --git a/tools/ui/src/lib/constants/huggingface.constants.ts b/tools/ui/src/lib/constants/huggingface.constants.ts
index 47b6a89be..9a9ce6885 100644
--- a/tools/ui/src/lib/constants/huggingface.constants.ts
+++ b/tools/ui/src/lib/constants/huggingface.constants.ts
@@ -43,6 +43,7 @@ export const HF_MODEL_LIST_EXPAND: readonly string[] = [
export const HF_MAIN_BRANCH = 'main';
export const HF_README_FILENAME = 'README.md';
export const HF_RAW_PATH = 'raw';
+export const HF_RESOLVE_PATH = 'resolve';
export const HF_TREE_PATH = 'tree';
// Pagination
@@ -55,7 +56,10 @@ export const HF_LINK_HEADER = 'Link';
export const HF_RETRY_ATTEMPTS = 3;
export const HF_RETRY_DELAY_MS = 1000;
+export const HF_HTTP_PARTIAL_CONTENT = 206;
export const HF_HTTP_NOT_FOUND = 404;
+/** Unauthorized, forbidden, not found: a gated, private or missing file, which a retry won't change. */
+export const HF_HTTP_NO_ACCESS_STATUSES: readonly number[] = [401, 403, HF_HTTP_NOT_FOUND];
export const HF_HTTP_SERVER_ERROR_MIN = 500;
// Search limits
@@ -129,6 +133,13 @@ export const HF_MODALITY_PIPELINE_TAGS: Readonly<
/** Filename token marking an mmproj sidecar sibling (unlocks vision / audio). */
export const HF_MMPROJ_FILENAME_TOKEN = 'mmproj';
+/** Bytes read from the start of an mmproj to reach its metadata, which comes before the tensors. */
+export const HF_MMPROJ_HEADER_BYTES = 65_536;
+
+/** mmproj metadata keys saying which encoders it carries (tools/mtmd/mtmd-impl.h). */
+export const GGUF_KEY_HAS_VISION_ENCODER = 'clip.has_vision_encoder';
+export const GGUF_KEY_HAS_AUDIO_ENCODER = 'clip.has_audio_encoder';
+
export const HF_TASK_TAGS: readonly string[] = [
'audio-classification',
'audio-to-audio',
diff --git a/tools/ui/src/lib/services/huggingface.service.ts b/tools/ui/src/lib/services/huggingface.service.ts
index 4cea6eb62..adc136c4a 100644
--- a/tools/ui/src/lib/services/huggingface.service.ts
+++ b/tools/ui/src/lib/services/huggingface.service.ts
@@ -1,4 +1,9 @@
-import { PATH_SEPARATOR, SETTINGS_KEYS } from '$lib/constants';
+import {
+ GGUF_KEY_HAS_AUDIO_ENCODER,
+ GGUF_KEY_HAS_VISION_ENCODER,
+ PATH_SEPARATOR,
+ SETTINGS_KEYS
+} from '$lib/constants';
import {
BYTE,
BYTE_LABEL,
@@ -24,19 +29,24 @@ import {
HF_GATED_TAG,
HF_GGUF_FILTER,
HF_GGUF_TAG,
+ HF_HTTP_NO_ACCESS_STATUSES,
HF_HTTP_NOT_FOUND,
+ HF_HTTP_PARTIAL_CONTENT,
HF_HTTP_SERVER_ERROR_MIN,
HF_LICENSE_TAG_PREFIX,
HF_LINK_HEADER,
HF_LINK_NEXT_REGEX,
HF_MAIN_BRANCH,
HF_MAX_LIMIT,
+ HF_MMPROJ_HEADER_BYTES,
+ HF_MODALITY_PIPELINE_TAGS,
HF_MODEL_LIST_EXPAND,
HF_PARAM_COUNT_REGEX,
HF_QUANT_PRECISION_REGEX,
HF_RAW_PATH,
HF_README_FILENAME,
HF_RECURSIVE_TREE_PARAM,
+ HF_RESOLVE_PATH,
HF_RETRY_ATTEMPTS,
HF_RETRY_DELAY_MS,
HF_SAFETENSORS_TAG,
@@ -69,12 +79,14 @@ import { HfEntryType, HfModelSort, SidecarForm } from '$lib/enums';
import { settingsStore } from '$lib/stores/settings/index.svelte';
import type {
HfCatalogEntry,
+ HfMmprojEncoders,
HfModelDetailInfo,
HfModelInfo,
HfModelSearchParams,
HfModelSibling
} from '$lib/types/huggingface';
-import { sidecarFromFileToken } from '$lib/utils';
+import type { ModelModalities } from '$lib/types/models';
+import { readGgufMetadata, sidecarFromFileToken } from '$lib/utils';
import { SvelteMap } from 'svelte/reactivity';
/** Fetch failure carrying the HTTP status, so retry logic tests the code instead of the message. */
@@ -107,6 +119,11 @@ export class HuggingFaceService {
private static detailsPending = new Map<string, Promise<HfModelDetailInfo | null>>();
+ // Encoders an mmproj carries, keyed by repo id: its header never changes, so one read per session.
+ private static mmprojEncodersCache = new SvelteMap<string, HfMmprojEncoders>();
+
+ private static mmprojEncodersPending = new Map<string, Promise<HfMmprojEncoders | null>>();
+
/**
* Map of quant token to its average bit-depth in bits-per-weight (bpw).
*/
@@ -160,8 +177,6 @@ export class HuggingFaceService {
return HuggingFaceService.detailsCache.get(hfRepoId);
}
- // GGUF Model Browsing
-
/**
* Collapse split GGUF shard sets (`-00001-of-00015.gguf`, ...) to their first
* shard, summing every shard's size so the kept entry reflects the whole
@@ -200,6 +215,8 @@ export class HuggingFaceService {
return result;
}
+ // GGUF Model Browsing
+
/**
* Extract the GGUF quantization token (e.g. `Q4_K_M`) and any sidecar type
* (`mtp`, `dflash`, `mmproj`, ...) from a `.gguf` filename.
@@ -355,8 +372,6 @@ export class HuggingFaceService {
return `${Math.floor(diffDays / DAYS_PER_YEAR)} ${YEARS_AGO_LABEL}`;
}
- // Model Details & Files
-
/**
* Format a min-max size range with one shared unit, e.g. `19.0-28.6 GB`.
*/
@@ -382,6 +397,8 @@ export class HuggingFaceService {
return `${fmt(min)}-${fmt(max)} ${unit}`;
}
+ // Model Details & Files
+
/**
* Avatar URL for an author (org or user). 404s when the author does not
* exist, so callers should provide a fallback.
@@ -544,6 +561,97 @@ export class HuggingFaceService {
return promise;
}
+ /**
+ * Which encoders a repo's mmproj carries, from the metadata at the start of the file: a ranged read of its
+ * first bytes, so the file itself is never downloaded. The pipeline tag can't say this: omni models are
+ * `any-to-any`, and their audio encoder lives in the same mmproj as the vision one. Null when the read fails.
+ */
+ static getMmprojEncoders(modelId: string, mmprojPath: string): Promise<HfMmprojEncoders | null> {
+ if (!HuggingFaceService.isEnabled()) return Promise.resolve(null);
+
+ const cached = HuggingFaceService.mmprojEncodersCache.get(modelId);
+
+ if (cached) return Promise.resolve(cached);
+
+ const pending = HuggingFaceService.mmprojEncodersPending.get(modelId);
+
+ if (pending) return pending;
+
+ // Do not encode the modelId, it contains slashes for author/name
+ const url = `${HF_BASE_URL}${PATH_SEPARATOR}${modelId}${PATH_SEPARATOR}${HF_RESOLVE_PATH}${PATH_SEPARATOR}${HF_MAIN_BRANCH}${PATH_SEPARATOR}${mmprojPath.split(PATH_SEPARATOR).map(encodeURIComponent).join(PATH_SEPARATOR)}`;
+ const promise = (async () => {
+ const response = await fetch(url, {
+ headers: { Range: `bytes=0-${HF_MMPROJ_HEADER_BYTES - 1}` }
+ });
+
+ // anything but a partial response would stream the whole file, which can be gigabytes
+ if (response.status !== HF_HTTP_PARTIAL_CONTENT) {
+ void response.body?.cancel();
+
+ // a gated, private or missing file answers the same on every read: keep what the repo says
+ if (HF_HTTP_NO_ACCESS_STATUSES.includes(response.status)) {
+ const unknown: HfMmprojEncoders = { audio: false, vision: null };
+
+ HuggingFaceService.mmprojEncodersCache.set(modelId, unknown);
+
+ return unknown;
+ }
+
+ throw new Error(`Failed to read ${mmprojPath}: ${response.status}`);
+ }
+
+ const { complete, metadata } = readGgufMetadata(await response.arrayBuffer());
+
+ if (metadata.size === 0) throw new Error(`${mmprojPath} has no GGUF metadata`);
+
+ // the converter writes these keys only when true, so one missing from a header read in full means no
+ // such encoder; a header cut short leaves vision unknown
+ const hasVision = metadata.get(GGUF_KEY_HAS_VISION_ENCODER) === true;
+ const encoders: HfMmprojEncoders = {
+ audio: metadata.get(GGUF_KEY_HAS_AUDIO_ENCODER) === true,
+ vision: hasVision || (complete ? false : null)
+ };
+
+ HuggingFaceService.mmprojEncodersCache.set(modelId, encoders);
+
+ return encoders;
+ })()
+ .catch((error: unknown) => {
+ // not cached: a rate limited or failed read should retry on the next mount
+ console.error(`Error reading the mmproj of ${modelId}:`, error);
+
+ return null;
+ })
+ .finally(() => HuggingFaceService.mmprojEncodersPending.delete(modelId));
+
+ HuggingFaceService.mmprojEncodersPending.set(modelId, promise);
+
+ return promise;
+ }
+
+ /**
+ * Modalities a repo offers, from its pipeline tag and its mmproj: vision from a vision tag or an mmproj
+ * (unless its header has no vision encoder, as in audio-only models), audio from an audio tag or
+ * an audio encoder in the mmproj (omni models are `any-to-any`), video from a video tag. Reactive: it
+ * updates once {@link getMmprojEncoders} has read the mmproj.
+ */
+ static getModalities(
+ modelId: string,
+ pipelineTag: string | null | undefined,
+ mmprojPath: string | undefined
+ ): ModelModalities {
+ const tag = pipelineTag ?? '';
+ const encoders = mmprojPath ? HuggingFaceService.mmprojEncodersCache.get(modelId) : undefined;
+
+ return {
+ audio: HF_MODALITY_PIPELINE_TAGS.audio.includes(tag) || encoders?.audio === true,
+ video: HF_MODALITY_PIPELINE_TAGS.video.includes(tag),
+ vision:
+ HF_MODALITY_PIPELINE_TAGS.vision.includes(tag) ||
+ (Boolean(mmprojPath) && encoders?.vision !== false)
+ };
+ }
+
static getModelUrl(modelId: string): string {
return `${HF_BASE_URL}${PATH_SEPARATOR}${modelId}`;
}
diff --git a/tools/ui/src/lib/types/huggingface.d.ts b/tools/ui/src/lib/types/huggingface.d.ts
index d578741c4..25a214841 100644
--- a/tools/ui/src/lib/types/huggingface.d.ts
+++ b/tools/ui/src/lib/types/huggingface.d.ts
@@ -172,6 +172,12 @@ export interface HfModelDetailInfo extends HfModelInfo {
}
/** A single entry in a model repository's file tree (`/tree` responses) */
+/** Encoders an mmproj's header says it carries; vision is null when the header was cut short before it said. */
+export interface HfMmprojEncoders {
+ audio: boolean;
+ vision: boolean | null;
+}
+
export interface HfModelSibling {
/** Relative path of the file or directory within the repo */
path: string;
diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts
index aa8ccc99c..1a3b18a22 100644
--- a/tools/ui/src/lib/types/index.ts
+++ b/tools/ui/src/lib/types/index.ts
@@ -47,6 +47,7 @@ export type {
HfModelGguf,
HfModelInfo,
HfModelSearchParams,
+ HfMmprojEncoders,
HfModelSibling,
HfModelSiblingRef
} from './huggingface';
diff --git a/tools/ui/src/lib/utils/gguf-metadata.ts b/tools/ui/src/lib/utils/gguf-metadata.ts
new file mode 100644
index 000000000..098548052
--- /dev/null
+++ b/tools/ui/src/lib/utils/gguf-metadata.ts
@@ -0,0 +1,153 @@
+/**
+ * Reads the metadata key/value pairs at the start of a GGUF file, from as many bytes of it as are given.
+ * Format: https://github.com/ggml-org/ggml/blob/master/docs/gguf.md
+ */
+
+const GGUF_MAGIC = 0x46554747; // "GGUF", little-endian
+
+enum GgufType {
+ UINT8 = 0,
+ INT8 = 1,
+ UINT16 = 2,
+ INT16 = 3,
+ UINT32 = 4,
+ INT32 = 5,
+ FLOAT32 = 6,
+ BOOL = 7,
+ STRING = 8,
+ ARRAY = 9,
+ UINT64 = 10,
+ INT64 = 11,
+ FLOAT64 = 12
+}
+
+const SCALAR_SIZE: Partial<Record<GgufType, number>> = {
+ [GgufType.BOOL]: 1,
+ [GgufType.FLOAT32]: 4,
+ [GgufType.FLOAT64]: 8,
+ [GgufType.INT8]: 1,
+ [GgufType.INT16]: 2,
+ [GgufType.INT32]: 4,
+ [GgufType.INT64]: 8,
+ [GgufType.UINT8]: 1,
+ [GgufType.UINT16]: 2,
+ [GgufType.UINT32]: 4,
+ [GgufType.UINT64]: 8
+};
+
+export type GgufValue = boolean | number | bigint | string;
+
+export interface GgufMetadata {
+ /** True when every pair was read; false when the buffer ended first or is not GGUF. */
+ complete: boolean;
+ metadata: Map<string, GgufValue>;
+}
+
+/**
+ * The scalar and string metadata of a GGUF file (arrays are skipped). A buffer cut short returns the pairs read
+ * before the cut; a buffer that is not GGUF returns an empty map.
+ */
+export function readGgufMetadata(buffer: ArrayBuffer): GgufMetadata {
+ const view = new DataView(buffer);
+ const decoder = new TextDecoder();
+ const metadata = new Map<string, GgufValue>();
+
+ let offset = 0;
+
+ const fits = (bytes: number) => offset + bytes <= view.byteLength;
+ const readString = (): string | null => {
+ if (!fits(8)) return null;
+
+ const length = Number(view.getBigUint64(offset, true));
+
+ if (!fits(8 + length)) return null;
+
+ const text = decoder.decode(new Uint8Array(buffer, offset + 8, length));
+
+ offset += 8 + length;
+
+ return text;
+ };
+ const readScalar = (type: GgufType): GgufValue | null => {
+ const size = SCALAR_SIZE[type];
+
+ if (size === undefined || !fits(size)) return null;
+
+ const at = offset;
+
+ offset += size;
+
+ switch (type) {
+ case GgufType.UINT8:
+ return view.getUint8(at);
+ case GgufType.INT8:
+ return view.getInt8(at);
+ case GgufType.UINT16:
+ return view.getUint16(at, true);
+ case GgufType.INT16:
+ return view.getInt16(at, true);
+ case GgufType.UINT32:
+ return view.getUint32(at, true);
+ case GgufType.INT32:
+ return view.getInt32(at, true);
+ case GgufType.FLOAT32:
+ return view.getFloat32(at, true);
+ case GgufType.BOOL:
+ return view.getUint8(at) !== 0;
+ case GgufType.UINT64:
+ return view.getBigUint64(at, true);
+ case GgufType.INT64:
+ return view.getBigInt64(at, true);
+ default:
+ return view.getFloat64(at, true);
+ }
+ };
+ // moves past an array's elements; false when the buffer ends inside it
+ const skipArray = (): boolean => {
+ if (!fits(12)) return false;
+
+ const type = view.getUint32(offset, true) as GgufType;
+ const count = Number(view.getBigUint64(offset + 4, true));
+
+ offset += 12;
+
+ for (let i = 0; i < count; i++) {
+ const value = type === GgufType.STRING ? readString() : readScalar(type);
+
+ if (value === null) return false;
+ }
+
+ return true;
+ };
+
+ // magic, version, tensor count, metadata count
+ if (!fits(24) || view.getUint32(0, true) !== GGUF_MAGIC) return { complete: false, metadata };
+
+ const count = Number(view.getBigUint64(16, true));
+
+ offset = 24;
+
+ for (let i = 0; i < count; i++) {
+ const key = readString();
+
+ if (key === null || !fits(4)) return { complete: false, metadata };
+
+ const type = view.getUint32(offset, true) as GgufType;
+
+ offset += 4;
+
+ if (type === GgufType.ARRAY) {
+ if (!skipArray()) return { complete: false, metadata };
+
+ continue;
+ }
+
+ const value = type === GgufType.STRING ? readString() : readScalar(type);
+
+ if (value === null) return { complete: false, metadata };
+
+ metadata.set(key, value);
+ }
+
+ return { complete: true, metadata };
+}
diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts
index 6fcc9229b..f8de87faf 100644
--- a/tools/ui/src/lib/utils/index.ts
+++ b/tools/ui/src/lib/utils/index.ts
@@ -366,6 +366,9 @@ export { detectToolUseSupport } from './chat-template-tool-detector';
// Model memory estimation
export { minMemoryTierGb } from './model-compatibility';
+// GGUF file metadata, e.g. which encoders an mmproj carries
+export { type GgufMetadata, type GgufValue, readGgufMetadata } from './gguf-metadata';
+
// Cryptography utilities
export { uuid } from './uuid';
diff --git a/tools/ui/tests/unit/gguf-metadata.test.ts b/tools/ui/tests/unit/gguf-metadata.test.ts
new file mode 100644
index 000000000..def354759
--- /dev/null
+++ b/tools/ui/tests/unit/gguf-metadata.test.ts
@@ -0,0 +1,98 @@
+import { readGgufMetadata } from '$lib/utils/gguf-metadata';
+import { describe, expect, it } from 'vitest';
+
+type Entry =
+ | [key: string, type: 'bool', value: boolean]
+ | [key: string, type: 'u32', value: number]
+ | [key: string, type: 'string', value: string]
+ | [key: string, type: 'string[]', value: string[]]
+ | [key: string, type: 'f32[]', value: number[]];
+
+/** A GGUF header with the given metadata and no tensors. */
+function gguf(entries: Entry[]): ArrayBuffer {
+ const bytes: number[] = [];
+ const u32 = (n: number) => bytes.push(...new Uint8Array(new Uint32Array([n]).buffer));
+ const u64 = (n: number) => bytes.push(...new Uint8Array(new BigUint64Array([BigInt(n)]).buffer));
+ const str = (s: string) => {
+ const encoded = new TextEncoder().encode(s);
+
+ u64(encoded.length);
+ bytes.push(...encoded);
+ };
+
+ u32(0x46554747);
+ u32(3);
+ u64(0);
+ u64(entries.length);
+
+ for (const [key, type, value] of entries) {
+ str(key);
+
+ if (type === 'bool') {
+ u32(7);
+ bytes.push(value ? 1 : 0);
+ } else if (type === 'u32') {
+ u32(4);
+ u32(value);
+ } else if (type === 'string') {
+ u32(8);
+ str(value);
+ } else if (type === 'string[]') {
+ u32(9);
+ u32(8);
+ u64(value.length);
+ value.forEach(str);
+ } else {
+ u32(9);
+ u32(6);
+ u64(value.length);
+ value.forEach((n) => bytes.push(...new Uint8Array(new Float32Array([n]).buffer)));
+ }
+ }
+
+ return new Uint8Array(bytes).buffer;
+}
+
+const MMPROJ: Entry[] = [
+ ['general.architecture', 'string', 'clip'],
+ ['clip.projector_type', 'string', 'gemma4'],
+ ['clip.vision.image_mean', 'f32[]', [0.5, 0.5, 0.5]],
+ ['tokenizer.ggml.tokens', 'string[]', ['<a>', '<b>']],
+ ['clip.has_vision_encoder', 'bool', true],
+ ['clip.has_audio_encoder', 'bool', true],
+ ['clip.audio.num_mel_bins', 'u32', 128]
+];
+
+describe('readGgufMetadata', () => {
+ it('reads scalars and strings, skipping arrays', () => {
+ const { complete, metadata } = readGgufMetadata(gguf(MMPROJ));
+
+ expect(complete).toBe(true);
+
+ expect(metadata.get('general.architecture')).toBe('clip');
+ expect(metadata.get('clip.has_vision_encoder')).toBe(true);
+ expect(metadata.get('clip.has_audio_encoder')).toBe(true);
+ expect(metadata.get('clip.audio.num_mel_bins')).toBe(128);
+ expect(metadata.has('clip.vision.image_mean')).toBe(false);
+ });
+
+ it('returns what it read before a buffer cut short', () => {
+ const full = gguf(MMPROJ);
+ const { complete, metadata } = readGgufMetadata(full.slice(0, full.byteLength - 20));
+
+ expect(complete).toBe(false);
+
+ expect(metadata.get('clip.has_vision_encoder')).toBe(true);
+ expect(metadata.has('clip.audio.num_mel_bins')).toBe(false);
+ });
+
+ it('returns nothing for a file that is not GGUF', () => {
+ expect(
+ readGgufMetadata(new TextEncoder().encode('not a gguf file at all...').buffer)
+ ).toStrictEqual({ complete: false, metadata: new Map() });
+ expect(readGgufMetadata(new ArrayBuffer(0))).toStrictEqual({
+ complete: false,
+ metadata: new Map()
+ });
+ });
+});
diff --git a/tools/ui/tests/unit/hf-mmproj-modalities.test.ts b/tools/ui/tests/unit/hf-mmproj-modalities.test.ts
new file mode 100644
index 000000000..a70c8296a
--- /dev/null
+++ b/tools/ui/tests/unit/hf-mmproj-modalities.test.ts
@@ -0,0 +1,148 @@
+import { HuggingFaceService } from '$lib/services/huggingface.service';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/** A GGUF header with the given bool metadata and no tensors. */
+function mmprojHeader(entries: Record<string, boolean>): ArrayBuffer {
+ const bytes: number[] = [];
+ const u32 = (n: number) => bytes.push(...new Uint8Array(new Uint32Array([n]).buffer));
+ const u64 = (n: number) => bytes.push(...new Uint8Array(new BigUint64Array([BigInt(n)]).buffer));
+
+ u32(0x46554747);
+ u32(3);
+ u64(0);
+ u64(Object.keys(entries).length);
+
+ for (const [key, value] of Object.entries(entries)) {
+ const encoded = new TextEncoder().encode(key);
+
+ u64(encoded.length);
+ bytes.push(...encoded);
+ u32(7);
+ bytes.push(value ? 1 : 0);
+ }
+
+ return new Uint8Array(bytes).buffer;
+}
+
+function respond(body: ArrayBuffer, status: number) {
+ const fetchMock = vi.fn(async () => new Response(body, { status }));
+
+ vi.stubGlobal('fetch', fetchMock);
+
+ return fetchMock;
+}
+
+describe('HuggingFaceService mmproj modalities', () => {
+ beforeEach(() => {
+ vi.spyOn(HuggingFaceService, 'isEnabled').mockReturnValue(true);
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+ });
+
+ it('takes audio from the pipeline tag and vision from the mmproj before its header is read', () => {
+ expect(HuggingFaceService.getModalities('org/unread', 'text-to-speech', 'mmproj.gguf')).toEqual(
+ { audio: true, video: false, vision: true }
+ );
+ expect(HuggingFaceService.getModalities('org/plain', 'text-generation', undefined)).toEqual({
+ audio: false,
+ video: false,
+ vision: false
+ });
+ });
+
+ it('adds audio for an omni mmproj, with a ranged read of an encoded path', async () => {
+ const fetchMock = respond(
+ mmprojHeader({ 'clip.has_audio_encoder': true, 'clip.has_vision_encoder': true }),
+ 206
+ );
+
+ await HuggingFaceService.getMmprojEncoders('org/omni', 'sub dir/mmproj#1.gguf');
+
+ expect(fetchMock).toHaveBeenCalledWith(
+ 'https://huggingface.co/org/omni/resolve/main/sub%20dir/mmproj%231.gguf',
+ { headers: { Range: 'bytes=0-65535' } }
+ );
+ expect(HuggingFaceService.getModalities('org/omni', 'any-to-any', 'mmproj.gguf')).toEqual({
+ audio: true,
+ video: false,
+ vision: true
+ });
+ });
+
+ it('drops vision for an audio-only mmproj', async () => {
+ // the converter writes the vision key only when true
+ respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 206);
+
+ await HuggingFaceService.getMmprojEncoders('org/audio-only', 'mmproj.gguf');
+
+ expect(HuggingFaceService.getModalities('org/audio-only', undefined, 'mmproj.gguf')).toEqual({
+ audio: true,
+ video: false,
+ vision: false
+ });
+ });
+
+ it('keeps vision when a header cut short never said', async () => {
+ const header = mmprojHeader({
+ 'clip.has_audio_encoder': true,
+ 'clip.has_vision_encoder': true
+ });
+
+ respond(header.slice(0, header.byteLength - 4), 206);
+
+ expect(await HuggingFaceService.getMmprojEncoders('org/cut', 'mmproj.gguf')).toEqual({
+ audio: true,
+ vision: null
+ });
+ expect(HuggingFaceService.getModalities('org/cut', undefined, 'mmproj.gguf').vision).toBe(true);
+ });
+
+ it('keeps vision only for a vision mmproj', async () => {
+ respond(mmprojHeader({ 'clip.has_vision_encoder': true }), 206);
+
+ await HuggingFaceService.getMmprojEncoders('org/vision', 'mmproj.gguf');
+
+ expect(HuggingFaceService.getModalities('org/vision', 'image-text-to-text', 'm.gguf')).toEqual({
+ audio: false,
+ video: false,
+ vision: true
+ });
+ });
+
+ it('keeps what the repo says for a gated mmproj, without reading it again', async () => {
+ const fetchMock = respond(new ArrayBuffer(0), 401);
+
+ expect(await HuggingFaceService.getMmprojEncoders('org/gated', 'mmproj.gguf')).toEqual({
+ audio: false,
+ vision: null
+ });
+ await HuggingFaceService.getMmprojEncoders('org/gated', 'mmproj.gguf');
+
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(HuggingFaceService.getModalities('org/gated', undefined, 'mmproj.gguf')).toEqual({
+ audio: false,
+ video: false,
+ vision: true
+ });
+ });
+
+ it('rejects a full response that ignored the range, and retries on the next call', async () => {
+ respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 200);
+
+ expect(await HuggingFaceService.getMmprojEncoders('org/no-range', 'mmproj.gguf')).toBeNull();
+ expect(HuggingFaceService.getModalities('org/no-range', undefined, 'mmproj.gguf').audio).toBe(
+ false
+ );
+
+ respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 206);
+
+ expect(await HuggingFaceService.getMmprojEncoders('org/no-range', 'mmproj.gguf')).toEqual({
+ audio: true,
+ vision: false
+ });
+ });
+});