Commit 28d37fe2e for llama.cpp
commit 28d37fe2e29cb83ae0e3a442675a952ff0fce9ef
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Fri Nov 13 10:33:17 2026 +0200
examples : clean-up old/obsolete stuff (#32184)
* examples : clean-up old/obsolete stuff
* examples : update references to removed files
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
[no ci]
diff --git a/.devops/nix/python-scripts.nix b/.devops/nix/python-scripts.nix
index 56ea18278..873b2ebe5 100644
--- a/.devops/nix/python-scripts.nix
+++ b/.devops/nix/python-scripts.nix
@@ -22,10 +22,6 @@ let
gitpython
tabulate
- # for examples/pydantic-models-to-grammar-examples.py
- docstring-parser
- pydantic
-
];
llama-python-test-deps = with python3Packages; [
diff --git a/docs/speculative.md b/docs/speculative.md
index d8f3bcb39..9072eedd3 100644
--- a/docs/speculative.md
+++ b/docs/speculative.md
@@ -189,7 +189,7 @@ llama-server ... --spec-type ngram-mod --spec-ngram-mod-n-match 24 --spec-ngram-
Applications:
-- Iterating over a block of text/code (e.g. in llama.vim)
+- Iterating over a block of text/code (e.g. in [llama.vim](https://github.com/ggml-org/llama.vim))
- Reasoning models (when they have to repeat their thinking in the final answer)
- Summarization
diff --git a/examples/json_schema_pydantic_example.py b/examples/json_schema_pydantic_example.py
deleted file mode 100644
index dae2a9b8a..000000000
--- a/examples/json_schema_pydantic_example.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Usage:
-#! ./llama-server -m some-model.gguf &
-#! pip install pydantic
-#! python json_schema_pydantic_example.py
-
-from pydantic import BaseModel, Field, TypeAdapter
-from annotated_types import MinLen
-from typing import Annotated, List, Optional
-import json, requests
-
-if True:
-
- def create_completion(*, response_model=None, endpoint="http://localhost:9931/v1/chat/completions", messages, **kwargs):
- '''
- Creates a chat completion using an OpenAI-compatible endpoint w/ JSON schema support
- (llama.cpp server, llama-cpp-python, Anyscale / Together...)
-
- The response_model param takes a type (+ supports Pydantic) and behaves just as w/ Instructor (see below)
- '''
- response_format = None
- type_adapter = None
-
- if response_model:
- type_adapter = TypeAdapter(response_model)
- schema = type_adapter.json_schema()
- messages = [{
- "role": "system",
- "content": f"You respond in JSON format with the following schema: {json.dumps(schema, indent=2)}"
- }] + messages
- response_format={"type": "json_object", "schema": schema}
-
- data = requests.post(endpoint, headers={"Content-Type": "application/json"},
- json=dict(messages=messages, response_format=response_format, **kwargs)).json()
- if 'error' in data:
- raise Exception(data['error']['message'])
-
- content = data["choices"][0]["message"]["content"]
- return type_adapter.validate_json(content) if type_adapter else content
-
-else:
-
- # This alternative branch uses Instructor + OpenAI client lib.
- # Instructor support streamed iterable responses, retry & more.
- # (see https://python.useinstructor.com/)
- #! pip install instructor openai
- import instructor, openai
- client = instructor.patch(
- openai.OpenAI(api_key="123", base_url="http://localhost:9931"),
- mode=instructor.Mode.JSON_SCHEMA)
- create_completion = client.chat.completions.create
-
-
-if __name__ == '__main__':
-
- class QAPair(BaseModel):
- class Config:
- extra = 'forbid' # triggers additionalProperties: false in the JSON schema
- question: str
- concise_answer: str
- justification: str
- stars: Annotated[int, Field(ge=1, le=5)]
-
- class PyramidalSummary(BaseModel):
- class Config:
- extra = 'forbid' # triggers additionalProperties: false in the JSON schema
- title: str
- summary: str
- question_answers: Annotated[List[QAPair], MinLen(2)]
- sub_sections: Optional[Annotated[List['PyramidalSummary'], MinLen(2)]]
-
- print("# Summary\n", create_completion(
- model="...",
- response_model=PyramidalSummary,
- messages=[{
- "role": "user",
- "content": f"""
- You are a highly efficient corporate document summarizer.
- Create a pyramidal summary of an imaginary internal document about our company processes
- (starting high-level, going down to each sub sections).
- Keep questions short, and answers even shorter (trivia / quizz style).
- """
- }]))
diff --git a/examples/llama.vim b/examples/llama.vim
deleted file mode 100644
index 23a281fc3..000000000
--- a/examples/llama.vim
+++ /dev/null
@@ -1,783 +0,0 @@
-" LLM-based text completion using llama.cpp
-"
-" requires:
-"
-" - neovim or vim
-" - curl
-" - llama.cpp server instance
-" - FIM-compatible model
-"
-" sample config:
-"
-" - Tab - accept the current suggestion
-" - Shift+Tab - accept just the first line of the suggestion
-" - Ctrl+F - toggle FIM completion manually
-"
-" make symlink or copy this file to ~/.config/nvim/autoload/llama.vim
-"
-" start the llama.cpp server with a FIM-compatible model. for example:
-"
-" $ llama-server -m {model.gguf} --port 8012 -ngl 99 -fa --ubatch-size 512 --batch-size 1024 --cache-reuse 256
-"
-" --batch-size [512, model max context]
-"
-" adjust the batch size to control how much of the provided local context will be used during the inference
-" lower values will use smaller part of the context around the cursor, which will result in faster processing
-"
-" --ubatch-size [64, 2048]
-"
-" chunks the batch into smaller chunks for faster processing
-" depends on the specific hardware. use llama-bench to profile and determine the best size
-"
-" --cache-reuse (ge:llama_config.n_predict, 1024]
-"
-" this should be either 0 (disabled) or strictly larger than g:llama_config.n_predict
-" using non-zero value enables context reuse on the server side which dramatically improves the performance at
-" large contexts. a value of 256 should be good for all cases
-"
-" run this once to initialise llama.vim:
-"
-" :call llama#init()
-"
-" more info: https://github.com/ggml-org/llama.cpp/pull/9787
-"
-
-" colors (adjust to your liking)
-highlight llama_hl_hint guifg=#ff772f ctermfg=202
-highlight llama_hl_info guifg=#77ff2f ctermfg=119
-
-" general parameters:
-"
-" endpoint: llama.cpp server endpoint
-" n_prefix: number of lines before the cursor location to include in the local prefix
-" n_suffix: number of lines after the cursor location to include in the local suffix
-" n_predict: max number of tokens to predict
-" t_max_prompt_ms: max allotted time for the prompt processing (TODO: not yet supported)
-" t_max_predict_ms: max allotted time for the prediction
-" show_info: show extra info about the inference (0 - disabled, 1 - statusline, 2 - inline)
-" auto_fim: trigger FIM completion automatically on cursor movement
-" max_line_suffix: do not auto-trigger FIM completion if there are more than this number of characters to the right of the cursor
-"
-" ring buffer of chunks, accumulated with time upon:
-"
-" - completion request
-" - yank
-" - entering a buffer
-" - leaving a buffer
-" - writing a file
-"
-" parameters for the ring-buffer with extra context:
-"
-" ring_n_chunks: max number of chunks to pass as extra context to the server (0 to disable)
-" ring_chunk_size: max size of the chunks (in number of lines)
-" note: adjust these numbers so that you don't overrun your context
-" at ring_n_chunks = 64 and ring_chunk_size = 64 you need ~32k context
-" ring_scope: the range around the cursor position (in number of lines) for gathering chunks after FIM
-" ring_update_ms: how often to process queued chunks in normal mode
-"
-let s:default_config = {
- \ 'endpoint': 'http://127.0.0.1:8012/infill',
- \ 'n_prefix': 256,
- \ 'n_suffix': 64,
- \ 'n_predict': 128,
- \ 't_max_prompt_ms': 500,
- \ 't_max_predict_ms': 3000,
- \ 'show_info': 2,
- \ 'auto_fim': v:true,
- \ 'max_line_suffix': 8,
- \ 'ring_n_chunks': 64,
- \ 'ring_chunk_size': 64,
- \ 'ring_scope': 1024,
- \ 'ring_update_ms': 1000,
- \ }
-
-let g:llama_config = get(g:, 'llama_config', s:default_config)
-
-function! s:get_indent(str)
- let l:count = 0
- for i in range(len(a:str))
- if a:str[i] == "\t"
- let l:count += &tabstop - 1
- else
- break
- endif
- endfor
- return l:count
-endfunction
-
-function! s:rand(i0, i1) abort
- return a:i0 + rand() % (a:i1 - a:i0 + 1)
-endfunction
-
-function! llama#init()
- if !executable('curl')
- echohl WarningMsg
- echo 'llama.vim requires the "curl" command to be available'
- echohl None
- return
- endif
-
- let s:pos_x = 0 " cursor position upon start of completion
- let s:pos_y = 0
-
- let s:line_cur = ''
-
- let s:line_cur_prefix = ''
- let s:line_cur_suffix = ''
-
- let s:ring_chunks = [] " current set of chunks used as extra context
- let s:ring_queued = [] " chunks that are queued to be sent for processing
- let s:ring_n_evict = 0
-
- let s:hint_shown = v:false
- let s:pos_y_pick = -9999 " last y where we picked a chunk
- let s:pos_dx = 0
- let s:content = []
- let s:can_accept = v:false
-
- let s:timer_fim = -1
- let s:t_fim_start = reltime() " used to measure total FIM time
- let s:t_last_move = reltime() " last time the cursor moved
-
- let s:current_job = v:null
-
- let s:ghost_text_nvim = exists('*nvim_buf_get_mark')
- let s:ghost_text_vim = has('textprop')
-
- if s:ghost_text_vim
- let s:hlgroup_hint = 'llama_hl_hint'
- let s:hlgroup_info = 'llama_hl_info'
-
- if empty(prop_type_get(s:hlgroup_hint))
- call prop_type_add(s:hlgroup_hint, {'highlight': s:hlgroup_hint})
- endif
- if empty(prop_type_get(s:hlgroup_info))
- call prop_type_add(s:hlgroup_info, {'highlight': s:hlgroup_info})
- endif
- endif
-
- augroup llama
- autocmd!
- autocmd InsertEnter * inoremap <expr> <silent> <C-F> llama#fim_inline(v:false)
- autocmd InsertLeavePre * call llama#fim_cancel()
-
- autocmd CursorMoved * call s:on_move()
- autocmd CursorMovedI * call s:on_move()
- autocmd CompleteChanged * call llama#fim_cancel()
-
- if g:llama_config.auto_fim
- autocmd CursorMovedI * call llama#fim(v:true)
- endif
-
- " gather chunks upon yanking
- autocmd TextYankPost * if v:event.operator ==# 'y' | call s:pick_chunk(v:event.regcontents, v:false, v:true) | endif
-
- " gather chunks upon entering/leaving a buffer
- autocmd BufEnter * call timer_start(100, {-> s:pick_chunk(getline(max([1, line('.') - g:llama_config.ring_chunk_size/2]), min([line('.') + g:llama_config.ring_chunk_size/2, line('$')])), v:true, v:true)})
- autocmd BufLeave * call s:pick_chunk(getline(max([1, line('.') - g:llama_config.ring_chunk_size/2]), min([line('.') + g:llama_config.ring_chunk_size/2, line('$')])), v:true, v:true)
-
- " gather chunk upon saving the file
- autocmd BufWritePost * call s:pick_chunk(getline(max([1, line('.') - g:llama_config.ring_chunk_size/2]), min([line('.') + g:llama_config.ring_chunk_size/2, line('$')])), v:true, v:true)
- augroup END
-
- silent! call llama#fim_cancel()
-
- " init background update of the ring buffer
- if g:llama_config.ring_n_chunks > 0
- call s:ring_update()
- endif
-endfunction
-
-" compute how similar two chunks of text are
-" 0 - no similarity, 1 - high similarity
-" TODO: figure out something better
-function! s:chunk_sim(c0, c1)
- let l:lines0 = len(a:c0)
- let l:lines1 = len(a:c1)
-
- let l:common = 0
-
- for l:line0 in a:c0
- for l:line1 in a:c1
- if l:line0 == l:line1
- let l:common += 1
- break
- endif
- endfor
- endfor
-
- return 2.0 * l:common / (l:lines0 + l:lines1)
-endfunction
-
-" pick a random chunk of size g:llama_config.ring_chunk_size from the provided text and queue it for processing
-"
-" no_mod - do not pick chunks from buffers with pending changes
-" do_evict - evict chunks that are very similar to the new one
-"
-function! s:pick_chunk(text, no_mod, do_evict)
- " do not pick chunks from buffers with pending changes or buffers that are not files
- if a:no_mod && (getbufvar(bufnr('%'), '&modified') || !buflisted(bufnr('%')) || !filereadable(expand('%')))
- return
- endif
-
- " if the extra context option is disabled - do nothing
- if g:llama_config.ring_n_chunks <= 0
- return
- endif
-
- " don't pick very small chunks
- if len(a:text) < 3
- return
- endif
-
- if len(a:text) + 1 < g:llama_config.ring_chunk_size
- let l:chunk = a:text
- else
- let l:l0 = s:rand(0, max([0, len(a:text) - g:llama_config.ring_chunk_size/2]))
- let l:l1 = min([l:l0 + g:llama_config.ring_chunk_size/2, len(a:text)])
-
- let l:chunk = a:text[l:l0:l:l1]
- endif
-
- let l:chunk_str = join(l:chunk, "\n") . "\n"
-
- " check if this chunk is already added
- let l:exist = v:false
-
- for i in range(len(s:ring_chunks))
- if s:ring_chunks[i].data == l:chunk
- let l:exist = v:true
- break
- endif
- endfor
-
- for i in range(len(s:ring_queued))
- if s:ring_queued[i].data == l:chunk
- let l:exist = v:true
- break
- endif
- endfor
-
- if l:exist
- return
- endif
-
- " evict queued chunks that are very similar to the new one
- for i in range(len(s:ring_queued) - 1, 0, -1)
- if s:chunk_sim(s:ring_queued[i].data, l:chunk) > 0.9
- if a:do_evict
- call remove(s:ring_queued, i)
- let s:ring_n_evict += 1
- else
- return
- endif
- endif
- endfor
-
- " also from s:ring_chunks
- for i in range(len(s:ring_chunks) - 1, 0, -1)
- if s:chunk_sim(s:ring_chunks[i].data, l:chunk) > 0.9
- if a:do_evict
- call remove(s:ring_chunks, i)
- let s:ring_n_evict += 1
- else
- return
- endif
- endif
- endfor
-
- " TODO: become parameter ?
- if len(s:ring_queued) == 16
- call remove(s:ring_queued, 0)
- endif
-
- call add(s:ring_queued, {'data': l:chunk, 'str': l:chunk_str, 'time': reltime(), 'filename': expand('%')})
-
- "let &statusline = 'extra context: ' . len(s:ring_chunks) . ' / ' . len(s:ring_queued)
-endfunction
-
-" picks a queued chunk, sends it for processing and adds it to s:ring_chunks
-" called every g:llama_config.ring_update_ms
-function! s:ring_update()
- call timer_start(g:llama_config.ring_update_ms, {-> s:ring_update()})
-
- " update only if in normal mode or if the cursor hasn't moved for a while
- if mode() !=# 'n' && reltimefloat(reltime(s:t_last_move)) < 3.0
- return
- endif
-
- if len(s:ring_queued) == 0
- return
- endif
-
- " move the first queued chunk to the ring buffer
- if len(s:ring_chunks) == g:llama_config.ring_n_chunks
- call remove(s:ring_chunks, 0)
- endif
-
- call add(s:ring_chunks, remove(s:ring_queued, 0))
-
- "let &statusline = 'updated context: ' . len(s:ring_chunks) . ' / ' . len(s:ring_queued)
-
- " send asynchronous job with the new extra context so that it is ready for the next FIM
- let l:extra_context = []
- for l:chunk in s:ring_chunks
- call add(l:extra_context, {
- \ 'text': l:chunk.str,
- \ 'time': l:chunk.time,
- \ 'filename': l:chunk.filename
- \ })
- endfor
-
- " no samplers needed here
- let l:request = json_encode({
- \ 'input_prefix': "",
- \ 'input_suffix': "",
- \ 'input_extra': l:extra_context,
- \ 'prompt': "",
- \ 'n_predict': 1,
- \ 'temperature': 0.0,
- \ 'stream': v:false,
- \ 'samplers': ["temperature"],
- \ 'cache_prompt': v:true,
- \ 't_max_prompt_ms': 1,
- \ 't_max_predict_ms': 1
- \ })
-
- let l:curl_command = [
- \ "curl",
- \ "--silent",
- \ "--no-buffer",
- \ "--request", "POST",
- \ "--url", g:llama_config.endpoint,
- \ "--header", "Content-Type: application/json",
- \ "--data", l:request
- \ ]
-
- " no callbacks because we don't need to process the response
- if s:ghost_text_nvim
- call jobstart(l:curl_command, {})
- elseif s:ghost_text_vim
- call job_start(l:curl_command, {})
- endif
-endfunction
-
-" necessary for 'inoremap <expr>'
-function! llama#fim_inline(is_auto) abort
- call llama#fim(a:is_auto)
- return ''
-endfunction
-
-" the main FIM call
-" takes local context around the cursor and sends it together with the extra context to the server for completion
-function! llama#fim(is_auto) abort
- " we already have a suggestion for the current cursor position
- if s:hint_shown && !a:is_auto
- call llama#fim_cancel()
- return
- endif
-
- call llama#fim_cancel()
-
- " avoid sending repeated requests too fast
- if reltimefloat(reltime(s:t_fim_start)) < 0.6
- if s:timer_fim != -1
- call timer_stop(s:timer_fim)
- let s:timer_fim = -1
- endif
-
- let s:t_fim_start = reltime()
- let s:timer_fim = timer_start(600, {-> llama#fim(v:true)})
- return
- endif
-
- let s:t_fim_start = reltime()
-
- let s:content = []
- let s:can_accept = v:false
-
- let s:pos_x = col('.') - 1
- let s:pos_y = line('.')
- let l:max_y = line('$')
-
- let l:lines_prefix = getline(max([1, s:pos_y - g:llama_config.n_prefix]), s:pos_y - 1)
- let l:lines_suffix = getline(s:pos_y + 1, min([l:max_y, s:pos_y + g:llama_config.n_suffix]))
-
- let s:line_cur = getline('.')
-
- let s:line_cur_prefix = strpart(s:line_cur, 0, s:pos_x)
- let s:line_cur_suffix = strpart(s:line_cur, s:pos_x)
-
- if a:is_auto && len(s:line_cur_suffix) > g:llama_config.max_line_suffix
- return
- endif
-
- let l:prefix = ""
- \ . join(l:lines_prefix, "\n")
- \ . "\n"
-
- let l:prompt = ""
- \ . s:line_cur_prefix
-
- let l:suffix = ""
- \ . s:line_cur_suffix
- \ . "\n"
- \ . join(l:lines_suffix, "\n")
- \ . "\n"
-
- " prepare the extra context data
- let l:extra_context = []
- for l:chunk in s:ring_chunks
- call add(l:extra_context, {
- \ 'text': l:chunk.str,
- \ 'time': l:chunk.time,
- \ 'filename': l:chunk.filename
- \ })
- endfor
-
- " the indentation of the current line
- let l:indent = strlen(matchstr(s:line_cur_prefix, '^\s*'))
-
- let l:request = json_encode({
- \ 'input_prefix': l:prefix,
- \ 'input_suffix': l:suffix,
- \ 'input_extra': l:extra_context,
- \ 'prompt': l:prompt,
- \ 'n_predict': g:llama_config.n_predict,
- \ 'n_indent': l:indent,
- \ 'top_k': 40,
- \ 'top_p': 0.99,
- \ 'stream': v:false,
- \ 'samplers': ["top_k", "top_p", "infill"],
- \ 'cache_prompt': v:true,
- \ 't_max_prompt_ms': g:llama_config.t_max_prompt_ms,
- \ 't_max_predict_ms': g:llama_config.t_max_predict_ms
- \ })
-
- let l:curl_command = [
- \ "curl",
- \ "--silent",
- \ "--no-buffer",
- \ "--request", "POST",
- \ "--url", g:llama_config.endpoint,
- \ "--header", "Content-Type: application/json",
- \ "--data", l:request
- \ ]
-
- if s:current_job != v:null
- if s:ghost_text_nvim
- call jobstop(s:current_job)
- elseif s:ghost_text_vim
- call job_stop(s:current_job)
- endif
- endif
-
- " send the request asynchronously
- if s:ghost_text_nvim
- let s:current_job = jobstart(l:curl_command, {
- \ 'on_stdout': function('s:fim_on_stdout', [s:pos_x, s:pos_y, a:is_auto]),
- \ 'on_exit': function('s:fim_on_exit'),
- \ 'stdout_buffered': v:true
- \ })
- elseif s:ghost_text_vim
- let s:current_job = job_start(l:curl_command, {
- \ 'out_cb': function('s:fim_on_stdout', [s:pos_x, s:pos_y, a:is_auto]),
- \ 'exit_cb': function('s:fim_on_exit')
- \ })
- endif
-
- " TODO: per-file location
- let l:delta_y = abs(s:pos_y - s:pos_y_pick)
-
- " gather some extra context nearby and process it in the background
- " only gather chunks if the cursor has moved a lot
- " TODO: something more clever? reranking?
- if a:is_auto && l:delta_y > 32
- " expand the prefix even further
- call s:pick_chunk(getline(max([1, s:pos_y - g:llama_config.ring_scope]), max([1, s:pos_y - g:llama_config.n_prefix])), v:false, v:false)
-
- " pick a suffix chunk
- call s:pick_chunk(getline(min([l:max_y, s:pos_y + g:llama_config.n_suffix]), min([l:max_y, s:pos_y + g:llama_config.n_suffix + g:llama_config.ring_chunk_size])), v:false, v:false)
-
- let s:pos_y_pick = s:pos_y
- endif
-endfunction
-
-" if first_line == v:true accept only the first line of the response
-function! llama#fim_accept(first_line)
- " insert the suggestion at the cursor location
- if s:can_accept && len(s:content) > 0
- call setline(s:pos_y, s:line_cur[:(s:pos_x - 1)] . s:content[0])
- if len(s:content) > 1
- if !a:first_line
- call append(s:pos_y, s:content[1:-1])
- endif
- endif
-
- " move the cursor to the end of the accepted text
- if !a:first_line && len(s:content) > 1
- call cursor(s:pos_y + len(s:content) - 1, s:pos_x + s:pos_dx + 1)
- else
- call cursor(s:pos_y, s:pos_x + len(s:content[0]))
- endif
- endif
-
- call llama#fim_cancel()
-endfunction
-
-function! llama#fim_cancel()
- let s:hint_shown = v:false
-
- " clear the virtual text
- let l:bufnr = bufnr('%')
-
- if s:ghost_text_nvim
- let l:id_vt_fim = nvim_create_namespace('vt_fim')
- call nvim_buf_clear_namespace(l:bufnr, l:id_vt_fim, 0, -1)
- elseif s:ghost_text_vim
- call prop_remove({'type': s:hlgroup_hint, 'all': v:true})
- call prop_remove({'type': s:hlgroup_info, 'all': v:true})
- endif
-
- " remove the mappings
- silent! iunmap <buffer> <Tab>
- silent! iunmap <buffer> <S-Tab>
- silent! iunmap <buffer> <Esc>
-endfunction
-
-function! s:on_move()
- let s:t_last_move = reltime()
-
- call llama#fim_cancel()
-endfunction
-
-" callback that processes the FIM result from the server and displays the suggestion
-function! s:fim_on_stdout(pos_x, pos_y, is_auto, job_id, data, event = v:null)
- if s:ghost_text_nvim
- let l:raw = join(a:data, "\n")
- elseif s:ghost_text_vim
- let l:raw = a:data
- endif
-
- if len(l:raw) == 0
- return
- endif
-
- if a:pos_x != col('.') - 1 || a:pos_y != line('.')
- return
- endif
-
- " show the suggestion only in insert mode
- if mode() !=# 'i'
- return
- endif
-
- let s:pos_x = a:pos_x
- let s:pos_y = a:pos_y
-
- let s:can_accept = v:true
- let l:has_info = v:false
-
- if s:can_accept && v:shell_error
- if !a:is_auto
- call add(s:content, "<| curl error: is the server on? |>")
- endif
- let s:can_accept = v:false
- endif
-
- let l:n_prompt = 0
- let l:t_prompt_ms = 1.0
- let l:s_prompt = 0
-
- let l:n_predict = 0
- let l:t_predict_ms = 1.0
- let l:s_predict = 0
-
- " get the generated suggestion
- if s:can_accept
- let l:response = json_decode(l:raw)
-
- for l:part in split(get(l:response, 'content', ''), "\n", 1)
- call add(s:content, l:part)
- endfor
-
- " remove trailing new lines
- while len(s:content) > 0 && s:content[-1] == ""
- call remove(s:content, -1)
- endwhile
-
- let l:generation_settings = get(l:response, 'generation_settings', {})
- let l:n_ctx = get(l:generation_settings, 'n_ctx', 0)
-
- let l:n_cached = get(l:response, 'tokens_cached', 0)
- let l:truncated = get(l:response, 'truncated', v:false)
-
- " if response.timings is available
- if len(get(l:response, 'timings', {})) > 0
- let l:has_info = v:true
- let l:timings = get(l:response, 'timings', {})
-
- let l:n_prompt = get(l:timings, 'prompt_n', 0)
- let l:t_prompt_ms = get(l:timings, 'prompt_ms', 1)
- let l:s_prompt = get(l:timings, 'prompt_per_second', 0)
-
- let l:n_predict = get(l:timings, 'predicted_n', 0)
- let l:t_predict_ms = get(l:timings, 'predicted_ms', 1)
- let l:s_predict = get(l:timings, 'predicted_per_second', 0)
- endif
- endif
-
- if len(s:content) == 0
- call add(s:content, "")
- let s:can_accept = v:false
- endif
-
- if len(s:content) == 0
- return
- endif
-
- " NOTE: the following is logic for discarding predictions that repeat existing text
- " the code is quite ugly and there is very likely a simpler and more canonical way to implement this
- "
- " still, I wonder if there is some better way that avoids having to do these special hacks?
- " on one hand, the LLM 'sees' the contents of the file before we start editing, so it is normal that it would
- " start generating whatever we have given it via the extra context. but on the other hand, it's not very
- " helpful to re-generate the same code that is already there
-
- " truncate the suggestion if the first line is empty
- if len(s:content) == 1 && s:content[0] == ""
- let s:content = [""]
- endif
-
- " ... and the next lines are repeated
- if len(s:content) > 1 && s:content[0] == "" && s:content[1:] == getline(s:pos_y + 1, s:pos_y + len(s:content) - 1)
- let s:content = [""]
- endif
-
- " truncate the suggestion if it repeats the suffix
- if len(s:content) == 1 && s:content[0] == s:line_cur_suffix
- let s:content = [""]
- endif
-
- " find the first non-empty line (strip whitespace)
- let l:cmp_y = s:pos_y + 1
- while l:cmp_y < line('$') && getline(l:cmp_y) =~? '^\s*$'
- let l:cmp_y += 1
- endwhile
-
- if (s:line_cur_prefix . s:content[0]) == getline(l:cmp_y)
- " truncate the suggestion if it repeats the next line
- if len(s:content) == 1
- let s:content = [""]
- endif
-
- " ... or if the second line of the suggestion is the prefix of line l:cmp_y + 1
- if len(s:content) == 2 && s:content[-1] == getline(l:cmp_y + 1)[:len(s:content[-1]) - 1]
- let s:content = [""]
- endif
-
- " ... or if the middle chunk of lines of the suggestion is the same as [l:cmp_y + 1, l:cmp_y + len(s:content) - 1)
- if len(s:content) > 2 && join(s:content[1:-1], "\n") == join(getline(l:cmp_y + 1, l:cmp_y + len(s:content) - 1), "\n")
- let s:content = [""]
- endif
- endif
-
- " keep only lines that have the same or larger whitespace prefix as s:line_cur_prefix
- "let l:indent = strlen(matchstr(s:line_cur_prefix, '^\s*'))
- "for i in range(1, len(s:content) - 1)
- " if strlen(matchstr(s:content[i], '^\s*')) < l:indent
- " let s:content = s:content[:i - 1]
- " break
- " endif
- "endfor
-
- let s:pos_dx = len(s:content[-1])
-
- let s:content[-1] .= s:line_cur_suffix
-
- call llama#fim_cancel()
-
- " display virtual text with the suggestion
- let l:bufnr = bufnr('%')
-
- if s:ghost_text_nvim
- let l:id_vt_fim = nvim_create_namespace('vt_fim')
- endif
-
- " construct the info message
- if g:llama_config.show_info > 0 && l:has_info
- let l:prefix = ' '
-
- if l:truncated
- let l:info = printf("%s | WARNING: the context is full: %d / %d, increase the server context size or reduce g:llama_config.ring_n_chunks",
- \ g:llama_config.show_info == 2 ? l:prefix : 'llama.vim',
- \ l:n_cached, l:n_ctx
- \ )
- else
- let l:info = printf("%s | c: %d / %d, r: %d / %d, e: %d, q: %d / 16 | p: %d (%.2f ms, %.2f t/s) | g: %d (%.2f ms, %.2f t/s) | t: %.2f ms",
- \ g:llama_config.show_info == 2 ? l:prefix : 'llama.vim',
- \ l:n_cached, l:n_ctx, len(s:ring_chunks), g:llama_config.ring_n_chunks, s:ring_n_evict, len(s:ring_queued),
- \ l:n_prompt, l:t_prompt_ms, l:s_prompt,
- \ l:n_predict, l:t_predict_ms, l:s_predict,
- \ 1000.0 * reltimefloat(reltime(s:t_fim_start))
- \ )
- endif
-
- if g:llama_config.show_info == 1
- " display the info in the statusline
- let &statusline = l:info
- let l:info = ''
- endif
- endif
-
- " display the suggestion and append the info to the end of the first line
- if s:ghost_text_nvim
- call nvim_buf_set_extmark(l:bufnr, l:id_vt_fim, s:pos_y - 1, s:pos_x - 1, {
- \ 'virt_text': [[s:content[0], 'llama_hl_hint'], [l:info, 'llama_hl_info']],
- \ 'virt_text_win_col': virtcol('.') - 1
- \ })
-
- call nvim_buf_set_extmark(l:bufnr, l:id_vt_fim, s:pos_y - 1, 0, {
- \ 'virt_lines': map(s:content[1:], {idx, val -> [[val, 'llama_hl_hint']]}),
- \ 'virt_text_win_col': virtcol('.')
- \ })
- elseif s:ghost_text_vim
- let l:new_suffix = s:content[0]
- if !empty(l:new_suffix)
- call prop_add(s:pos_y, s:pos_x + 1, {
- \ 'type': s:hlgroup_hint,
- \ 'text': l:new_suffix
- \ })
- endif
- for line in s:content[1:]
- call prop_add(s:pos_y, 0, {
- \ 'type': s:hlgroup_hint,
- \ 'text': line,
- \ 'text_padding_left': s:get_indent(line),
- \ 'text_align': 'below'
- \ })
- endfor
- if !empty(l:info)
- call prop_add(s:pos_y, 0, {
- \ 'type': s:hlgroup_info,
- \ 'text': l:info,
- \ 'text_padding_left': col('$'),
- \ 'text_wrap': 'truncate'
- \ })
- endif
- endif
-
- " setup accept shortcuts
- inoremap <buffer> <Tab> <C-O>:call llama#fim_accept(v:false)<CR>
- inoremap <buffer> <S-Tab> <C-O>:call llama#fim_accept(v:true)<CR>
-
- let s:hint_shown = v:true
-endfunction
-
-function! s:fim_on_exit(job_id, exit_code, event = v:null)
- if a:exit_code != 0
- echom "Job failed with exit code: " . a:exit_code
- endif
-
- let s:current_job = v:null
-endfunction
diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py
deleted file mode 100644
index ff6e120e4..000000000
--- a/examples/pydantic_models_to_grammar.py
+++ /dev/null
@@ -1,1322 +0,0 @@
-from __future__ import annotations
-
-import inspect
-import json
-import re
-from copy import copy
-from enum import Enum
-from inspect import getdoc, isclass
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union, get_args, get_origin, get_type_hints
-
-from docstring_parser import parse
-from pydantic import BaseModel, create_model
-
-if TYPE_CHECKING:
- from types import GenericAlias
-else:
- # python 3.8 compat
- from typing import _GenericAlias as GenericAlias
-
-# TODO: fix this
-# pyright: reportAttributeAccessIssue=information
-
-
-class PydanticDataType(Enum):
- """
- Defines the data types supported by the grammar_generator.
-
- Attributes:
- STRING (str): Represents a string data type.
- BOOLEAN (str): Represents a boolean data type.
- INTEGER (str): Represents an integer data type.
- FLOAT (str): Represents a float data type.
- OBJECT (str): Represents an object data type.
- ARRAY (str): Represents an array data type.
- ENUM (str): Represents an enum data type.
- CUSTOM_CLASS (str): Represents a custom class data type.
- """
-
- STRING = "string"
- TRIPLE_QUOTED_STRING = "triple_quoted_string"
- MARKDOWN_CODE_BLOCK = "markdown_code_block"
- BOOLEAN = "boolean"
- INTEGER = "integer"
- FLOAT = "float"
- OBJECT = "object"
- ARRAY = "array"
- ENUM = "enum"
- ANY = "any"
- NULL = "null"
- CUSTOM_CLASS = "custom-class"
- CUSTOM_DICT = "custom-dict"
- SET = "set"
-
-
-def map_pydantic_type_to_gbnf(pydantic_type: type[Any]) -> str:
- origin_type = get_origin(pydantic_type)
- origin_type = pydantic_type if origin_type is None else origin_type
-
- if isclass(origin_type) and issubclass(origin_type, str):
- return PydanticDataType.STRING.value
- elif isclass(origin_type) and issubclass(origin_type, bool):
- return PydanticDataType.BOOLEAN.value
- elif isclass(origin_type) and issubclass(origin_type, int):
- return PydanticDataType.INTEGER.value
- elif isclass(origin_type) and issubclass(origin_type, float):
- return PydanticDataType.FLOAT.value
- elif isclass(origin_type) and issubclass(origin_type, Enum):
- return PydanticDataType.ENUM.value
-
- elif isclass(origin_type) and issubclass(origin_type, BaseModel):
- return format_model_and_field_name(origin_type.__name__)
- elif origin_type is list:
- element_type = get_args(pydantic_type)[0]
- return f"{map_pydantic_type_to_gbnf(element_type)}-list"
- elif origin_type is set:
- element_type = get_args(pydantic_type)[0]
- return f"{map_pydantic_type_to_gbnf(element_type)}-set"
- elif origin_type is Union:
- union_types = get_args(pydantic_type)
- union_rules = [map_pydantic_type_to_gbnf(ut) for ut in union_types]
- return f"union-{'-or-'.join(union_rules)}"
- elif origin_type is Optional:
- element_type = get_args(pydantic_type)[0]
- return f"optional-{map_pydantic_type_to_gbnf(element_type)}"
- elif isclass(origin_type):
- return f"{PydanticDataType.CUSTOM_CLASS.value}-{format_model_and_field_name(origin_type.__name__)}"
- elif origin_type is dict:
- key_type, value_type = get_args(pydantic_type)
- return f"custom-dict-key-type-{format_model_and_field_name(map_pydantic_type_to_gbnf(key_type))}-value-type-{format_model_and_field_name(map_pydantic_type_to_gbnf(value_type))}"
- else:
- return "unknown"
-
-
-def format_model_and_field_name(model_name: str) -> str:
- parts = re.findall("[A-Z][^A-Z]*", model_name)
- if not parts: # Check if the list is empty
- return model_name.lower().replace("_", "-")
- return "-".join(part.lower().replace("_", "-") for part in parts)
-
-
-def generate_list_rule(element_type):
- """
- Generate a GBNF rule for a list of a given element type.
-
- :param element_type: The type of the elements in the list (e.g., 'string').
- :return: A string representing the GBNF rule for a list of the given type.
- """
- rule_name = f"{map_pydantic_type_to_gbnf(element_type)}-list"
- element_rule = map_pydantic_type_to_gbnf(element_type)
- list_rule = rf'{rule_name} ::= "[" {element_rule} ("," {element_rule})* "]"'
- return list_rule
-
-
-def get_members_structure(cls, rule_name):
- if issubclass(cls, Enum):
- # Handle Enum types
- members = [f'"\\"{member.value}\\""' for name, member in cls.__members__.items()]
- return f"{cls.__name__.lower()} ::= " + " | ".join(members)
- if cls.__annotations__ and cls.__annotations__ != {}:
- result = f'{rule_name} ::= "{{"'
- # Modify this comprehension
- members = [
- f' "\\"{name}\\"" ":" {map_pydantic_type_to_gbnf(param_type)}'
- for name, param_type in get_type_hints(cls).items()
- if name != "self"
- ]
-
- result += '"," '.join(members)
- result += ' "}"'
- return result
- if rule_name == "custom-class-any":
- result = f"{rule_name} ::= "
- result += "value"
- return result
-
- init_signature = inspect.signature(cls.__init__)
- parameters = init_signature.parameters
- result = f'{rule_name} ::= "{{"'
- # Modify this comprehension too
- members = [
- f' "\\"{name}\\"" ":" {map_pydantic_type_to_gbnf(param.annotation)}'
- for name, param in parameters.items()
- if name != "self" and param.annotation != inspect.Parameter.empty
- ]
-
- result += '", "'.join(members)
- result += ' "}"'
- return result
-
-
-def regex_to_gbnf(regex_pattern: str) -> str:
- """
- Translate a basic regex pattern to a GBNF rule.
- Note: This function handles only a subset of simple regex patterns.
- """
- gbnf_rule = regex_pattern
-
- # Translate common regex components to GBNF
- gbnf_rule = gbnf_rule.replace("\\d", "[0-9]")
- gbnf_rule = gbnf_rule.replace("\\s", "[ \t\n]")
-
- # Handle quantifiers and other regex syntax that is similar in GBNF
- # (e.g., '*', '+', '?', character classes)
-
- return gbnf_rule
-
-
-def generate_gbnf_integer_rules(max_digit=None, min_digit=None):
- """
-
- Generate GBNF Integer Rules
-
- Generates GBNF (Generalized Backus-Naur Form) rules for integers based on the given maximum and minimum digits.
-
- Parameters:
- max_digit (int): The maximum number of digits for the integer. Default is None.
- min_digit (int): The minimum number of digits for the integer. Default is None.
-
- Returns:
- integer_rule (str): The identifier for the integer rule generated.
- additional_rules (list): A list of additional rules generated based on the given maximum and minimum digits.
-
- """
- additional_rules = []
-
- # Define the rule identifier based on max_digit and min_digit
- integer_rule = "integer-part"
- if max_digit is not None:
- integer_rule += f"-max{max_digit}"
- if min_digit is not None:
- integer_rule += f"-min{min_digit}"
-
- # Handling Integer Rules
- if max_digit is not None or min_digit is not None:
- # Start with an empty rule part
- integer_rule_part = ""
-
- # Add mandatory digits as per min_digit
- if min_digit is not None:
- integer_rule_part += "[0-9] " * min_digit
-
- # Add optional digits up to max_digit
- if max_digit is not None:
- optional_digits = max_digit - (min_digit if min_digit is not None else 0)
- integer_rule_part += "".join(["[0-9]? " for _ in range(optional_digits)])
-
- # Trim the rule part and append it to additional rules
- integer_rule_part = integer_rule_part.strip()
- if integer_rule_part:
- additional_rules.append(f"{integer_rule} ::= {integer_rule_part}")
-
- return integer_rule, additional_rules
-
-
-def generate_gbnf_float_rules(max_digit=None, min_digit=None, max_precision=None, min_precision=None):
- """
- Generate GBNF float rules based on the given constraints.
-
- :param max_digit: Maximum number of digits in the integer part (default: None)
- :param min_digit: Minimum number of digits in the integer part (default: None)
- :param max_precision: Maximum number of digits in the fractional part (default: None)
- :param min_precision: Minimum number of digits in the fractional part (default: None)
- :return: A tuple containing the float rule and additional rules as a list
-
- Example Usage:
- max_digit = 3
- min_digit = 1
- max_precision = 2
- min_precision = 1
- generate_gbnf_float_rules(max_digit, min_digit, max_precision, min_precision)
-
- Output:
- ('float-3-1-2-1', ['integer-part-max3-min1 ::= [0-9] [0-9] [0-9]?', 'fractional-part-max2-min1 ::= [0-9] [0-9]?', 'float-3-1-2-1 ::= integer-part-max3-min1 "." fractional-part-max2-min
- *1'])
-
- Note:
- GBNF stands for Generalized Backus-Naur Form, which is a notation technique to specify the syntax of programming languages or other formal grammars.
- """
- additional_rules = []
-
- # Define the integer part rule
- integer_part_rule = (
- "integer-part"
- + (f"-max{max_digit}" if max_digit is not None else "")
- + (f"-min{min_digit}" if min_digit is not None else "")
- )
-
- # Define the fractional part rule based on precision constraints
- fractional_part_rule = "fractional-part"
- fractional_rule_part = ""
- if max_precision is not None or min_precision is not None:
- fractional_part_rule += (f"-max{max_precision}" if max_precision is not None else "") + (
- f"-min{min_precision}" if min_precision is not None else ""
- )
- # Minimum number of digits
- fractional_rule_part = "[0-9]" * (min_precision if min_precision is not None else 1)
- # Optional additional digits
- fractional_rule_part += "".join(
- [" [0-9]?"] * ((max_precision - (
- min_precision if min_precision is not None else 1)) if max_precision is not None else 0)
- )
- additional_rules.append(f"{fractional_part_rule} ::= {fractional_rule_part}")
-
- # Define the float rule
- float_rule = f"float-{max_digit if max_digit is not None else 'X'}-{min_digit if min_digit is not None else 'X'}-{max_precision if max_precision is not None else 'X'}-{min_precision if min_precision is not None else 'X'}"
- additional_rules.append(f'{float_rule} ::= {integer_part_rule} "." {fractional_part_rule}')
-
- # Generating the integer part rule definition, if necessary
- if max_digit is not None or min_digit is not None:
- integer_rule_part = "[0-9]"
- if min_digit is not None and min_digit > 1:
- integer_rule_part += " [0-9]" * (min_digit - 1)
- if max_digit is not None:
- integer_rule_part += "".join([" [0-9]?"] * (max_digit - (min_digit if min_digit is not None else 1)))
- additional_rules.append(f"{integer_part_rule} ::= {integer_rule_part.strip()}")
-
- return float_rule, additional_rules
-
-
-def generate_gbnf_rule_for_type(
- model_name, field_name, field_type, is_optional, processed_models, created_rules, field_info=None
-) -> tuple[str, list[str]]:
- """
- Generate GBNF rule for a given field type.
-
- :param model_name: Name of the model.
-
- :param field_name: Name of the field.
- :param field_type: Type of the field.
- :param is_optional: Whether the field is optional.
- :param processed_models: List of processed models.
- :param created_rules: List of created rules.
- :param field_info: Additional information about the field (optional).
-
- :return: Tuple containing the GBNF type and a list of additional rules.
- :rtype: tuple[str, list]
- """
- rules = []
-
- field_name = format_model_and_field_name(field_name)
- gbnf_type = map_pydantic_type_to_gbnf(field_type)
-
- origin_type = get_origin(field_type)
- origin_type = field_type if origin_type is None else origin_type
-
- if isclass(origin_type) and issubclass(origin_type, BaseModel):
- nested_model_name = format_model_and_field_name(field_type.__name__)
- nested_model_rules, _ = generate_gbnf_grammar(field_type, processed_models, created_rules)
- rules.extend(nested_model_rules)
- gbnf_type, rules = nested_model_name, rules
- elif isclass(origin_type) and issubclass(origin_type, Enum):
- enum_values = [f'"\\"{e.value}\\""' for e in field_type] # Adding escaped quotes
- enum_rule = f"{model_name}-{field_name} ::= {' | '.join(enum_values)}"
- rules.append(enum_rule)
- gbnf_type, rules = model_name + "-" + field_name, rules
- elif origin_type is list: # Array
- element_type = get_args(field_type)[0]
- element_rule_name, additional_rules = generate_gbnf_rule_for_type(
- model_name, f"{field_name}-element", element_type, is_optional, processed_models, created_rules
- )
- rules.extend(additional_rules)
- array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """
- rules.append(array_rule)
- gbnf_type, rules = model_name + "-" + field_name, rules
-
- elif origin_type is set: # Array
- element_type = get_args(field_type)[0]
- element_rule_name, additional_rules = generate_gbnf_rule_for_type(
- model_name, f"{field_name}-element", element_type, is_optional, processed_models, created_rules
- )
- rules.extend(additional_rules)
- array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """
- rules.append(array_rule)
- gbnf_type, rules = model_name + "-" + field_name, rules
-
- elif gbnf_type.startswith("custom-class-"):
- rules.append(get_members_structure(field_type, gbnf_type))
- elif gbnf_type.startswith("custom-dict-"):
- key_type, value_type = get_args(field_type)
-
- additional_key_type, additional_key_rules = generate_gbnf_rule_for_type(
- model_name, f"{field_name}-key-type", key_type, is_optional, processed_models, created_rules
- )
- additional_value_type, additional_value_rules = generate_gbnf_rule_for_type(
- model_name, f"{field_name}-value-type", value_type, is_optional, processed_models, created_rules
- )
- gbnf_type = rf'{gbnf_type} ::= "{{" ( {additional_key_type} ": " {additional_value_type} ("," "\n" ws {additional_key_type} ":" {additional_value_type})* )? "}}" '
-
- rules.extend(additional_key_rules)
- rules.extend(additional_value_rules)
- elif gbnf_type.startswith("union-"):
- union_types = get_args(field_type)
- union_rules = []
-
- for union_type in union_types:
- if isinstance(union_type, GenericAlias):
- union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(
- model_name, field_name, union_type, False, processed_models, created_rules
- )
- union_rules.append(union_gbnf_type)
- rules.extend(union_rules_list)
-
- elif not issubclass(union_type, type(None)):
- union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(
- model_name, field_name, union_type, False, processed_models, created_rules
- )
- union_rules.append(union_gbnf_type)
- rules.extend(union_rules_list)
-
- # Defining the union grammar rule separately
- if len(union_rules) == 1:
- union_grammar_rule = f"{model_name}-{field_name}-optional ::= {' | '.join(union_rules)} | null"
- else:
- union_grammar_rule = f"{model_name}-{field_name}-union ::= {' | '.join(union_rules)}"
- rules.append(union_grammar_rule)
- if len(union_rules) == 1:
- gbnf_type = f"{model_name}-{field_name}-optional"
- else:
- gbnf_type = f"{model_name}-{field_name}-union"
- elif isclass(origin_type) and issubclass(origin_type, str):
- if field_info and hasattr(field_info, "json_schema_extra") and field_info.json_schema_extra is not None:
- triple_quoted_string = field_info.json_schema_extra.get("triple_quoted_string", False)
- markdown_string = field_info.json_schema_extra.get("markdown_code_block", False)
-
- gbnf_type = PydanticDataType.TRIPLE_QUOTED_STRING.value if triple_quoted_string else PydanticDataType.STRING.value
- gbnf_type = PydanticDataType.MARKDOWN_CODE_BLOCK.value if markdown_string else gbnf_type
-
- elif field_info and hasattr(field_info, "pattern"):
- # Convert regex pattern to grammar rule
- regex_pattern = field_info.regex.pattern
- gbnf_type = f"pattern-{field_name} ::= {regex_to_gbnf(regex_pattern)}"
- else:
- gbnf_type = PydanticDataType.STRING.value
-
- elif (
- isclass(origin_type)
- and issubclass(origin_type, float)
- and field_info
- and hasattr(field_info, "json_schema_extra")
- and field_info.json_schema_extra is not None
- ):
- # Retrieve precision attributes for floats
- max_precision = (
- field_info.json_schema_extra.get("max_precision") if field_info and hasattr(field_info,
- "json_schema_extra") else None
- )
- min_precision = (
- field_info.json_schema_extra.get("min_precision") if field_info and hasattr(field_info,
- "json_schema_extra") else None
- )
- max_digits = field_info.json_schema_extra.get("max_digit") if field_info and hasattr(field_info,
- "json_schema_extra") else None
- min_digits = field_info.json_schema_extra.get("min_digit") if field_info and hasattr(field_info,
- "json_schema_extra") else None
-
- # Generate GBNF rule for float with given attributes
- gbnf_type, rules = generate_gbnf_float_rules(
- max_digit=max_digits, min_digit=min_digits, max_precision=max_precision, min_precision=min_precision
- )
-
- elif (
- isclass(origin_type)
- and issubclass(origin_type, int)
- and field_info
- and hasattr(field_info, "json_schema_extra")
- and field_info.json_schema_extra is not None
- ):
- # Retrieve digit attributes for integers
- max_digits = field_info.json_schema_extra.get("max_digit") if field_info and hasattr(field_info,
- "json_schema_extra") else None
- min_digits = field_info.json_schema_extra.get("min_digit") if field_info and hasattr(field_info,
- "json_schema_extra") else None
-
- # Generate GBNF rule for integer with given attributes
- gbnf_type, rules = generate_gbnf_integer_rules(max_digit=max_digits, min_digit=min_digits)
- else:
- gbnf_type, rules = gbnf_type, []
-
- return gbnf_type, rules
-
-
-def generate_gbnf_grammar(model: type[BaseModel], processed_models: set[type[BaseModel]], created_rules: dict[str, list[str]]) -> tuple[list[str], bool]:
- """
-
- Generate GBnF Grammar
-
- Generates a GBnF grammar for a given model.
-
- :param model: A Pydantic model class to generate the grammar for. Must be a subclass of BaseModel.
- :param processed_models: A set of already processed models to prevent infinite recursion.
- :param created_rules: A dict containing already created rules to prevent duplicates.
- :return: A list of GBnF grammar rules in string format. And two booleans indicating if an extra markdown or triple quoted string is in the grammar.
- Example Usage:
- ```
- model = MyModel
- processed_models = set()
- created_rules = dict()
-
- gbnf_grammar = generate_gbnf_grammar(model, processed_models, created_rules)
- ```
- """
- if model in processed_models:
- return [], False
-
- processed_models.add(model)
- model_name = format_model_and_field_name(model.__name__)
-
- if not issubclass(model, BaseModel):
- # For non-Pydantic classes, generate model_fields from __annotations__ or __init__
- if hasattr(model, "__annotations__") and model.__annotations__:
- model_fields = {name: (typ, ...) for name, typ in get_type_hints(model).items()}
- else:
- init_signature = inspect.signature(model.__init__)
- parameters = init_signature.parameters
- model_fields = {name: (param.annotation, param.default) for name, param in parameters.items() if
- name != "self"}
- else:
- # For Pydantic models, use model_fields and check for ellipsis (required fields)
- model_fields = get_type_hints(model)
-
- model_rule_parts = []
- nested_rules = []
- has_markdown_code_block = False
- has_triple_quoted_string = False
- look_for_markdown_code_block = False
- look_for_triple_quoted_string = False
- for field_name, field_info in model_fields.items():
- if not issubclass(model, BaseModel):
- field_type, default_value = field_info
- # Check if the field is optional (not required)
- is_optional = (default_value is not inspect.Parameter.empty) and (default_value is not Ellipsis)
- else:
- field_type = field_info
- field_info = model.model_fields[field_name]
- is_optional = field_info.is_required is False and get_origin(field_type) is Optional
- rule_name, additional_rules = generate_gbnf_rule_for_type(
- model_name, format_model_and_field_name(field_name), field_type, is_optional, processed_models,
- created_rules, field_info
- )
- look_for_markdown_code_block = True if rule_name == "markdown_code_block" else False
- look_for_triple_quoted_string = True if rule_name == "triple_quoted_string" else False
- if not look_for_markdown_code_block and not look_for_triple_quoted_string:
- if rule_name not in created_rules:
- created_rules[rule_name] = additional_rules
- model_rule_parts.append(f' ws "\\"{field_name}\\"" ":" ws {rule_name}') # Adding escaped quotes
- nested_rules.extend(additional_rules)
- else:
- has_triple_quoted_string = look_for_triple_quoted_string
- has_markdown_code_block = look_for_markdown_code_block
-
- fields_joined = r' "," "\n" '.join(model_rule_parts)
- model_rule = rf'{model_name} ::= "{{" "\n" {fields_joined} "\n" ws "}}"'
-
- has_special_string = False
- if has_triple_quoted_string:
- model_rule += '"\\n" ws "}"'
- model_rule += '"\\n" triple-quoted-string'
- has_special_string = True
- if has_markdown_code_block:
- model_rule += '"\\n" ws "}"'
- model_rule += '"\\n" markdown-code-block'
- has_special_string = True
- all_rules = [model_rule] + nested_rules
-
- return all_rules, has_special_string
-
-
-def generate_gbnf_grammar_from_pydantic_models(
- models: list[type[BaseModel]], outer_object_name: str | None = None, outer_object_content: str | None = None,
- list_of_outputs: bool = False
-) -> str:
- """
- Generate GBNF Grammar from Pydantic Models.
-
- This method takes a list of Pydantic models and uses them to generate a GBNF grammar string. The generated grammar string can be used for parsing and validating data using the generated
- * grammar.
-
- Args:
- models (list[type[BaseModel]]): A list of Pydantic models to generate the grammar from.
- outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling.
- outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling.
- list_of_outputs (str, optional): Allows a list of output objects
- Returns:
- str: The generated GBNF grammar string.
-
- Examples:
- models = [UserModel, PostModel]
- grammar = generate_gbnf_grammar_from_pydantic(models)
- print(grammar)
- # Output:
- # root ::= UserModel | PostModel
- # ...
- """
- processed_models: set[type[BaseModel]] = set()
- all_rules = []
- created_rules: dict[str, list[str]] = {}
- if outer_object_name is None:
- for model in models:
- model_rules, _ = generate_gbnf_grammar(model, processed_models, created_rules)
- all_rules.extend(model_rules)
-
- if list_of_outputs:
- root_rule = r'root ::= (" "| "\n") "[" ws grammar-models ("," ws grammar-models)* ws "]"' + "\n"
- else:
- root_rule = r'root ::= (" "| "\n") grammar-models' + "\n"
- root_rule += "grammar-models ::= " + " | ".join(
- [format_model_and_field_name(model.__name__) for model in models])
- all_rules.insert(0, root_rule)
- return "\n".join(all_rules)
- elif outer_object_name is not None:
- if list_of_outputs:
- root_rule = (
- rf'root ::= (" "| "\n") "[" ws {format_model_and_field_name(outer_object_name)} ("," ws {format_model_and_field_name(outer_object_name)})* ws "]"'
- + "\n"
- )
- else:
- root_rule = f"root ::= {format_model_and_field_name(outer_object_name)}\n"
-
- model_rule = (
- rf'{format_model_and_field_name(outer_object_name)} ::= (" "| "\n") "{{" ws "\"{outer_object_name}\"" ":" ws grammar-models'
- )
-
- fields_joined = " | ".join(
- [rf"{format_model_and_field_name(model.__name__)}-grammar-model" for model in models])
-
- grammar_model_rules = f"\ngrammar-models ::= {fields_joined}"
- mod_rules = []
- for model in models:
- mod_rule = rf"{format_model_and_field_name(model.__name__)}-grammar-model ::= "
- mod_rule += (
- rf'"\"{model.__name__}\"" "," ws "\"{outer_object_content}\"" ":" ws {format_model_and_field_name(model.__name__)}' + "\n"
- )
- mod_rules.append(mod_rule)
- grammar_model_rules += "\n" + "\n".join(mod_rules)
-
- for model in models:
- model_rules, has_special_string = generate_gbnf_grammar(model, processed_models,
- created_rules)
-
- if not has_special_string:
- model_rules[0] += r'"\n" ws "}"'
-
- all_rules.extend(model_rules)
-
- all_rules.insert(0, root_rule + model_rule + grammar_model_rules)
- return "\n".join(all_rules)
-
-
-def get_primitive_grammar(grammar):
- """
- Returns the needed GBNF primitive grammar for a given GBNF grammar string.
-
- Args:
- grammar (str): The string containing the GBNF grammar.
-
- Returns:
- str: GBNF primitive grammar string.
- """
- type_list: list[type[object]] = []
- if "string-list" in grammar:
- type_list.append(str)
- if "boolean-list" in grammar:
- type_list.append(bool)
- if "integer-list" in grammar:
- type_list.append(int)
- if "float-list" in grammar:
- type_list.append(float)
- additional_grammar = [generate_list_rule(t) for t in type_list]
- primitive_grammar = r"""
-boolean ::= "true" | "false"
-null ::= "null"
-string ::= "\"" (
- [^"\\] |
- "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
- )* "\"" ws
-ws ::= ([ \t\n] ws)?
-float ::= ("-"? ([0] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
-
-integer ::= [0-9]+"""
-
- any_block = ""
- if "custom-class-any" in grammar:
- any_block = """
-value ::= object | array | string | number | boolean | null
-
-object ::=
- "{" ws (
- string ":" ws value
- ("," ws string ":" ws value)*
- )? "}" ws
-
-array ::=
- "[" ws (
- value
- ("," ws value)*
- )? "]" ws
-
-number ::= integer | float"""
-
- markdown_code_block_grammar = ""
- if "markdown-code-block" in grammar:
- markdown_code_block_grammar = r'''
-markdown-code-block ::= opening-triple-ticks markdown-code-block-content closing-triple-ticks
-markdown-code-block-content ::= ( [^`] | "`" [^`] | "`" "`" [^`] )*
-opening-triple-ticks ::= "```" "python" "\n" | "```" "c" "\n" | "```" "cpp" "\n" | "```" "txt" "\n" | "```" "text" "\n" | "```" "json" "\n" | "```" "javascript" "\n" | "```" "css" "\n" | "```" "html" "\n" | "```" "markdown" "\n"
-closing-triple-ticks ::= "```" "\n"'''
-
- if "triple-quoted-string" in grammar:
- markdown_code_block_grammar = r"""
-triple-quoted-string ::= triple-quotes triple-quoted-string-content triple-quotes
-triple-quoted-string-content ::= ( [^'] | "'" [^'] | "'" "'" [^'] )*
-triple-quotes ::= "'''" """
- return "\n" + "\n".join(additional_grammar) + any_block + primitive_grammar + markdown_code_block_grammar
-
-
-def generate_markdown_documentation(
- pydantic_models: list[type[BaseModel]], model_prefix="Model", fields_prefix="Fields",
- documentation_with_field_description=True
-) -> str:
- """
- Generate markdown documentation for a list of Pydantic models.
-
- Args:
- pydantic_models (list[type[BaseModel]]): list of Pydantic model classes.
- model_prefix (str): Prefix for the model section.
- fields_prefix (str): Prefix for the fields section.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- str: Generated text documentation.
- """
- documentation = ""
- pyd_models: list[tuple[type[BaseModel], bool]] = [(model, True) for model in pydantic_models]
- for model, add_prefix in pyd_models:
- if add_prefix:
- documentation += f"{model_prefix}: {model.__name__}\n"
- else:
- documentation += f"Model: {model.__name__}\n"
-
- # Handling multi-line model description with proper indentation
-
- class_doc = getdoc(model)
- base_class_doc = getdoc(BaseModel)
- class_description = class_doc if class_doc and class_doc != base_class_doc else ""
- if class_description != "":
- documentation += " Description: "
- documentation += format_multiline_description(class_description, 0) + "\n"
-
- if add_prefix:
- # Indenting the fields section
- documentation += f" {fields_prefix}:\n"
- else:
- documentation += f" Fields:\n" # noqa: F541
- if isclass(model) and issubclass(model, BaseModel):
- for name, field_type in get_type_hints(model).items():
- # if name == "markdown_code_block":
- # continue
- if get_origin(field_type) == list:
- element_type = get_args(field_type)[0]
- if isclass(element_type) and issubclass(element_type, BaseModel):
- pyd_models.append((element_type, False))
- if get_origin(field_type) == Union:
- element_types = get_args(field_type)
- for element_type in element_types:
- if isclass(element_type) and issubclass(element_type, BaseModel):
- pyd_models.append((element_type, False))
- documentation += generate_field_markdown(
- name, field_type, model, documentation_with_field_description=documentation_with_field_description
- )
- documentation += "\n"
-
- if hasattr(model, "Config") and hasattr(model.Config,
- "json_schema_extra") and "example" in model.Config.json_schema_extra:
- documentation += f" Expected Example Output for {format_model_and_field_name(model.__name__)}:\n"
- json_example = json.dumps(model.Config.json_schema_extra["example"])
- documentation += format_multiline_description(json_example, 2) + "\n"
-
- return documentation
-
-
-def generate_field_markdown(
- field_name: str, field_type: type[Any], model: type[BaseModel], depth=1,
- documentation_with_field_description=True
-) -> str:
- """
- Generate markdown documentation for a Pydantic model field.
-
- Args:
- field_name (str): Name of the field.
- field_type (type[Any]): Type of the field.
- model (type[BaseModel]): Pydantic model class.
- depth (int): Indentation depth in the documentation.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- str: Generated text documentation for the field.
- """
- indent = " " * depth
-
- field_info = model.model_fields.get(field_name)
- field_description = field_info.description if field_info and field_info.description else ""
-
- origin_type = get_origin(field_type)
- origin_type = field_type if origin_type is None else origin_type
-
- if origin_type == list:
- element_type = get_args(field_type)[0]
- field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)} of {format_model_and_field_name(element_type.__name__)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
- elif origin_type == Union:
- element_types = get_args(field_type)
- types = []
- for element_type in element_types:
- types.append(format_model_and_field_name(element_type.__name__))
- field_text = f"{indent}{field_name} ({' or '.join(types)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
- else:
- field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
-
- if not documentation_with_field_description:
- return field_text
-
- if field_description != "":
- field_text += f" Description: {field_description}\n"
-
- # Check for and include field-specific examples if available
- if hasattr(model, "Config") and hasattr(model.Config,
- "json_schema_extra") and "example" in model.Config.json_schema_extra:
- field_example = model.Config.json_schema_extra["example"].get(field_name)
- if field_example is not None:
- example_text = f"'{field_example}'" if isinstance(field_example, str) else field_example
- field_text += f"{indent} Example: {example_text}\n"
-
- if isclass(origin_type) and issubclass(origin_type, BaseModel):
- field_text += f"{indent} Details:\n"
- for name, type_ in get_type_hints(field_type).items():
- field_text += generate_field_markdown(name, type_, field_type, depth + 2)
-
- return field_text
-
-
-def format_json_example(example: dict[str, Any], depth: int) -> str:
- """
- Format a JSON example into a readable string with indentation.
-
- Args:
- example (dict): JSON example to be formatted.
- depth (int): Indentation depth.
-
- Returns:
- str: Formatted JSON example string.
- """
- indent = " " * depth
- formatted_example = "{\n"
- for key, value in example.items():
- value_text = f"'{value}'" if isinstance(value, str) else value
- formatted_example += f"{indent}{key}: {value_text},\n"
- formatted_example = formatted_example.rstrip(",\n") + "\n" + indent + "}"
- return formatted_example
-
-
-def generate_text_documentation(
- pydantic_models: list[type[BaseModel]], model_prefix="Model", fields_prefix="Fields",
- documentation_with_field_description=True
-) -> str:
- """
- Generate text documentation for a list of Pydantic models.
-
- Args:
- pydantic_models (list[type[BaseModel]]): List of Pydantic model classes.
- model_prefix (str): Prefix for the model section.
- fields_prefix (str): Prefix for the fields section.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- str: Generated text documentation.
- """
- documentation = ""
- pyd_models: list[tuple[type[BaseModel], bool]] = [(model, True) for model in pydantic_models]
- for model, add_prefix in pyd_models:
- if add_prefix:
- documentation += f"{model_prefix}: {model.__name__}\n"
- else:
- documentation += f"Model: {model.__name__}\n"
-
- # Handling multi-line model description with proper indentation
-
- class_doc = getdoc(model)
- base_class_doc = getdoc(BaseModel)
- class_description = class_doc if class_doc and class_doc != base_class_doc else ""
- if class_description != "":
- documentation += " Description: "
- documentation += "\n" + format_multiline_description(class_description, 2) + "\n"
-
- if isclass(model) and issubclass(model, BaseModel):
- documentation_fields = ""
- for name, field_type in get_type_hints(model).items():
- # if name == "markdown_code_block":
- # continue
- if get_origin(field_type) == list:
- element_type = get_args(field_type)[0]
- if isclass(element_type) and issubclass(element_type, BaseModel):
- pyd_models.append((element_type, False))
- if get_origin(field_type) == Union:
- element_types = get_args(field_type)
- for element_type in element_types:
- if isclass(element_type) and issubclass(element_type, BaseModel):
- pyd_models.append((element_type, False))
- documentation_fields += generate_field_text(
- name, field_type, model, documentation_with_field_description=documentation_with_field_description
- )
- if documentation_fields != "":
- if add_prefix:
- documentation += f" {fields_prefix}:\n{documentation_fields}"
- else:
- documentation += f" Fields:\n{documentation_fields}"
- documentation += "\n"
-
- if hasattr(model, "Config") and hasattr(model.Config,
- "json_schema_extra") and "example" in model.Config.json_schema_extra:
- documentation += f" Expected Example Output for {format_model_and_field_name(model.__name__)}:\n"
- json_example = json.dumps(model.Config.json_schema_extra["example"])
- documentation += format_multiline_description(json_example, 2) + "\n"
-
- return documentation
-
-
-def generate_field_text(
- field_name: str, field_type: type[Any], model: type[BaseModel], depth=1,
- documentation_with_field_description=True
-) -> str:
- """
- Generate text documentation for a Pydantic model field.
-
- Args:
- field_name (str): Name of the field.
- field_type (type[Any]): Type of the field.
- model (type[BaseModel]): Pydantic model class.
- depth (int): Indentation depth in the documentation.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- str: Generated text documentation for the field.
- """
- indent = " " * depth
-
- field_info = model.model_fields.get(field_name)
- field_description = field_info.description if field_info and field_info.description else ""
-
- if get_origin(field_type) == list:
- element_type = get_args(field_type)[0]
- field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)} of {format_model_and_field_name(element_type.__name__)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
- elif get_origin(field_type) == Union:
- element_types = get_args(field_type)
- types = []
- for element_type in element_types:
- types.append(format_model_and_field_name(element_type.__name__))
- field_text = f"{indent}{field_name} ({' or '.join(types)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
- else:
- field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)})"
- if field_description != "":
- field_text += ":\n"
- else:
- field_text += "\n"
-
- if not documentation_with_field_description:
- return field_text
-
- if field_description != "":
- field_text += f"{indent} Description: " + field_description + "\n"
-
- # Check for and include field-specific examples if available
- if hasattr(model, "Config") and hasattr(model.Config,
- "json_schema_extra") and "example" in model.Config.json_schema_extra:
- field_example = model.Config.json_schema_extra["example"].get(field_name)
- if field_example is not None:
- example_text = f"'{field_example}'" if isinstance(field_example, str) else field_example
- field_text += f"{indent} Example: {example_text}\n"
-
- if isclass(field_type) and issubclass(field_type, BaseModel):
- field_text += f"{indent} Details:\n"
- for name, type_ in get_type_hints(field_type).items():
- field_text += generate_field_text(name, type_, field_type, depth + 2)
-
- return field_text
-
-
-def format_multiline_description(description: str, indent_level: int) -> str:
- """
- Format a multiline description with proper indentation.
-
- Args:
- description (str): Multiline description.
- indent_level (int): Indentation level.
-
- Returns:
- str: Formatted multiline description.
- """
- indent = " " * indent_level
- return indent + description.replace("\n", "\n" + indent)
-
-
-def save_gbnf_grammar_and_documentation(
- grammar, documentation, grammar_file_path="./grammar.gbnf", documentation_file_path="./grammar_documentation.md"
-):
- """
- Save GBNF grammar and documentation to specified files.
-
- Args:
- grammar (str): GBNF grammar string.
- documentation (str): Documentation string.
- grammar_file_path (str): File path to save the GBNF grammar.
- documentation_file_path (str): File path to save the documentation.
-
- Returns:
- None
- """
- try:
- with open(grammar_file_path, "w") as file:
- file.write(grammar + get_primitive_grammar(grammar))
- print(f"Grammar successfully saved to {grammar_file_path}")
- except IOError as e:
- print(f"An error occurred while saving the grammar file: {e}")
-
- try:
- with open(documentation_file_path, "w") as file:
- file.write(documentation)
- print(f"Documentation successfully saved to {documentation_file_path}")
- except IOError as e:
- print(f"An error occurred while saving the documentation file: {e}")
-
-
-def remove_empty_lines(string):
- """
- Remove empty lines from a string.
-
- Args:
- string (str): Input string.
-
- Returns:
- str: String with empty lines removed.
- """
- lines = string.splitlines()
- non_empty_lines = [line for line in lines if line.strip() != ""]
- string_no_empty_lines = "\n".join(non_empty_lines)
- return string_no_empty_lines
-
-
-def generate_and_save_gbnf_grammar_and_documentation(
- pydantic_model_list,
- grammar_file_path="./generated_grammar.gbnf",
- documentation_file_path="./generated_grammar_documentation.md",
- outer_object_name: str | None = None,
- outer_object_content: str | None = None,
- model_prefix: str = "Output Model",
- fields_prefix: str = "Output Fields",
- list_of_outputs: bool = False,
- documentation_with_field_description=True,
-):
- """
- Generate GBNF grammar and documentation, and save them to specified files.
-
- Args:
- pydantic_model_list: List of Pydantic model classes.
- grammar_file_path (str): File path to save the generated GBNF grammar.
- documentation_file_path (str): File path to save the generated documentation.
- outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling.
- outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling.
- model_prefix (str): Prefix for the model section in the documentation.
- fields_prefix (str): Prefix for the fields section in the documentation.
- list_of_outputs (bool): Whether the output is a list of items.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- None
- """
- documentation = generate_markdown_documentation(
- pydantic_model_list, model_prefix, fields_prefix,
- documentation_with_field_description=documentation_with_field_description
- )
- grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content,
- list_of_outputs)
- grammar = remove_empty_lines(grammar)
- save_gbnf_grammar_and_documentation(grammar, documentation, grammar_file_path, documentation_file_path)
-
-
-def generate_gbnf_grammar_and_documentation(
- pydantic_model_list,
- outer_object_name: str | None = None,
- outer_object_content: str | None = None,
- model_prefix: str = "Output Model",
- fields_prefix: str = "Output Fields",
- list_of_outputs: bool = False,
- documentation_with_field_description=True,
-):
- """
- Generate GBNF grammar and documentation for a list of Pydantic models.
-
- Args:
- pydantic_model_list: List of Pydantic model classes.
- outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling.
- outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling.
- model_prefix (str): Prefix for the model section in the documentation.
- fields_prefix (str): Prefix for the fields section in the documentation.
- list_of_outputs (bool): Whether the output is a list of items.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- tuple: GBNF grammar string, documentation string.
- """
- documentation = generate_markdown_documentation(
- copy(pydantic_model_list), model_prefix, fields_prefix,
- documentation_with_field_description=documentation_with_field_description
- )
- grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content,
- list_of_outputs)
- grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar))
- return grammar, documentation
-
-
-def generate_gbnf_grammar_and_documentation_from_dictionaries(
- dictionaries: list[dict[str, Any]],
- outer_object_name: str | None = None,
- outer_object_content: str | None = None,
- model_prefix: str = "Output Model",
- fields_prefix: str = "Output Fields",
- list_of_outputs: bool = False,
- documentation_with_field_description=True,
-):
- """
- Generate GBNF grammar and documentation from a list of dictionaries.
-
- Args:
- dictionaries (list[dict]): List of dictionaries representing Pydantic models.
- outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling.
- outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling.
- model_prefix (str): Prefix for the model section in the documentation.
- fields_prefix (str): Prefix for the fields section in the documentation.
- list_of_outputs (bool): Whether the output is a list of items.
- documentation_with_field_description (bool): Include field descriptions in the documentation.
-
- Returns:
- tuple: GBNF grammar string, documentation string.
- """
- pydantic_model_list = create_dynamic_models_from_dictionaries(dictionaries)
- documentation = generate_markdown_documentation(
- copy(pydantic_model_list), model_prefix, fields_prefix,
- documentation_with_field_description=documentation_with_field_description
- )
- grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content,
- list_of_outputs)
- grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar))
- return grammar, documentation
-
-
-def create_dynamic_model_from_function(func: Callable[..., Any]):
- """
- Creates a dynamic Pydantic model from a given function's type hints and adds the function as a 'run' method.
-
- Args:
- func (Callable): A function with type hints from which to create the model.
-
- Returns:
- A dynamic Pydantic model class with the provided function as a 'run' method.
- """
-
- # Get the signature of the function
- sig = inspect.signature(func)
-
- # Parse the docstring
- assert func.__doc__ is not None
- docstring = parse(func.__doc__)
-
- dynamic_fields = {}
- param_docs = []
- for param in sig.parameters.values():
- # Exclude 'self' parameter
- if param.name == "self":
- continue
-
- # Assert that the parameter has a type annotation
- if param.annotation == inspect.Parameter.empty:
- raise TypeError(f"""Parameter '{param.name}' in function '{getattr(func, "__name__", "")}' lacks a type annotation""")
-
- # Find the parameter's description in the docstring
- param_doc = next((d for d in docstring.params if d.arg_name == param.name), None)
-
- # Assert that the parameter has a description
- if not param_doc or not param_doc.description:
- raise ValueError(
- f"""Parameter '{param.name}' in function '{getattr(func, "__name__", "")}' lacks a description in the docstring""")
-
- # Add parameter details to the schema
- param_docs.append((param.name, param_doc))
- if param.default == inspect.Parameter.empty:
- default_value = ...
- else:
- default_value = param.default
- dynamic_fields[param.name] = (
- param.annotation if param.annotation != inspect.Parameter.empty else str, default_value)
- # Creating the dynamic model
- dynamic_model = create_model(f"{getattr(func, '__name__')}", **dynamic_fields) # ty: ignore[no-matching-overload]
-
- for name, param_doc in param_docs:
- dynamic_model.model_fields[name].description = param_doc.description
-
- dynamic_model.__doc__ = docstring.short_description
-
- def run_method_wrapper(self):
- func_args = {name: getattr(self, name) for name, _ in dynamic_fields.items()}
- return func(**func_args)
-
- # Adding the wrapped function as a 'run' method
- setattr(dynamic_model, "run", run_method_wrapper)
- return dynamic_model
-
-
-def add_run_method_to_dynamic_model(model: type[BaseModel], func: Callable[..., Any]):
- """
- Add a 'run' method to a dynamic Pydantic model, using the provided function.
-
- Args:
- model (type[BaseModel]): Dynamic Pydantic model class.
- func (Callable): Function to be added as a 'run' method to the model.
-
- Returns:
- type[BaseModel]: Pydantic model class with the added 'run' method.
- """
-
- def run_method_wrapper(self):
- func_args = {name: getattr(self, name) for name in model.model_fields}
- return func(**func_args)
-
- # Adding the wrapped function as a 'run' method
- setattr(model, "run", run_method_wrapper)
-
- return model
-
-
-def create_dynamic_models_from_dictionaries(dictionaries: list[dict[str, Any]]):
- """
- Create a list of dynamic Pydantic model classes from a list of dictionaries.
-
- Args:
- dictionaries (list[dict]): List of dictionaries representing model structures.
-
- Returns:
- list[type[BaseModel]]: List of generated dynamic Pydantic model classes.
- """
- dynamic_models = []
- for func in dictionaries:
- model_name = format_model_and_field_name(func.get("name", ""))
- dyn_model = convert_dictionary_to_pydantic_model(func, model_name)
- dynamic_models.append(dyn_model)
- return dynamic_models
-
-
-def map_grammar_names_to_pydantic_model_class(pydantic_model_list):
- output = {}
- for model in pydantic_model_list:
- output[format_model_and_field_name(model.__name__)] = model
-
- return output
-
-
-def json_schema_to_python_types(schema):
- type_map = {
- "any": Any,
- "string": str,
- "number": float,
- "integer": int,
- "boolean": bool,
- "array": list,
- }
- return type_map[schema]
-
-
-def list_to_enum(enum_name, values):
- return Enum(enum_name, {value: value for value in values})
-
-
-def convert_dictionary_to_pydantic_model(dictionary: dict[str, Any], model_name: str = "CustomModel") -> type[Any]:
- """
- Convert a dictionary to a Pydantic model class.
-
- Args:
- dictionary (dict): Dictionary representing the model structure.
- model_name (str): Name of the generated Pydantic model.
-
- Returns:
- type[BaseModel]: Generated Pydantic model class.
- """
- fields: dict[str, Any] = {}
-
- if "properties" in dictionary:
- for field_name, field_data in dictionary.get("properties", {}).items():
- if field_data == "object":
- submodel = convert_dictionary_to_pydantic_model(dictionary, f"{model_name}_{field_name}")
- fields[field_name] = (submodel, ...)
- else:
- field_type = field_data.get("type", "str")
-
- if field_data.get("enum", []):
- fields[field_name] = (list_to_enum(field_name, field_data.get("enum", [])), ...)
- elif field_type == "array":
- items = field_data.get("items", {})
- if items != {}:
- array = {"properties": items}
- array_type = convert_dictionary_to_pydantic_model(array, f"{model_name}_{field_name}_items")
- fields[field_name] = (list[array_type], ...)
- else:
- fields[field_name] = (list, ...)
- elif field_type == "object":
- submodel = convert_dictionary_to_pydantic_model(field_data, f"{model_name}_{field_name}")
- fields[field_name] = (submodel, ...)
- elif field_type == "required":
- required = field_data.get("enum", [])
- for key, field in fields.items():
- if key not in required:
- optional_type = fields[key][0]
- fields[key] = (Optional[optional_type], ...)
- else:
- field_type = json_schema_to_python_types(field_type)
- fields[field_name] = (field_type, ...)
- if "function" in dictionary:
- for field_name, field_data in dictionary.get("function", {}).items():
- if field_name == "name":
- model_name = field_data
- elif field_name == "description":
- fields["__doc__"] = field_data
- elif field_name == "parameters":
- return convert_dictionary_to_pydantic_model(field_data, f"{model_name}")
-
- if "parameters" in dictionary:
- field_data = {"function": dictionary}
- return convert_dictionary_to_pydantic_model(field_data, f"{model_name}")
- if "required" in dictionary:
- required = dictionary.get("required", [])
- for key, field in fields.items():
- if key not in required:
- optional_type = fields[key][0]
- fields[key] = (Optional[optional_type], ...)
- custom_model = create_model(model_name, **fields)
- return custom_model
diff --git a/examples/pydantic_models_to_grammar_examples.py b/examples/pydantic_models_to_grammar_examples.py
deleted file mode 100755
index 35905fb97..000000000
--- a/examples/pydantic_models_to_grammar_examples.py
+++ /dev/null
@@ -1,312 +0,0 @@
-#!/usr/bin/env python3
-
-"""Function calling example using pydantic models."""
-
-from __future__ import annotations
-
-import argparse
-import datetime
-import json
-import logging
-import textwrap
-import sys
-from enum import Enum
-from typing import Optional, Union
-
-import requests
-from pydantic import BaseModel, Field
-from pydantic_models_to_grammar import (add_run_method_to_dynamic_model, convert_dictionary_to_pydantic_model,
- create_dynamic_model_from_function, generate_gbnf_grammar_and_documentation)
-
-
-def create_completion(host, prompt, gbnf_grammar):
- """Calls the /completion API on llama-server.
-
- See
- https://github.com/ggml-org/llama.cpp/tree/HEAD/tools/server#api-endpoints
- """
- print(f" Request:\n Grammar:\n{textwrap.indent(gbnf_grammar, ' ')}\n Prompt:\n{textwrap.indent(prompt.rstrip(), ' ')}")
- headers = {"Content-Type": "application/json"}
- data = {"prompt": prompt, "grammar": gbnf_grammar}
- result = requests.post(f"http://{host}/completion", headers=headers, json=data).json()
- assert data.get("error") is None, data
- logging.info("Result: %s", result)
- content = result["content"]
- print(f" Model: {result['model']}")
- print(f" Result:\n{textwrap.indent(json.dumps(json.loads(content), indent=2), ' ')}")
- return content
-
-
-# A function for the agent to send a message to the user.
-class SendMessageToUser(BaseModel):
- """Send a message to the User."""
- chain_of_thought: str = Field(..., description="Your chain of thought while sending the message.")
- message: str = Field(..., description="Message you want to send to the user.")
-
- def run(self):
- print(f"SendMessageToUser: {self.message}")
-
-
-def example_rce(host):
- """Minimal test case where the LLM call an arbitrary python function."""
- print("- example_rce")
- tools = [SendMessageToUser]
- gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation(
- pydantic_model_list=tools, outer_object_name="function",
- outer_object_content="function_parameters", model_prefix="Function", fields_prefix="Parameters")
- system_message = "You are an advanced AI, tasked to assist the user by calling functions in JSON format. The following are the available functions and their parameters and types:\n\n" + documentation
- user_message = "What is 42 * 42?"
- prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message}<|im_end|>\n<|im_start|>assistant"
- text = create_completion(host, prompt, gbnf_grammar)
- json_data = json.loads(text)
- tools_map = {tool.__name__:tool for tool in tools}
- # This finds "SendMessageToUser":
- tool = tools_map.get(json_data["function"])
- if not tool:
- print(f"Error: unknown tool {json_data['function']}")
- return 1
- tool(**json_data["function_parameters"]).run()
- return 0
-
-
-# Enum for the calculator tool.
-class MathOperation(Enum):
- ADD = "add"
- SUBTRACT = "subtract"
- MULTIPLY = "multiply"
- DIVIDE = "divide"
-
-
-# Simple pydantic calculator tool for the agent that can add, subtract,
-# multiply, and divide. Docstring and description of fields will be used in
-# system prompt.
-class Calculator(BaseModel):
- """Perform a math operation on two numbers."""
- number_one: Union[int, float] = Field(..., description="First number.")
- operation: MathOperation = Field(..., description="Math operation to perform.")
- number_two: Union[int, float] = Field(..., description="Second number.")
-
- def run(self):
- if self.operation == MathOperation.ADD:
- return self.number_one + self.number_two
- elif self.operation == MathOperation.SUBTRACT:
- return self.number_one - self.number_two
- elif self.operation == MathOperation.MULTIPLY:
- return self.number_one * self.number_two
- elif self.operation == MathOperation.DIVIDE:
- return self.number_one / self.number_two
- else:
- raise ValueError("Unknown operation.")
-
-
-def example_calculator(host):
- """Have the LLM ask to get a calculation done.
-
- Here the grammar gets generated by passing the available function models to
- generate_gbnf_grammar_and_documentation function. This also generates a
- documentation usable by the LLM.
-
- pydantic_model_list is the list of pydantic models outer_object_name is an
- optional name for an outer object around the actual model object. Like a
- "function" object with "function_parameters" which contains the actual model
- object. If None, no outer object will be generated outer_object_content is
- the name of outer object content.
-
- model_prefix is the optional prefix for models in the documentation. (Default="Output Model")
- fields_prefix is the prefix for the model fields in the documentation. (Default="Output Fields")
- """
- print("- example_calculator")
- tools = [SendMessageToUser, Calculator]
- gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation(
- pydantic_model_list=tools, outer_object_name="function",
- outer_object_content="function_parameters", model_prefix="Function", fields_prefix="Parameters")
- system_message = "You are an advanced AI, tasked to assist the user by calling functions in JSON format. The following are the available functions and their parameters and types:\n\n" + documentation
- user_message1 = "What is 42 * 42?"
- prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message1}<|im_end|>\n<|im_start|>assistant"
- text = create_completion(host, prompt, gbnf_grammar)
- json_data = json.loads(text)
- expected = {
- "function": "Calculator",
- "function_parameters": {
- "number_one": 42,
- "operation": "multiply",
- "number_two": 42
- }
- }
- if json_data != expected:
- print(" Result is not as expected!")
- tools_map = {tool.__name__:tool for tool in tools}
- # This finds "Calculator":
- tool = tools_map.get(json_data["function"])
- if not tool:
- print(f"Error: unknown tool {json_data['function']}")
- return 1
- result = tool(**json_data["function_parameters"]).run()
- print(f" Call {json_data['function']} gave result {result}")
- return 0
-
-
-class Category(Enum):
- """The category of the book."""
- Fiction = "Fiction"
- NonFiction = "Non-Fiction"
-
-
-class Book(BaseModel):
- """Represents an entry about a book."""
- title: str = Field(..., description="Title of the book.")
- author: str = Field(..., description="Author of the book.")
- published_year: Optional[int] = Field(..., description="Publishing year of the book.")
- keywords: list[str] = Field(..., description="A list of keywords.")
- category: Category = Field(..., description="Category of the book.")
- summary: str = Field(..., description="Summary of the book.")
-
-
-def example_struct(host):
- """A example structured output based on pydantic models.
-
- The LLM will create an entry for a Book database out of an unstructured
- text. We need no additional parameters other than our list of pydantic
- models.
- """
- print("- example_struct")
- tools = [Book]
- gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation(pydantic_model_list=tools)
- system_message = "You are an advanced AI, tasked to create a dataset entry in JSON for a Book. The following is the expected output model:\n\n" + documentation
- text = """The Feynman Lectures on Physics is a physics textbook based on some lectures by Richard Feynman, a Nobel laureate who has sometimes been called "The Great Explainer". The lectures were presented before undergraduate students at the California Institute of Technology (Caltech), during 1961–1963. The book's co-authors are Feynman, Robert B. Leighton, and Matthew Sands."""
- prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant"
- text = create_completion(host, prompt, gbnf_grammar)
- json_data = json.loads(text)
- # In this case, there's no function nor function_parameters.
- # Here the result will vary based on the LLM used.
- keys = sorted(["title", "author", "published_year", "keywords", "category", "summary"])
- if keys != sorted(json_data.keys()):
- print(f"Unexpected result: {sorted(json_data.keys())}")
- return 1
- book = Book(**json_data)
- print(f" As a Book object: %s" % book)
- return 0
-
-
-def get_current_datetime(output_format: Optional[str] = None):
- """Get the current date and time in the given format.
-
- Args:
- output_format: formatting string for the date and time, defaults to '%Y-%m-%d %H:%M:%S'
- """
- return datetime.datetime.now().strftime(output_format or "%Y-%m-%d %H:%M:%S")
-
-
-# Example function to get the weather.
-def get_current_weather(location, unit):
- """Get the current weather in a given location"""
- if "London" in location:
- return json.dumps({"location": "London", "temperature": "42", "unit": unit.value})
- elif "New York" in location:
- return json.dumps({"location": "New York", "temperature": "24", "unit": unit.value})
- elif "North Pole" in location:
- return json.dumps({"location": "North Pole", "temperature": "-42", "unit": unit.value})
- return json.dumps({"location": location, "temperature": "unknown"})
-
-
-def example_concurrent(host):
- """An example for parallel function calling with a Python function, a pydantic
- function model and an OpenAI like function definition.
- """
- print("- example_concurrent")
- # Function definition in OpenAI style.
- current_weather_tool = {
- "type": "function",
- "function": {
- "name": "get_current_weather",
- "description": "Get the current weather in a given location",
- "parameters": {
- "type": "object",
- "properties": {
- "location": {
- "type": "string",
- "description": "The city and state, e.g. San Francisco, CA",
- },
- "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
- },
- "required": ["location"],
- },
- },
- }
- # Convert OpenAI function definition into pydantic model.
- current_weather_tool_model = convert_dictionary_to_pydantic_model(current_weather_tool)
- # Add the actual function to a pydantic model.
- current_weather_tool_model = add_run_method_to_dynamic_model(current_weather_tool_model, get_current_weather)
-
- # Convert normal Python function to a pydantic model.
- current_datetime_model = create_dynamic_model_from_function(get_current_datetime)
-
- tools = [SendMessageToUser, Calculator, current_datetime_model, current_weather_tool_model]
- gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation(
- pydantic_model_list=tools, outer_object_name="function",
- outer_object_content="params", model_prefix="Function", fields_prefix="Parameters", list_of_outputs=True)
- system_message = "You are an advanced AI assistant. You are interacting with the user and with your environment by calling functions. You call functions by writing JSON objects, which represent specific function calls.\nBelow is a list of your available function calls:\n\n" + documentation
- text = """Get the date and time, get the current weather in celsius in London and solve the following calculation: 42 * 42"""
- prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant"
- text = create_completion(host, prompt, gbnf_grammar)
- json_data = json.loads(text)
- expected = [
- {
- "function": "get_current_datetime",
- "params": {
- "output_format": "%Y-%m-%d %H:%M:%S"
- }
- },
- {
- "function": "get_current_weather",
- "params": {
- "location": "London",
- "unit": "celsius"
- }
- },
- {
- "function": "Calculator",
- "params": {
- "number_one": 42,
- "operation": "multiply",
- "number_two": 42
- }
- }
- ]
- res = 0
- if json_data != expected:
- print(" Result is not as expected!")
- print(" This can happen on highly quantized models")
- res = 1
- tools_map = {tool.__name__:tool for tool in tools}
- for call in json_data:
- tool = tools_map.get(call["function"])
- if not tool:
- print(f"Error: unknown tool {call['function']}")
- return 1
- result = tool(**call["params"]).run()
- print(f" Call {call['function']} returned {result}")
- # Should output something like this:
- # Call get_current_datetime returned 2024-07-15 09:50:38
- # Call get_current_weather returned {"location": "London", "temperature": "42", "unit": "celsius"}
- # Call Calculator returned 1764
- return res
-
-
-def main():
- parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
- parser.add_argument("--host", default="localhost:9931", help="llama.cpp server")
- parser.add_argument("-v", "--verbose", action="store_true", help="enables logging")
- args = parser.parse_args()
- logging.basicConfig(level=logging.INFO if args.verbose else logging.ERROR)
- ret = 0
- # Comment out below to only run the example you want.
- ret = ret or example_rce(args.host)
- ret = ret or example_calculator(args.host)
- ret = ret or example_struct(args.host)
- ret = ret or example_concurrent(args.host)
- return ret
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/examples/realtime_client.py b/examples/realtime_client.py
deleted file mode 100644
index 3b806fab3..000000000
--- a/examples/realtime_client.py
+++ /dev/null
@@ -1,211 +0,0 @@
-#!/usr/bin/env python3
-"""Example client for the llama-server /v1/realtime websocket, with the openai SDK.
-
-Start a server with an audio model, e.g. LFM3-Audio:
-
- llama-server -hf LiquidAI/LFM3-Audio-2B-GGUF
-
-Spoken conversation, the audio of each wav file is one user turn. It is sent in chunks as it would come from a
-microphone, the server prefills it as it arrives so the response starts right after the last chunk:
-
- python examples/realtime_client.py --instructions "Respond with interleaved text and audio." question.wav
-
-Output modalities: left out, the model picks; "text", "audio" or "text,audio" (interleaved) force the response.
-Without --instructions, a forced mode comes with the system prompt of the LFM3-Audio or LFM3-Omni model card:
-
- python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
-
-Tool calls: --tools gives the conversation a demo tool, the client runs its calls and sends the results back:
-
- python examples/realtime_client.py --tools --text "Set the volume to 30."
-
-Text turns go in with --text, transcription sessions transcribe each file:
-
- python examples/realtime_client.py --text "What is the capital of France?"
- python examples/realtime_client.py --type transcription --language en speech.wav
-
-Live transcription models (Voxtral Realtime, LFM3-Audio live ASR) transcribe while the audio is sent, a transcription
-session opens with ?intent=transcription:
-
- python examples/realtime_client.py --type transcription speech.wav
-
-Requirements: pip install "openai[realtime]" numpy
-"""
-
-import argparse
-import asyncio
-import base64
-import json
-import time
-import wave
-
-import numpy as np
-from openai import AsyncOpenAI
-
-
-# system prompts of the LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B model cards for a forced output mode
-MODE_INSTRUCTIONS = {
- "audio": {"text,audio": "Respond with interleaved text and audio."},
- "omni": {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
-}
-# and with tools: calls, or an answer in interleaved text and speech
-TOOLS_INSTRUCTIONS = "Respond with function calls, and interleaved text and audio.\n\nList of tools:"
-
-# the demo tool of --tools
-DEMO_TOOLS = [{
- "type": "function",
- "name": "set_volume",
- "description": "Set the device speaker volume.",
- "parameters": {
- "type": "object",
- "properties": {"volume": {"type": "integer", "description": "Volume from 0 to 100."}},
- "required": ["volume"],
- },
-}]
-
-
-def run_tool(name, arguments):
- if name == "set_volume":
- return {"volume": json.loads(arguments)["volume"]}
- return {"error": f"unknown tool {name}"}
-
-
-def read_wav(path, rate):
- # PCM16 wav -> mono int16 at the given rate
- with wave.open(path, "rb") as f:
- assert f.getsampwidth() == 2, "the example reads PCM16 wav files"
- x = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16).astype(np.float32)
- x = x.reshape(-1, f.getnchannels()).mean(axis=1)
- sr = f.getframerate()
- if sr != rate:
- n = int(round(len(x) * rate / sr))
- x = np.interp(np.arange(n) * sr / rate, np.arange(len(x)), x)
- return np.clip(x, -32768, 32767).astype(np.int16)
-
-
-def write_wav(path, pcm, rate):
- with wave.open(path, "wb") as f:
- f.setnchannels(1)
- f.setsampwidth(2)
- f.setframerate(rate)
- f.writeframes(pcm.tobytes())
-
-
-async def main(args):
- # the SDK derives a wss:// URL from base_url, a plain http server needs ws://
- client = AsyncOpenAI(base_url=args.url, api_key=args.api_key, websocket_base_url=args.url.replace("http", "ws", 1))
- query = {"intent": "transcription"} if args.type == "transcription" else {}
- async with client.realtime.connect(model=args.model, extra_query=query) as conn:
- ev = await conn.recv()
- if ev.type == "error":
- print("error:", ev.error.message)
- return
- # the audio goes in at the rate of the model, PCM16 mono
- session_info = ev.model_dump(warnings=False)["session"]
- audio_fmt = session_info["audio"]
- card = "omni" if "omni" in str(session_info.get("model", "")).lower() else "audio"
- in_rate = audio_fmt["input"]["format"]["rate"]
- out_rate = (audio_fmt.get("output") or {}).get("format", {}).get("rate", 24000)
-
- session = {"type": args.type}
- if args.type == "realtime":
- instructions = args.instructions
- if instructions is None:
- instructions = TOOLS_INSTRUCTIONS if args.tools else MODE_INSTRUCTIONS[card].get(args.output_modalities)
- if args.tools:
- session["tools"] = DEMO_TOOLS
- if instructions is not None:
- session["instructions"] = instructions
- if args.output_modalities:
- session["output_modalities"] = args.output_modalities.split(",")
- elif args.language:
- session["audio"] = {"input": {"transcription": {"language": args.language}}}
- await conn.session.update(session=session) # ty: ignore[invalid-argument-type]
- ev = await conn.recv()
- if ev.type == "error":
- print("error:", ev.error.message)
- return
-
- turns = [("audio", path) for path in args.wav] + [("text", text) for text in args.text]
- for i, (kind, src) in enumerate(turns):
- t_last = 0.0
-
- # the input is sent while the events are read: live transcription arrives while the audio is sent
- async def send_turn():
- nonlocal t_last
- if kind == "audio":
- pcm = read_wav(src, in_rate)
- step = in_rate * args.chunk_ms // 1000
- for j in range(0, len(pcm), step):
- chunk = pcm[j:j + step]
- await conn.input_audio_buffer.append(audio=base64.b64encode(chunk.tobytes()).decode())
- if args.realtime:
- await asyncio.sleep(len(chunk) / in_rate)
- if args.type == "transcription":
- # "final" ends the input, models that transcribe with a delay then complete the last item
- await conn.send({"type": "input_audio_buffer.commit", "final": i == len(args.wav) - 1}) # ty: ignore[invalid-argument-type]
- else:
- await conn.conversation.item.create(item={
- "type": "message", "role": "user", "content": [{"type": "input_text", "text": src}]})
- if args.type == "realtime":
- # the audio not committed yet ends the user turn
- await conn.response.create()
- t_last = time.perf_counter()
-
- print(f"--- turn {i} ({kind}: {src})")
- sender = asyncio.create_task(send_turn())
- t_first = None
- audio = []
- async for ev in conn:
- if ev.type == "response.output_text.delta" or ev.type == "conversation.item.input_audio_transcription.delta":
- t_first = t_first or time.perf_counter()
- print(ev.delta, end="", flush=True)
- elif ev.type == "response.output_audio.delta":
- t_first = t_first or time.perf_counter()
- audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
- elif ev.type == "response.done":
- # the calls of the response are run, their results go back and the model answers them
- calls = [o for o in ev.model_dump()["response"]["output"] if o["type"] == "function_call"]
- for call in calls:
- result = json.dumps(run_tool(call["name"], call["arguments"]))
- print(f"\n[call {call['name']}({call['arguments']}) -> {result}]", flush=True)
- await conn.conversation.item.create(item={
- "type": "function_call_output", "call_id": call["call_id"], "output": result})
- if not calls:
- break
- await conn.response.create()
- elif ev.type == "conversation.item.input_audio_transcription.completed":
- break
- elif ev.type == "error":
- print("\nerror:", ev.error.message)
- return
- await sender
- print()
- if t_first is not None and t_first > t_last:
- print(f"first output {1000 * (t_first - t_last):.0f} ms after the end of the input")
- elif t_first is not None:
- print(f"output started {1000 * (t_last - t_first):.0f} ms before the end of the input, "
- f"done {1000 * (time.perf_counter() - t_last):.0f} ms after it")
- if audio:
- pcm_out = np.concatenate(audio)
- path = f"{args.out.removesuffix('.wav')}_{i}.wav" if len(turns) > 1 else args.out
- write_wav(path, pcm_out, out_rate)
- print(f"audio: {len(pcm_out) / out_rate:.2f} s -> {path}")
-
-
-if __name__ == "__main__":
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- p.add_argument("wav", nargs="*", help="PCM16 wav files, one user turn each")
- p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
- p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
- p.add_argument("--instructions", help="system prompt of the conversation")
- p.add_argument("--tools", action="store_true", help="give the conversation a demo tool (set_volume), its calls are run")
- p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
- p.add_argument("--language", help="transcription language, e.g. en")
- p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
- p.add_argument("--realtime", action=argparse.BooleanOptionalAction, default=True, help="send the audio in real time")
- p.add_argument("--out", default="response.wav", help="where the audio of the responses goes")
- p.add_argument("--url", default="http://127.0.0.1:9931/v1")
- p.add_argument("--api-key", default="none")
- p.add_argument("--model", default="default", help="model name, used by the router")
- asyncio.run(main(p.parse_args()))
diff --git a/examples/reason-act.sh b/examples/reason-act.sh
deleted file mode 100755
index 3c801920d..000000000
--- a/examples/reason-act.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-cd `dirname $0`
-cd ..
-
-# get -m model parameter otherwise defer to default
-if [ "$1" == "-m" ]; then
- MODEL="-m $2 "
-fi
-
-./llama-cli $MODEL --color \
- -f ./prompts/reason-act.txt \
- -i --interactive-first \
- --top_k 10000 --temp 0.2 --repeat_penalty 1 -t 7 -c 2048 \
- -r "Question:" -r "Observation:" --in-prefix " " \
- -n -1
diff --git a/examples/server-llama2-13B.sh b/examples/server-llama2-13B.sh
deleted file mode 100755
index fd5a57588..000000000
--- a/examples/server-llama2-13B.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd "$(dirname "$0")/.." || exit
-
-# Specify the model you want to use here:
-MODEL="${MODEL:-./models/llama-2-13b-chat.ggmlv3.q5_K_M.bin}"
-PROMPT_TEMPLATE=${PROMPT_TEMPLATE:-./prompts/chat-system.txt}
-
-# Adjust to the number of CPU cores you want to use.
-N_THREAD="${N_THREAD:-12}"
-
-# Note: you can also override the generation options by specifying them on the command line:
-GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 4096 --batch-size 1024}"
-
-
-# shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS
-./llama-server $GEN_OPTIONS \
- --model "$MODEL" \
- --threads "$N_THREAD" \
- --rope-freq-scale 1.0 \
- "$@"
-
-# I used this to test the model with mps, but omitted it from the general purpose. If you want to use it, just specify it on the command line.
-# -ngl 1 \
diff --git a/examples/server_embd.py b/examples/server_embd.py
deleted file mode 100644
index f8b0ffecd..000000000
--- a/examples/server_embd.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import asyncio
-import asyncio.threads
-import requests
-import numpy as np
-
-
-n = 8
-
-result = []
-
-async def requests_post_async(*args, **kwargs):
- return await asyncio.threads.to_thread(requests.post, *args, **kwargs)
-
-async def main():
- model_url = "http://127.0.0.1:6900"
- responses: list[requests.Response] = await asyncio.gather(*[requests_post_async(
- url= f"{model_url}/embedding",
- json= {"content": "a "*1022}
- ) for i in range(n)])
-
- for response in responses:
- embedding = response.json()["embedding"]
- print(embedding[-8:])
- result.append(embedding)
-
-asyncio.run(main())
-
-# compute cosine similarity
-
-for i in range(n-1):
- for j in range(i+1, n):
- embedding1 = np.array(result[i])
- embedding2 = np.array(result[j])
- similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2))
- print(f"Similarity between {i} and {j}: {similarity:.2f}")
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 6c6bea949..b18db3859 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -4,7 +4,6 @@
-r ./requirements-compare-llama-bench.txt
-r ./requirements-server-bench.txt
--r ./requirements-pydantic.txt
-r ./requirements-test-tokenizer-random.txt
-r ./requirements-convert_hf_to_gguf.txt
diff --git a/requirements/requirements-pydantic.txt b/requirements/requirements-pydantic.txt
deleted file mode 100644
index 1738494ed..000000000
--- a/requirements/requirements-pydantic.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-docstring_parser~=0.15
-pydantic~=2.11.7
-requests~=2.32.3
diff --git a/tools/server/README.md b/tools/server/README.md
index 27d141751..066ecb4dd 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1654,8 +1654,6 @@ LFM3-Omni is not trained to reason, run it with `--reasoning off` so that its tu
Tools go into the system prompt with the chat template, and the tool calls are parsed as in `/v1/chat/completions`. The client runs the calls of a response, sends their results as `function_call_output` items, then asks for the next response.
-See [examples/realtime_client.py](../../examples/realtime_client.py) for a client with the `openai` library.
-
### POST `/v1/audio/speech`: OpenAI-compatible Speech API
Generates speech from text, for models that can generate audio. The request is converted to a [`/tts`](#post-tts-generate-speech-audio-from-text) request, so the `/tts` options such as `lang`, `seed` or `speaker_ref_b64` can also be used.
diff --git a/ty.toml b/ty.toml
index 1a11151e6..2774cdbb8 100644
--- a/ty.toml
+++ b/ty.toml
@@ -22,13 +22,3 @@ unresolved-reference = "ignore"
import-not-found = "ignore"
missing-attribute = "ignore"
missing-union-attribute = "ignore"
-
-[[overrides]]
-include = [
- "./examples/pydantic_models_to_grammar.py",
-]
-
-[overrides.rules]
-unsupported-operator = "ignore"
-not-subscriptable = "ignore"
-invalid-type-form = "ignore"