Commit b78a39a2f for llama.cpp
commit b78a39a2f93b13a79a3e01aff3f14274efb43afc
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Fri Sep 11 22:00:57 2026 +0300
ci : run test-backend-ops as a dedicated ci/run.sh test (#28740)
* ci : run test-backend-ops as a dedicated gg test
Run test-backend-ops as a separate gg test in ci/run.sh so it is executed outside ctest. With GG_BUILD_HIGH_PERF it keeps the existing CPU-only invocation (-b CPU); otherwise it runs all available backends without a backend filter.
Remove the dedicated backend-ops workflow and keep test-backend-ops as a built target that is not registered with ctest to avoid duplicate runs.
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* ci : run test-backend-ops earlier and enable high-perf on kleidiai
Move the test-backend-ops gg test before test-llama-archs.
Enable GG_BUILD_HIGH_PERF and LLAMA_ARG_THREADS on the Graviton4 KleidiAI job and use the standard self-hosted results/mnt paths.
Add TODO markers for decoupling tests from libllama.
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* ci : run test-backend-ops in parallel
Pass -j $(nproc) to test-backend-ops in both high-perf and all-backend modes.
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* ci : disable parallel tests for ROCm
* cont : disable parallel tests with MoltenVK
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index 8879a6af1..86aba456c 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -33,7 +33,7 @@ env:
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
+ CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
jobs:
ubuntu-24-openvino:
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index fda4879e2..02a38466f 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -395,7 +395,11 @@ jobs:
- name: Test
id: ggml-ci
run: |
- LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+ LLAMA_ARG_THREADS=$(nproc) \
+ GG_BUILD_HIGH_PERF=1 \
+ GG_BUILD_NO_BF16=1 \
+ GG_BUILD_EXTRA_TESTS_0=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
cpu-arm64-graviton4-kleidiai:
runs-on: ah-ubuntu_22_04-c8g_8x
@@ -434,6 +438,8 @@ jobs:
- name: Test
id: ggml-ci
run: |
+ LLAMA_ARG_THREADS=$(nproc) \
GG_BUILD_KLEIDIAI=1 \
GG_BUILD_EXTRA_TESTS_0=1 \
- bash ./ci/run.sh ./tmp/results ./tmp/mnt
+ GG_BUILD_HIGH_PERF=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index 9de52e990..21d2a7735 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -164,9 +164,7 @@ jobs:
export GGML_VK_VISIBLE_DEVICES=0
export GGML_VK_DISABLE_F16=1
export GGML_VK_DISABLE_COOPMAT=1
- # This is using llvmpipe and runs slower than other backends
- # test-backend-ops is too slow on llvmpipe, skip it
- ctest -L main -E test-backend-ops --verbose --timeout 900
+ ctest -L main --verbose --timeout 900
windows:
runs-on: windows-2025
diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml
index e624e3ba8..8277abcc4 100644
--- a/.github/workflows/build-webgpu.yml
+++ b/.github/workflows/build-webgpu.yml
@@ -190,6 +190,4 @@ jobs:
id: cmake_test
run: |
cd build
- # This is using llvmpipe and runs slower than other backends
- # test-backend-ops is too slow on llvmpipe, skip it
- ctest -L main -E test-backend-ops --verbose --timeout 900
+ ctest -L main --verbose --timeout 900
diff --git a/ci/run.sh b/ci/run.sh
index 294cbe57b..1ceb19fd5 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -190,7 +190,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
+ CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
fi
## helpers
@@ -250,7 +250,7 @@ function gg_run_ctest_debug {
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
- (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
+ (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
set +e
}
@@ -768,25 +768,43 @@ function gg_check_build_requirements {
fi
}
-function gg_run_test_backend_ops_cpu {
+function gg_run_test_backend_ops {
cd ${SRC}
cd build-ci-release
set -e
- (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log
+ local args_extra="-j $(nproc)"
+
+ # TODO: fix multi-threaded for ROCm
+ # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
+ if [ ! -z ${GG_BUILD_ROCM} ]; then
+ args_extra=""
+ fi
+
+ # TODO: MoltenVK bug?
+ # https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
+ if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
+ args_extra=""
+ fi
+
+ if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
+ (time ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
+ else
+ (time ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
+ fi
set +e
}
-function gg_sum_test_backend_ops_cpu {
+function gg_sum_test_backend_ops {
gg_printf '### %s\n\n' "${ci}"
- gg_printf 'Runs test-backend-ops for CPU backend\n'
+ gg_printf 'Runs test-backend-ops\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
- gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)"
+ gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)"
gg_printf '```\n'
gg_printf '\n'
}
@@ -819,13 +837,11 @@ ret=0
test $ret -eq 0 && gg_run ctest_debug
test $ret -eq 0 && gg_run ctest_release
+test $ret -eq 0 && gg_run test_backend_ops
+
test $ret -eq 0 && gg_run test_llama_archs_models
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
-if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
- test $ret -eq 0 && gg_run test_backend_ops_cpu
-fi
-
if [ -z ${GG_BUILD_LOW_PERF} ]; then
test $ret -eq 0 && gg_run embd_bge_small
test $ret -eq 0 && gg_run rerank_tiny
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0c4e4d5a9..b3559a173 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -10,7 +10,7 @@ function(llama_build source)
endif()
add_executable(${TEST_TARGET} ${TEST_SOURCES})
- target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common)
+ target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) # TODO: [TAG_TESTS_LLAMA_LINK]
if (LLAMA_TESTS_INSTALL)
install(TARGETS ${TEST_TARGET} RUNTIME)
endif()
@@ -310,15 +310,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
# TODO: repair known memory leaks
llama_build_and_test(test-opt.cpp)
endif()
-llama_build_and_test(test-backend-ops.cpp)
-
-# the tensor API kernels come from a separate metallib - check they produce correct results
-# ref: https://github.com/ggml-org/llama.cpp/issues/27473
-if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)
- llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor
- ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K)
- set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1)
-endif()
+
+# TODO: make this test (and others) not link `libllama` as it is not needed [TAG_TESTS_LLAMA_LINK]
+llama_build(test-backend-ops.cpp)
llama_build_and_test(test-model-load-cancel.cpp LABEL "model")
llama_build_and_test(test-autorelease.cpp LABEL "model")