Commit 11ef8d60d6 for ffmpeg
commit 11ef8d60d670b6908185a4780ad1a33b77783b0f
Author: Niklas Haas <git@haasn.dev>
Date: Fri Oct 23 15:22:32 2026 +0200
avfilter/vf_libplacebo: add libplacebo_scene filter
This filter takes a scene graph in the libplacebo scene JSON format and
exposes it as a filter. The number of inputs and their properties are
determined by the scene graph.
I decided to make this a separate filter because it invalidates most of
the existing options, including e.g. the existing scaling and fitting logic.
For simplicity, scene graphs are always assumed to have a fixed canvas size,
which the user continues specifying at init time.
Requires libplacebo API version 386 or newer, which we probe the existence
for as a separate capability so we can selectively expose the filter.
Sponsored-by: nxtedition AB
Signed-off-by: Niklas Haas <git@haasn.dev>
diff --git a/configure b/configure
index e5f8c56a36..f70591050c 100755
--- a/configure
+++ b/configure
@@ -2659,6 +2659,7 @@ HAVE_LIST="
ioctl_posix
iokit
libdrm_getfb2
+ libplacebo_scene
libvmaf_picture_convert
libvmaf_picture_wrap
makeinfo_html
@@ -4192,6 +4193,7 @@ ladspa_filter_deps="ladspa libdl"
lcevc_filter_deps="liblcevc_dec"
lensfun_filter_deps="liblensfun version3"
libplacebo_filter_deps="libplacebo vulkan"
+libplacebo_scene_filter_deps="libplacebo libplacebo_scene vulkan"
lv2_filter_deps="lv2"
mcdeint_filter_deps="avcodec gpl"
metadata_filter_deps="avformat"
@@ -7340,6 +7342,7 @@ enabled libopus && {
}
}
enabled libplacebo && require_pkg_config libplacebo "libplacebo >= 7.351.0" libplacebo/vulkan.h pl_vulkan_create
+enabled libplacebo && check_pkg_config libplacebo_scene "libplacebo >= 7.386.0" libplacebo/compositor.h pl_queue_set_status
enabled libpulse && require_pkg_config libpulse libpulse pulse/pulseaudio.h pa_context_new
enabled libqrencode && require_pkg_config libqrencode libqrencode qrencode.h QRcode_encodeString
enabled libquirc && require libquirc quirc.h quirc_decode -lquirc
diff --git a/doc/filters.texi b/doc/filters.texi
index cc2b28a72b..ff3f636804 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -17098,6 +17098,7 @@ ffmpeg -i input.mov -vf lensfun=make=Canon:model="Canon EOS 100D":lens_model="Ca
Low Complexity Enhancement Video Codec filter based on liblcevc_dec
(@url{https://github.com/v-novaltd/LCEVCdec}).
+@anchor{libplacebo}
@section libplacebo
Flexible GPU-accelerated processing filter based on libplacebo
@@ -17859,6 +17860,60 @@ ffmpeg -init_hw_device vulkan -hwaccel vaapi -hwaccel_output_format vaapi ... -v
@end example
@end itemize
+@section libplacebo_scene
+Composite one or more inputs onto a canvas of fixed size and frame rate,
+according to a JSON scene graph. See https://libplacebo.org/json/ for more
+information.
+
+@subsection Options
+@table @option
+@item scene
+JSON scene graph description. Required. Use @code{/scene=file.json} to load it
+from a file.
+
+@item w
+@item h
+Output frame width and height. Required. Constant expressions are accepted.
+
+@item fps
+Output frame rate. Required.
+
+@item sar
+Output sample aspect ratio. The default of @code{0} means 1:1.
+
+@item format
+@item colorspace
+@item range
+@item color_primaries
+@item color_trc
+@item chroma_location
+@item alpha_mode
+@item inherit_device
+@item shader_cache
+@item apply_filmgrain
+@item apply_dolbyvision
+Same as for the @ref{libplacebo} filter.
+@end table
+
+The output color properties default to the scene's @code{color}, @code{repr}
+and @code{chroma_location}. They only control the output frame tags; rendering
+always follows the scene, including any animated changes.
+
+@subsection Example
+Show a second input in the bottom right quarter between 1 and 3 seconds:
+@example
+ffmpeg -i main.mkv -i overlay.mkv -filter_complex \
+ "[0][1]libplacebo_scene=w=1920:h=1080:fps=25:/scene=scene.json" out.mkv
+@end example
+with @file{scene.json} containing:
+@example
+@{ "inputs": [
+ @{ @},
+ @{ "start_pts": 1, "stop_pts": 3,
+ "dst_rect": @{ "x0": 0.5, "y0": 0.5 @} @}
+] @}
+@end example
+
@anchor{libvmaf}
@section libvmaf
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index f41dbeed2a..6d052562e8 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -378,6 +378,7 @@ OBJS-$(CONFIG_LATTICEPAL_FILTER) += vf_latticepal.o
OBJS-$(CONFIG_LENSCORRECTION_FILTER) += vf_lenscorrection.o
OBJS-$(CONFIG_LENSFUN_FILTER) += vf_lensfun.o
OBJS-$(CONFIG_LIBPLACEBO_FILTER) += vf_libplacebo.o vulkan.o vulkan_filter.o scale_eval.o
+OBJS-$(CONFIG_LIBPLACEBO_SCENE_FILTER) += vf_libplacebo.o vulkan.o vulkan_filter.o scale_eval.o
OBJS-$(CONFIG_LIBVMAF_FILTER) += vf_libvmaf.o framesync.o
OBJS-$(CONFIG_LIBVMAF_CUDA_FILTER) += vf_libvmaf.o framesync.o
OBJS-$(CONFIG_LIMITDIFF_FILTER) += vf_limitdiff.o framesync.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 931ac8481e..9f07bf83dc 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -351,6 +351,7 @@ extern const FFFilter ff_vf_lcevc;
extern const FFFilter ff_vf_lenscorrection;
extern const FFFilter ff_vf_lensfun;
extern const FFFilter ff_vf_libplacebo;
+extern const FFFilter ff_vf_libplacebo_scene;
extern const FFFilter ff_vf_libvmaf;
extern const FFFilter ff_vf_libvmaf_cuda;
extern const FFFilter ff_vf_limitdiff;
diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index a0b4d29585..161119bbc9 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -18,6 +18,8 @@
#include <math.h>
+#include "config_components.h"
+
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/eval.h"
@@ -39,6 +41,12 @@
#include <libplacebo/utils/frame_queue.h>
#include <libplacebo/vulkan.h>
+#if PL_API_VER >= 386
+#include <libplacebo/compositor.h>
+#else
+typedef const struct pl_scene_t *pl_scene;
+#endif
+
enum {
TONE_MAP_AUTO,
TONE_MAP_CLIP,
@@ -161,6 +169,10 @@ typedef struct LibplaceboContext {
int nb_inputs;
int nb_active;
+ /* scene graph (libplacebo_scene only) */
+ char *scene_json;
+ pl_scene scene;
+
/* settings */
char *out_format_string;
enum AVPixelFormat out_format;
@@ -1936,3 +1948,295 @@ const FFFilter ff_vf_libplacebo = {
FILTER_QUERY_FUNC2(libplacebo_query_format),
.flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE | FF_FILTER_FLAG_DYNAMIC_FRAME_PARAMS,
};
+
+#if CONFIG_LIBPLACEBO_SCENE_FILTER
+
+static int scene_setup_gpu(AVFilterContext *avctx)
+{
+ LibplaceboContext *s = avctx->priv;
+ pl_compositor comp = s->scene->comp;
+ if (comp->inputs[0]->renderer)
+ return 0; /* already initialized */
+
+ if (!pl_compositor_init(comp, s->gpu))
+ return AVERROR_EXTERNAL;
+
+ return 0;
+}
+
+static int scene_config_input(AVFilterLink *inlink)
+{
+ int err;
+ AVFilterContext *avctx = inlink->dst;
+ LibplaceboContext *s = avctx->priv;
+
+ if (s->inherit_device)
+ RET(inherit_device(inlink));
+ RET(init_vulkan(avctx, choose_default_device(avctx)));
+ RET(scene_setup_gpu(avctx));
+ return config_input_vulkan(inlink);
+
+fail:
+ return err;
+}
+
+static int scene_init(AVFilterContext *avctx)
+{
+ int err;
+ LibplaceboContext *s = avctx->priv;
+ pl_compositor comp;
+
+ if (!s->scene_json || !s->w_expr || !s->h_expr || !s->fps_string) {
+ av_log(avctx, AV_LOG_ERROR, "The 'scene', 'w', 'h' and 'fps' options "
+ "are required\n");
+ return AVERROR(EINVAL);
+ }
+
+ RET(av_parse_video_rate(&s->fps, s->fps_string));
+ RET(init_common(avctx));
+
+ s->scene = pl_scene_create_json(s->log, s->scene_json, strlen(s->scene_json));
+ if (!s->scene) {
+ av_log(avctx, AV_LOG_ERROR, "Failed parsing scene description\n");
+ return AVERROR(EINVAL);
+ }
+
+ comp = s->scene->comp;
+ s->nb_inputs = comp->num_inputs;
+ if (!s->nb_inputs) {
+ av_log(avctx, AV_LOG_ERROR, "Scene contains no inputs\n");
+ return AVERROR(EINVAL);
+ }
+
+ /* Output color properties not set on the filter default to the scene's */
+ if (s->colorspace < 0 && comp->repr.sys)
+ s->colorspace = pl_system_to_av(comp->repr.sys);
+ if (s->color_range < 0 && comp->repr.levels)
+ s->color_range = pl_levels_to_av(comp->repr.levels);
+ if (s->alpha_mode < 0 && comp->repr.alpha)
+ s->alpha_mode = pl_alpha_to_av(comp->repr.alpha);
+ if (s->chroma_location < 0 && comp->chroma_location)
+ s->chroma_location = pl_chroma_to_av(comp->chroma_location);
+ if (s->color_primaries < 0 && comp->color.primaries)
+ s->color_primaries = pl_primaries_to_av(comp->color.primaries);
+ if (s->color_trc < 0 && comp->color.transfer)
+ s->color_trc = pl_transfer_to_av(comp->color.transfer);
+
+ RET(append_inputs(avctx, s->nb_inputs, &scene_config_input));
+
+ if (!s->inherit_device) {
+ RET(init_vulkan(avctx, choose_default_device(avctx)));
+ RET(scene_setup_gpu(avctx));
+ } /* else deferred to scene_config_input() */
+
+ return 0;
+
+fail:
+ return err;
+}
+
+static void scene_uninit(AVFilterContext *avctx)
+{
+ LibplaceboContext *s = avctx->priv;
+ pl_scene_destroy(&s->scene);
+ libplacebo_uninit(avctx);
+}
+
+static int scene_eval_dim(AVFilterContext *avctx, const char *expr, int *out)
+{
+ double res;
+ int err = av_expr_parse_and_eval(&res, expr, NULL, NULL, NULL, NULL,
+ NULL, NULL, NULL, 0, avctx);
+ if (err < 0 || res < 1 || res > INT_MAX) {
+ av_log(avctx, AV_LOG_ERROR, "Invalid output dimension: %s\n", expr);
+ return err < 0 ? err : AVERROR(EINVAL);
+ }
+
+ *out = lrint(res);
+ return 0;
+}
+
+static int scene_config_output(AVFilterLink *outlink)
+{
+ int err;
+ AVFilterContext *avctx = outlink->src;
+ LibplaceboContext *s = avctx->priv;
+ FilterLink *l = ff_filter_link(outlink);
+
+ RET(scene_eval_dim(avctx, s->w_expr, &outlink->w));
+ RET(scene_eval_dim(avctx, s->h_expr, &outlink->h));
+ outlink->sample_aspect_ratio = s->out_sar.num ? s->out_sar : (AVRational){ 1, 1 };
+ outlink->time_base = av_inv_q(s->fps);
+ l->frame_rate = s->fps;
+
+ RET(config_output_vulkan(outlink));
+ return 0;
+
+fail:
+ return err;
+}
+
+static int scene_handle_input(AVFilterContext *ctx, int idx)
+{
+ int ret, status;
+ LibplaceboContext *s = ctx->priv;
+ AVFilterLink *inlink = ctx->inputs[idx];
+ pl_input input = s->scene->comp->inputs[idx];
+ const bool deinterlace = input->params && input->params->deinterlace_params;
+ AVFrame *in;
+ int64_t pts;
+
+ while ((ret = ff_inlink_consume_frame(inlink, &in)) > 0)
+ push_frame(s, input->queue, inlink, in, deinterlace);
+
+ if (ret < 0)
+ return ret;
+
+ if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
+ pl_queue_set_status(input->queue, status == AVERROR_EOF ? PL_QUEUE_EOF : PL_QUEUE_ERR);
+ input->user_data = (void *) (intptr_t) status;
+ }
+
+ return 0;
+}
+
+static int scene_output_frame(AVFilterContext *ctx, int64_t pts,
+ const struct pl_queue_params *qparams,
+ const struct pl_compositor_status *st)
+{
+ int err = 0;
+ bool ok;
+ LibplaceboContext *s = ctx->priv;
+ AVFilterLink *outlink = ctx->outputs[0];
+ const AVFrame *ref = st->base ? pl_get_mapped_avframe(st->base) : NULL;
+ struct pl_frame target;
+ AVFrame *out;
+
+ out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+ if (!out)
+ return AVERROR(ENOMEM);
+
+ RET(set_output_props(s, outlink, out, ref, true)); /* always progressive */
+ out->pts = pts;
+ out->duration = 1;
+
+ RET(map_output(s, out, &target));
+ ok = pl_compositor_render(s->scene->comp, &target, qparams, NULL);
+ RET(unmap_output(s, &target, out));
+ if (!ok) {
+ err = AVERROR_EXTERNAL;
+ goto fail;
+ }
+ return ff_filter_frame(outlink, out);
+
+fail:
+ av_frame_free(&out);
+ return err;
+}
+
+static int scene_activate(AVFilterContext *ctx)
+{
+ int ret, retry = 0;
+ LibplaceboContext *s = ctx->priv;
+ AVFilterLink *outlink = ctx->outputs[0];
+ FilterLink *outl = ff_filter_link(outlink);
+ pl_compositor comp = s->scene->comp;
+
+ FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, ctx);
+ pl_log_level_update(s->log, get_log_level());
+
+ for (int i = 0; i < s->nb_inputs; i++) {
+ if ((ret = scene_handle_input(ctx, i)) < 0)
+ return ret;
+ }
+
+ if (!ff_outlink_frame_wanted(outlink))
+ return FFERROR_NOT_READY;
+
+ const int64_t out_pts = outl->frame_count_out;
+ const struct pl_queue_params qparams = *pl_queue_params(
+ .pts = TS2T(out_pts, outlink->time_base),
+ .vsync_duration = av_q2d(outlink->time_base),
+ );
+
+ pl_scene_update(s->scene, qparams.pts);
+
+ struct pl_compositor_status st;
+ st = pl_compositor_update(comp, &qparams, pl_canvas_size(
+ .width = outlink->w,
+ .height = outlink->h,
+ .pixel_aspect_ratio = av_q2d(outlink->sample_aspect_ratio),
+ ));
+
+ for (int i = 0; i < s->nb_inputs; i++) {
+ AVFilterLink *inlink = ctx->inputs[i];
+ switch (st.input_status[i].status) {
+ case PL_QUEUE_MORE:
+ if (!ff_outlink_get_status(inlink)) {
+ ff_inlink_request_frame(inlink);
+ retry = 1;
+ }
+ break;
+ case PL_QUEUE_EOF:
+ /* Also closes inputs that reached their stop_pts early */
+ ff_inlink_set_status(inlink, AVERROR_EOF);
+ break;
+ case PL_QUEUE_ERR: {
+ /* Forward the error of the failed input */
+ const int status = (intptr_t) comp->inputs[i]->user_data;
+ if (!status || status == AVERROR_EOF)
+ return AVERROR_EXTERNAL;
+ ff_outlink_set_status(outlink, status, out_pts);
+ return 0;
+ }
+ default:
+ break;
+ }
+ }
+
+ if (retry)
+ return 0;
+
+ if (st.status == PL_QUEUE_EOF) {
+ ff_outlink_set_status(outlink, AVERROR_EOF, out_pts);
+ return 0;
+ }
+
+ return scene_output_frame(ctx, out_pts, &qparams, &st);
+}
+
+static const AVOption libplacebo_scene_options[] = {
+ { "scene", "JSON scene graph description", OFFSET(scene_json), AV_OPT_TYPE_STRING, .flags = STATIC },
+ { "w", "Output video frame width", OFFSET(w_expr), AV_OPT_TYPE_STRING, .flags = STATIC },
+ { "h", "Output video frame height", OFFSET(h_expr), AV_OPT_TYPE_STRING, .flags = STATIC },
+ { "fps", "Output video frame rate", OFFSET(fps_string), AV_OPT_TYPE_STRING, .flags = STATIC },
+ OPTS_COMMON
+ { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(libplacebo_scene);
+
+static const AVFilterPad libplacebo_scene_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = &scene_config_output,
+ },
+};
+
+const FFFilter ff_vf_libplacebo_scene = {
+ .p.name = "libplacebo_scene",
+ .p.description = NULL_IF_CONFIG_SMALL("Composite inputs according to a libplacebo scene graph"),
+ .p.priv_class = &libplacebo_scene_class,
+ .p.flags = AVFILTER_FLAG_HWDEVICE | AVFILTER_FLAG_DYNAMIC_INPUTS,
+ .priv_size = sizeof(LibplaceboContext),
+ .init = &scene_init,
+ .uninit = &scene_uninit,
+ .activate = &scene_activate,
+ .process_command = &ff_filter_process_command,
+ FILTER_OUTPUTS(libplacebo_scene_outputs),
+ FILTER_QUERY_FUNC2(libplacebo_query_format),
+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE | FF_FILTER_FLAG_DYNAMIC_FRAME_PARAMS,
+};
+
+#endif /* CONFIG_LIBPLACEBO_SCENE_FILTER */