Commit b4738344cd for ffmpeg
commit b4738344cdbe95522c84de41d34d15527830712b
Author: Kacper Michajłow <kasper93@gmail.com>
Date: Sat Oct 17 01:37:53 2026 +0200
avformat/hls: apply the X-TIMESTAMP-MAP header of WebVTT segments
Set the timestamp_map option of the webvtt demuxer for the subtitle
segments in place of prefer_hls_mpegts_pts, which no demuxer has, so the
cue times land on the MPEG-TS timeline of the audio and video. That time
is on the 33 bit PES clock, so each cue is put on the period of that clock
nearest to the position of the media, which is read first when both are
enabled. A cue read without any media does not become that position, and
a cue still buffered when the media is enabled is placed relative to it
before it is returned.
Signed-off-by: Kacper Michajłow <kasper93@gmail.com>
diff --git a/doc/demuxers.texi b/doc/demuxers.texi
index c1edd448cc..e3e2018471 100644
--- a/doc/demuxers.texi
+++ b/doc/demuxers.texi
@@ -574,6 +574,9 @@ available in a metadata key named "variant_bitrate". It is unset when
the stream is shared by variants with different bitrates. The value is
always available in the corresponding program's metadata.
+The X-TIMESTAMP-MAP header of WebVTT subtitle segments is applied, so the
+cue times are on the timeline of the other renditions.
+
It accepts the following options:
@table @option
diff --git a/libavformat/hls.c b/libavformat/hls.c
index eff7477223..f4d519a093 100644
--- a/libavformat/hls.c
+++ b/libavformat/hls.c
@@ -117,6 +117,7 @@ struct playlist {
int index;
AVFormatContext *ctx;
AVPacket *pkt;
+ int64_t last_cue_pts; /* reference for the cues read without media */
int has_noheader_flag;
/* main demuxer streams associated with this playlist
@@ -416,6 +417,7 @@ static struct playlist *new_playlist(HLSContext *c, const char *url,
av_strlcpy(pls->url, abs_url, sizeof(pls->url));
pls->ts_offset = AV_NOPTS_VALUE;
+ pls->last_cue_pts = AV_NOPTS_VALUE;
pls->seek_timestamp = AV_NOPTS_VALUE;
pls->first_read_seq_no = -1;
@@ -2122,6 +2124,60 @@ static int init_subtitle_context(struct playlist *pls)
return ret;
}
+static AVRational get_timebase(struct playlist *pls);
+
+/* The position of the media, the reference for the cue times: the last
+ * returned packet, or a packet buffered from a media playlist before
+ * anything was returned. AV_NOPTS_VALUE without media. */
+static int64_t media_position(HLSContext *c)
+{
+ if (c->cur_timestamp != AV_NOPTS_VALUE)
+ return c->cur_timestamp;
+ for (int i = 0; i < c->n_playlists; i++) {
+ struct playlist *pls = c->playlists[i];
+
+ if (!pls->is_subtitle && pls->needed && pls->pkt->data &&
+ pls->pkt->dts != AV_NOPTS_VALUE)
+ return av_rescale_q(pls->pkt->dts, get_timebase(pls), AV_TIME_BASE_Q);
+ }
+ return AV_NOPTS_VALUE;
+}
+
+/* The MPEG-TS time of a cue is on the 33 bit PES clock, this is the period
+ * of that clock nearest to ref, both in MPEG_TIME_BASE_Q. */
+static int64_t nearest_period(int64_t pts, int64_t ref)
+{
+ const int64_t period = 1LL << 33, half = 1LL << 32;
+ int64_t d = pts - ref;
+
+ return pts - (d >= 0 ? d + half : d - half) / period * period;
+}
+
+static void unwrap_pkt(AVPacket *pkt, AVRational tb, int64_t ref)
+{
+ int64_t pts;
+
+ if (pkt->pts == AV_NOPTS_VALUE)
+ return;
+ pts = nearest_period(av_rescale_q(pkt->pts, tb, MPEG_TIME_BASE_Q), ref);
+ pkt->pts = pkt->dts = av_rescale_q(pts, MPEG_TIME_BASE_Q, tb);
+}
+
+/* Put a cue on the period nearest to the media position, where the
+ * timestamps of the other renditions continue after a wrap. Without media
+ * a cue follows the previous cue of its playlist. */
+static void unwrap_cue_pts(HLSContext *c, struct playlist *v, AVPacket *pkt)
+{
+ AVRational tb = v->ctx->streams[pkt->stream_index]->time_base;
+ int64_t ref = media_position(c);
+
+ if (ref != AV_NOPTS_VALUE)
+ unwrap_pkt(pkt, tb, av_rescale_q(ref, AV_TIME_BASE_Q, MPEG_TIME_BASE_Q));
+ else if (v->last_cue_pts != AV_NOPTS_VALUE)
+ unwrap_pkt(pkt, tb, av_rescale_q(v->last_cue_pts, tb, MPEG_TIME_BASE_Q));
+ v->last_cue_pts = pkt->pts;
+}
+
static int read_subtitle_packet(struct playlist *v, AVPacket *pkt)
{
HLSContext *c = v->parent->priv_data;
@@ -2143,7 +2199,8 @@ restart:
ret = av_read_frame(v->ctx, v->pkt);
if (!ret) {
- return ret;
+ unwrap_cue_pts(c, v, v->pkt);
+ return 0;
}
ff_format_io_close(v->parent, &v->input);
v->cur_seq_no++;
@@ -2603,7 +2660,7 @@ static int hls_read_header(AVFormatContext *s)
highest_cur_seq_no = FFMAX(highest_cur_seq_no, pls->cur_seq_no);
}
- av_dict_set(&c->seg_format_opts, "prefer_hls_mpegts_pts", "1", 0);
+ av_dict_set(&c->seg_format_opts, "timestamp_map", "1", 0);
/* Open the demuxer for each playlist */
for (i = 0; i < c->n_playlists; i++) {
@@ -2849,8 +2906,10 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
pls->input_read_done = 0;
ff_format_io_close(pls->parent, &pls->input_next);
pls->input_next_requested = 0;
- if (pls->is_subtitle)
+ if (pls->is_subtitle) {
avformat_close_input(&pls->ctx);
+ pls->last_cue_pts = AV_NOPTS_VALUE;
+ }
pls->needed = 0;
changed = 1;
av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);
@@ -2927,16 +2986,24 @@ static int metadata_has_changed(const AVDictionary *current,
static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
{
HLSContext *c = s->priv_data;
- int ret, i, minplaylist = -1;
+ int ret, i, minplaylist = -1, deferred = 0, unanchored = 0;
recheck_discard_flags(s, c->first_packet);
c->first_packet = 0;
+retry:
for (i = 0; i < c->n_playlists; i++) {
struct playlist *pls = c->playlists[i];
/* Make sure we've got one buffered packet from each open playlist
* stream */
if (pls->needed && !pls->pkt->data) {
+ /* a cue is placed relative to the media, so the media playlists
+ * buffer their packets first */
+ if (pls->is_subtitle && !unanchored &&
+ media_position(c) == AV_NOPTS_VALUE) {
+ deferred = 1;
+ continue;
+ }
while (1) {
int64_t ts_diff;
AVRational tb;
@@ -3038,6 +3105,14 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
av_packet_unref(pls->pkt);
}
}
+ }
+
+ for (i = 0; i < c->n_playlists; i++) {
+ struct playlist *pls = c->playlists[i];
+
+ /* a cue buffered without media is placed once the media is */
+ if (pls->is_subtitle && pls->pkt->data)
+ unwrap_cue_pts(c, pls, pls->pkt);
/* Check if this stream has the packet with the lowest dts */
if (pls->pkt->data) {
struct playlist *minpls = minplaylist < 0 ?
@@ -3055,6 +3130,13 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
}
}
+ /* without media the cues are read as they are */
+ if (minplaylist < 0 && deferred) {
+ deferred = 0;
+ unanchored = 1;
+ goto retry;
+ }
+
/* If we got a packet, return it */
if (minplaylist >= 0) {
struct playlist *pls = c->playlists[minplaylist];
@@ -3115,7 +3197,9 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
av_packet_move_ref(pkt, pls->pkt);
pkt->stream_index = st->index;
- if (pkt->dts != AV_NOPTS_VALUE)
+ /* a cue read without media does not set the media position */
+ if (pkt->dts != AV_NOPTS_VALUE &&
+ (!pls->is_subtitle || c->first_timestamp != AV_NOPTS_VALUE))
c->cur_timestamp = av_rescale_q(pkt->dts,
ist->time_base,
AV_TIME_BASE_Q);
@@ -3247,8 +3331,10 @@ static int hls_read_seek(AVFormatContext *s, int stream_index,
for (j = 0; j < (int)pls->ctx->nb_streams; j++)
ffstream(pls->ctx->streams[j])->pts_wrap_reference = AV_NOPTS_VALUE;
}
- if (pls->is_subtitle)
+ if (pls->is_subtitle) {
avformat_close_input(&pls->ctx);
+ pls->last_cue_pts = AV_NOPTS_VALUE;
+ }
/* Reset the init segment so it's re-fetched and served appropriately */
pls->cur_init_section = NULL;
diff --git a/tests/ref/fate/hls-shared-renditions-demux b/tests/ref/fate/hls-shared-renditions-demux
index 31a8195d48..105f039181 100644
--- a/tests/ref/fate/hls-shared-renditions-demux
+++ b/tests/ref/fate/hls-shared-renditions-demux
@@ -27,11 +27,11 @@ packet|codec_type=audio|stream_index=1|pts=162000|pts_time=1.800000|dts=162000|d
packet|codec_type=audio|stream_index=1|pts=168480|pts_time=1.872000|dts=168480|dts_time=1.872000|duration=6480|duration_time=0.072000|size=1440|pos=39668|flags=K__|data_hash=CRC32:cb00dd5b|side_datum/mpegts_stream_id:side_data_type=MPEGTS Stream ID|side_datum/mpegts_stream_id:id=192
packet|codec_type=audio|stream_index=1|pts=174960|pts_time=1.944000|dts=174960|dts_time=1.944000|duration=6480|duration_time=0.072000|size=1440|pos=41172|flags=K__|data_hash=CRC32:cb00dd5b|side_datum/mpegts_stream_id:side_data_type=MPEGTS Stream ID|side_datum/mpegts_stream_id:id=192
program|program_id=0|program_num=0|nb_streams=2|pmt_pid=0|pcr_pid=0|tag:variant_bitrate=64000
-stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/1000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
+stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
stream|index=1|codec_name=mp2|profile=unknown|level=-99|codec_type=audio|codec_tag_string=[4][0][0][0]|codec_tag=0x0004|mime_codec_string=mp4a.40.33|sample_fmt=fltp|sample_rate=16000|channels=1|channel_layout=mono|bits_per_sample=0|initial_padding=0|seek_preroll=0|id=0x1|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=160000|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=28|disposition:default=1|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=en|tag:comment=English
program|program_id=1|program_num=0|nb_streams=2|pmt_pid=0|pcr_pid=0|tag:variant_bitrate=128000
-stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/1000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
+stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
stream|index=1|codec_name=mp2|profile=unknown|level=-99|codec_type=audio|codec_tag_string=[4][0][0][0]|codec_tag=0x0004|mime_codec_string=mp4a.40.33|sample_fmt=fltp|sample_rate=16000|channels=1|channel_layout=mono|bits_per_sample=0|initial_padding=0|seek_preroll=0|id=0x1|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=160000|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=28|disposition:default=1|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=en|tag:comment=English
-stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/1000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
+stream|index=0|codec_name=webvtt|profile=unknown|level=-99|codec_type=subtitle|codec_tag_string=[0][0][0][0]|codec_tag=0x0000|width=N/A|height=N/A|id=0x0|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=N/A|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=de|tag:comment=German
stream|index=1|codec_name=mp2|profile=unknown|level=-99|codec_type=audio|codec_tag_string=[4][0][0][0]|codec_tag=0x0004|mime_codec_string=mp4a.40.33|sample_fmt=fltp|sample_rate=16000|channels=1|channel_layout=mono|bits_per_sample=0|initial_padding=0|seek_preroll=0|id=0x1|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/90000|start_pts=0|start_time=0.000000|duration_ts=N/A|duration=N/A|bit_rate=160000|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=28|disposition:default=1|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:non_diegetic=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0|disposition:multilayer=0|tag:language=en|tag:comment=English
format|filename=hls-shared-renditions.m3u8|nb_streams=2|nb_programs=2|nb_stream_groups=0|format_name=hls|start_time=0.000000|duration=2.016000|size=365|bit_rate=1448|probe_score=100
diff --git a/tests/ref/fate/hls-webvtt-discont-demux b/tests/ref/fate/hls-webvtt-discont-demux
index 5ea461665c..f45e94adf7 100644
--- a/tests/ref/fate/hls-webvtt-discont-demux
+++ b/tests/ref/fate/hls-webvtt-discont-demux
@@ -1 +1,7 @@
WEBVTT
+
+00:10.500 --> 00:11.500
+hello
+
+00:10.500 --> 00:11.500
+hello
diff --git a/tests/ref/fate/hls-webvtt-map-demux b/tests/ref/fate/hls-webvtt-map-demux
index 5ea461665c..b82bd1435f 100644
--- a/tests/ref/fate/hls-webvtt-map-demux
+++ b/tests/ref/fate/hls-webvtt-map-demux
@@ -1 +1,19 @@
WEBVTT
+
+00:10.500 --> 00:11.500
+first
+
+00:11.000 --> 00:14.500 line:1
+long
+
+00:11.000 --> 00:14.500 line:1
+long
+
+00:13.000 --> 00:13.900 align:start
+second
+
+00:13.000 --> 00:16.500 line:1
+long
+
+00:14.500 --> 00:15.500
+third
diff --git a/tests/ref/fate/hls-webvtt-seek-demux b/tests/ref/fate/hls-webvtt-seek-demux
index 5ea461665c..b2742c8c66 100644
--- a/tests/ref/fate/hls-webvtt-seek-demux
+++ b/tests/ref/fate/hls-webvtt-seek-demux
@@ -1 +1,4 @@
WEBVTT
+
+00:14.500 --> 00:15.500
+third
diff --git a/tests/ref/fate/hls-webvtt-wrap-demux b/tests/ref/fate/hls-webvtt-wrap-demux
index 6cb6cf1f33..c65f428be5 100644
--- a/tests/ref/fate/hls-webvtt-wrap-demux
+++ b/tests/ref/fate/hls-webvtt-wrap-demux
@@ -1,8 +1,8 @@
-#tb 0: 1/1000
+#tb 0: 1/90000
#media_type 0: subtitle
#codec_id 0: webvtt
-0, 500, 500, 1000, 6, 0x08730273
-0, 1500, 1500, 1000, 6, 0x08a3028b
-0, 1500, 1500, 1000, 6, 0x08a3028b
-0, 3000, 3000, 900, 5, 0x06150212
-0, 4500, 4500, 1000, 4, 0x042d01b4
+0, -109572, -109572, 90000, 6, 0x08730273
+0, -19572, -19572, 90000, 6, 0x08a3028b
+0, -19572, -19572, 90000, 6, 0x08a3028b
+0, 115428, 115428, 81000, 5, 0x06150212
+0, 250428, 250428, 90000, 4, 0x042d01b4