Commit 1da5eadf70 for freeswitch.com
commit 1da5eadf70ffc8aee88f1636fb9ad1fcd3d3bcd2
Author: Dmitry Verenitsin <morbit85@gmail.com>
Date: Thu Nov 19 01:12:22 2026 +0500
[mod_av, mod_openh264] Validate H.264 FU-A and H.263 packet lengths before depacketizing, add decoder unit tests (#3301)
diff --git a/src/mod/applications/mod_av/avcodec.c b/src/mod/applications/mod_av/avcodec.c
index 20f9abe719..3966122a32 100644
--- a/src/mod/applications/mod_av/avcodec.c
+++ b/src/mod/applications/mod_av/avcodec.c
@@ -552,10 +552,15 @@ static switch_status_t buffer_h264_nalu(h264_codec_context_t *context, switch_fr
{
uint8_t nalu_type = 0;
uint8_t *data = frame->data;
- uint8_t nalu_hdr = *data;
+ uint8_t nalu_hdr;
uint8_t sync_bytes[] = {0, 0, 0, 1};
switch_buffer_t *buffer = context->nalu_buffer;
+ if (!frame->datalen) { /* one byte NAL unit header */
+ return SWITCH_STATUS_RESTART;
+ }
+
+ nalu_hdr = *data;
nalu_type = nalu_hdr & 0x1f;
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "nalu=%02x mark=%d seq=%d ts=%d len=%d\n", nalu_hdr, frame->m, frame->seq, frame->timestamp, frame->datalen);
@@ -575,8 +580,14 @@ static switch_status_t buffer_h264_nalu(h264_codec_context_t *context, switch_fr
if ((nalu_type == 7 || nalu_type == 8) && frame->m) frame->m = SWITCH_FALSE;
if (nalu_type == 28) { // 0x1c FU-A
- int start = *(data + 1) & 0x80;
- int end = *(data + 1) & 0x40;
+ int start, end;
+
+ if (frame->datalen < 2) { /* FU indicator + FU header, the payload may be empty */
+ return SWITCH_STATUS_RESTART;
+ }
+
+ start = *(data + 1) & 0x80;
+ end = *(data + 1) & 0x40;
nalu_type = *(data + 1) & 0x1f;
@@ -588,6 +599,11 @@ static switch_status_t buffer_h264_nalu(h264_codec_context_t *context, switch_fr
switch_buffer_zero(buffer);
}
} else if (end) {
+ if (!context->nalu_28_start) {
+ /* the start fragment was lost, this tail is undecodable */
+ return SWITCH_STATUS_RESTART;
+ }
+
context->nalu_28_start = 0;
} else if (!context->nalu_28_start) {
return SWITCH_STATUS_RESTART;
@@ -662,6 +678,11 @@ static switch_status_t buffer_h263_packets(h264_codec_context_t *context, switch
h263_payload_header_t *h = (h263_payload_header_t *)data;
int delta = 0;
+ if (frame->datalen < 4) { /* Mode A header */
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "INVALID PACKET\n");
+ return SWITCH_STATUS_FALSE;
+ }
+
if (h->f) {
if (h->p) {
header_len = 12; // Mode C
@@ -741,7 +762,7 @@ static switch_status_t buffer_h263_packets(h264_codec_context_t *context, switch
static switch_status_t buffer_h263_rfc4629_packets(h264_codec_context_t *context, switch_frame_t *frame)
{
uint8_t *data = frame->data;
- uint16_t header = ntohs(*((uint16_t *)frame->data));
+ uint16_t header;
int startcode, vrc, picture_header;
int len = frame->datalen;
@@ -749,6 +770,8 @@ static switch_status_t buffer_h263_rfc4629_packets(h264_codec_context_t *context
return SWITCH_STATUS_FALSE;
}
+ header = ntohs(*((uint16_t *)frame->data));
+
startcode = (header & 0x0400) >> 9;
vrc = header & 0x0200;
picture_header = (header & 0x01f8) >> 3;
@@ -1713,6 +1736,10 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %u mark:%d\n", frame->datalen, ntohl(frame->timestamp), frame->m);
+ if (!frame->data) {
+ return SWITCH_STATUS_FALSE;
+ }
+
//if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp &&
// (!frame->m) && (!context->last_received_complete_picture)) {
// possible packet loss
@@ -1723,19 +1750,23 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
context->last_received_timestamp = frame->timestamp;
context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE;
- if (context->av_codec_id == AV_CODEC_ID_H263) {
- status = buffer_h263_packets(context, frame);
- } else if (context->av_codec_id == AV_CODEC_ID_H263P) {
- status = buffer_h263_rfc4629_packets(context, frame);
- } else {
- status = buffer_h264_nalu(context, frame);
- }
+ /* an empty packet adds nothing, but its marker bit still closes the picture below */
+ if (frame->datalen > 0) {
+ if (context->av_codec_id == AV_CODEC_ID_H263) {
+ status = buffer_h263_packets(context, frame);
+ } else if (context->av_codec_id == AV_CODEC_ID_H263P) {
+ status = buffer_h263_rfc4629_packets(context, frame);
+ } else {
+ status = buffer_h264_nalu(context, frame);
+ }
- if (status == SWITCH_STATUS_RESTART) {
- switch_set_flag(frame, SFF_WAIT_KEY_FRAME);
- switch_buffer_zero(context->nalu_buffer);
- context->nalu_28_start = 0;
- return SWITCH_STATUS_MORE_DATA;
+ if (status == SWITCH_STATUS_RESTART) {
+ switch_set_flag(frame, SFF_WAIT_KEY_FRAME);
+ switch_buffer_zero(context->nalu_buffer);
+ context->nalu_28_start = 0;
+
+ return SWITCH_STATUS_MORE_DATA;
+ }
}
if (frame->m) {
diff --git a/src/mod/applications/mod_av/test/test_mod_av.c b/src/mod/applications/mod_av/test/test_mod_av.c
index 69468dbda2..502f5ccfd9 100644
--- a/src/mod/applications/mod_av/test/test_mod_av.c
+++ b/src/mod/applications/mod_av/test/test_mod_av.c
@@ -48,6 +48,29 @@ static fctcl_init_t my_cl_options[] = {
FCTCL_INIT_NULL /* Sentinel */
};
+/* payload bytes per FU-A fragment when the round-trip test re-fragments a NAL unit */
+#define FU_A_TEST_PAYLOAD 32
+
+/* Feed one RTP payload to the decoder the way the media layer does: a fresh frame per packet,
+ * the marker on the last packet of a picture and the timestamp advancing per picture. */
+static switch_status_t decode_rtp_payload(switch_codec_t *codec, switch_frame_t *frame, void *data, uint32_t datalen, switch_bool_t m)
+{
+ switch_status_t status;
+
+ frame->data = data;
+ frame->datalen = datalen;
+ frame->m = m;
+ frame->flags = 0;
+ frame->img = NULL;
+ frame->seq++;
+
+ status = switch_core_codec_decode_video(codec, frame);
+
+ if (m) frame->timestamp += 3000;
+
+ return status;
+}
+
FST_CORE_BEGIN("conf")
{
const char *loop_;
@@ -153,11 +176,269 @@ FST_CORE_BEGIN("conf")
}
FST_TEST_END()
+ /* RTP payloads shorter than the headers their NAL type implies are rejected by the depacketizer,
+ * while FU-A fragments with an empty payload, which RFC 6184 allows, are accepted */
+ FST_TEST_BEGIN(decoder_short_packet_test)
+ {
+ switch_status_t status;
+ switch_codec_t codec = { 0 };
+ switch_codec_settings_t codec_settings = {{ 0 }};
+ switch_frame_t frame = { 0 };
+ /* each packet buffer is exactly the length it claims */
+ uint8_t fu_a_indicator_only[] = { 0x7c };
+ uint8_t fu_a_start_empty[] = { 0x7c, 0x85 };
+ uint8_t fu_a_middle_empty[] = { 0x7c, 0x05 };
+ uint8_t fu_a_end_empty[] = { 0x7c, 0x45 };
+ uint8_t fu_a_start[] = { 0x7c, 0x85, 0x01, 0x02 };
+ uint8_t fu_a_end[] = { 0x7c, 0x45, 0x03, 0x04 };
+ uint8_t stap_a_indicator_only[] = { 0x78 };
+ uint8_t stap_a_half_size[] = { 0x78, 0x00 };
+ uint8_t single_nalu_header_only[] = { 0x65 };
+ int i;
+
+ switch_set_string(codec_settings.video.config_profile_name, "conference");
+
+ status = switch_core_codec_init(&codec,
+ "H264",
+ NULL,
+ NULL,
+ 0,
+ 0,
+ 1, SWITCH_CODEC_FLAG_DECODE,
+ &codec_settings, fst_pool);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+
+ frame.payload = 96;
+ frame.timestamp = 3000;
+
+ /* an empty payload adds nothing */
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, 0, SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+
+ /* an FU-A without the FU header is dropped and a key frame requested, an FU-A with an empty payload is accepted */
+ for (i = 0; i < 2; i++) {
+ switch_bool_t m = i ? SWITCH_TRUE : SWITCH_FALSE;
+
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, sizeof(fu_a_indicator_only), m);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+
+ status = decode_rtp_payload(&codec, &frame, fu_a_start_empty, sizeof(fu_a_start_empty), m);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+ }
+
+ /* the same packets in the middle of a fragment being reassembled, then empty middle and end fragments completing it */
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, sizeof(fu_a_indicator_only), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, fu_a_middle_empty, sizeof(fu_a_middle_empty), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, fu_a_end_empty, sizeof(fu_a_end_empty), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+
+ /* an end fragment whose start fragment was lost is dropped */
+ status = decode_rtp_payload(&codec, &frame, fu_a_end, sizeof(fu_a_end), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+
+ /* an empty packet carrying the marker closes the picture: the fragment started before it is
+ * handed to the decoder, so a later end fragment has no start to attach to */
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, 0, SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+ status = decode_rtp_payload(&codec, &frame, fu_a_end, sizeof(fu_a_end), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+
+ /* a STAP-A with no room for a NALU size, and a single NALU of only a header, are accepted */
+ status = decode_rtp_payload(&codec, &frame, stap_a_indicator_only, sizeof(stap_a_indicator_only), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, stap_a_half_size, sizeof(stap_a_half_size), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, single_nalu_header_only, sizeof(single_nalu_header_only), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+
+ switch_core_codec_destroy(&codec);
+ }
+ FST_TEST_END()
+
+ /* Encode a few pictures, re-fragment every NAL unit into FU-A packets and make sure the
+ * depacketizer reassembles them into pictures */
+ FST_TEST_BEGIN(decoder_fu_a_roundtrip_test)
+ {
+ switch_status_t status;
+ switch_codec_t codec = { 0 };
+ switch_codec_settings_t codec_settings = {{ 0 }};
+ switch_image_t *img;
+ uint8_t buf[SWITCH_DEFAULT_VIDEO_SIZE + 12];
+ uint8_t fragment[FU_A_TEST_PAYLOAD + 2];
+ switch_frame_t enc_frame = { 0 };
+ switch_frame_t dec_frame = { 0 };
+ int pictures;
+ int encoded = 0;
+ int split_fragments = 0;
+ int decoded = 0;
+
+ switch_set_string(codec_settings.video.config_profile_name, "conference");
+
+ if (!fctcl_is("--disable-hw")) {
+ codec_settings.video.try_hardware_encoder = 1;
+ }
+
+ status = switch_core_codec_init(&codec,
+ "H264",
+ NULL,
+ NULL,
+ 0,
+ 0,
+ 1, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE,
+ &codec_settings, fst_pool);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+
+ img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, 320, 240, 1);
+ fst_check(img != NULL);
+
+ enc_frame.packet = buf;
+ enc_frame.packetlen = sizeof(buf);
+ enc_frame.data = buf + 12;
+ enc_frame.payload = 96;
+ enc_frame.img = img;
+
+ dec_frame.payload = 96;
+ dec_frame.timestamp = 3000;
+
+ for (pictures = 0; pictures < 10; pictures++) {
+ switch_status_t encode_status;
+
+ do {
+ uint8_t *nalu = buf + 12;
+ uint8_t nalu_type;
+
+ enc_frame.datalen = SWITCH_DEFAULT_VIDEO_SIZE;
+ encode_status = switch_core_codec_encode_video(&codec, &enc_frame);
+
+ if (encode_status != SWITCH_STATUS_SUCCESS && encode_status != SWITCH_STATUS_MORE_DATA) break;
+ if (enc_frame.datalen == 0) break;
+ if (enc_frame.m) encoded++;
+
+ nalu_type = nalu[0] & 0x1f;
+
+ if (nalu_type >= 1 && nalu_type <= 23 && enc_frame.datalen > FU_A_TEST_PAYLOAD) {
+ /* a single NAL unit packet: split it into FU-A fragments as RFC 6184 section 5.8 describes */
+ uint32_t offset = 1; /* the NAL header travels in the FU indicator and FU header instead */
+
+ while (offset < enc_frame.datalen) {
+ uint32_t chunk = enc_frame.datalen - offset;
+ switch_bool_t last;
+
+ if (chunk > FU_A_TEST_PAYLOAD) chunk = FU_A_TEST_PAYLOAD;
+ last = (offset + chunk == enc_frame.datalen) ? SWITCH_TRUE : SWITCH_FALSE;
+
+ fragment[0] = (nalu[0] & 0xe0) | 28;
+ fragment[1] = (nalu[0] & 0x1f) | (offset == 1 ? 0x80 : 0) | (last ? 0x40 : 0);
+ memcpy(fragment + 2, nalu + offset, chunk);
+ offset += chunk;
+
+ status = decode_rtp_payload(&codec, &dec_frame, fragment, chunk + 2, (last && enc_frame.m) ? SWITCH_TRUE : SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_SUCCESS || status == SWITCH_STATUS_MORE_DATA);
+ split_fragments++;
+
+ if (dec_frame.img) {
+ fst_check(dec_frame.img->d_w == 320 && dec_frame.img->d_h == 240);
+ decoded++;
+ }
+ }
+ } else {
+ status = decode_rtp_payload(&codec, &dec_frame, nalu, enc_frame.datalen, enc_frame.m);
+ fst_check(status == SWITCH_STATUS_SUCCESS || status == SWITCH_STATUS_MORE_DATA);
+
+ if (dec_frame.img) {
+ fst_check(dec_frame.img->d_w == 320 && dec_frame.img->d_h == 240);
+ decoded++;
+ }
+ }
+ } while (encode_status == SWITCH_STATUS_MORE_DATA);
+ }
+
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "%d pictures encoded, %d FU-A fragments made by the test, %d pictures decoded\n", encoded, split_fragments, decoded);
+ fst_check(encoded > 0);
+ fst_check(split_fragments > 0);
+ fst_check(decoded > 0);
+ fst_check(decoded + 2 >= encoded);
+
+ switch_core_codec_destroy(&codec);
+ switch_img_free(&img);
+ }
+ FST_TEST_END()
+
+ /* H.263 payload headers are checked against the packet length before they are read. The decoder
+ * reports success for a packet its depacketizer rejected, so these only make sure nothing is
+ * read past a short packet and no picture comes out of it. */
+ FST_TEST_BEGIN(decoder_h263_short_packet_test)
+ {
+ const char *names[] = { "H263", "H263-1998" };
+ int c;
+
+ for (c = 0; c < 2; c++) {
+ switch_status_t status;
+ switch_codec_t codec = { 0 };
+ switch_codec_settings_t codec_settings = {{ 0 }};
+ switch_frame_t frame = { 0 };
+ /* each packet buffer is exactly the length it claims */
+ uint8_t one[] = { 0x00 };
+ uint8_t two[] = { 0x00, 0x00 };
+ uint8_t three[] = { 0x00, 0x00, 0x00 };
+ int i;
+
+ switch_set_string(codec_settings.video.config_profile_name, "conference");
+
+ status = switch_core_codec_init(&codec,
+ names[c],
+ NULL,
+ NULL,
+ 0,
+ 0,
+ 1, SWITCH_CODEC_FLAG_DECODE,
+ &codec_settings, fst_pool);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+
+ frame.payload = 96;
+ frame.timestamp = 3000;
+
+ for (i = 0; i < 2; i++) {
+ switch_bool_t m = i ? SWITCH_TRUE : SWITCH_FALSE;
+
+ status = decode_rtp_payload(&codec, &frame, one, 0, m);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, one, sizeof(one), m);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, two, sizeof(two), m);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ status = decode_rtp_payload(&codec, &frame, three, sizeof(three), m);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+ fst_check(frame.img == NULL);
+ }
+
+ switch_core_codec_destroy(&codec);
+ }
+ }
+ FST_TEST_END()
+
FST_TEARDOWN_BEGIN()
{
- const char *err = NULL;
+ /* the module stays loaded across the tests, FST_MODULE_END unloads it after the last one */
switch_sleep(1000000);
- fst_check(switch_loadable_module_unload_module(SWITCH_GLOBAL_dirs.mod_dir, (char *)"mod_av", SWITCH_TRUE, &err) == SWITCH_STATUS_SUCCESS);
}
FST_TEARDOWN_END()
}
diff --git a/src/mod/codecs/mod_openh264/mod_openh264.cpp b/src/mod/codecs/mod_openh264/mod_openh264.cpp
index be92f63b7f..8d9c225ac6 100644
--- a/src/mod/codecs/mod_openh264/mod_openh264.cpp
+++ b/src/mod/codecs/mod_openh264/mod_openh264.cpp
@@ -186,13 +186,18 @@ static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_fram
uint8_t nalu_idc = 0;
uint8_t nalu_type = 0;
uint8_t *data = (uint8_t *)frame->data;
- uint8_t nalu_hdr = *data;
+ uint8_t nalu_hdr;
uint8_t sync_bytes[] = {0, 0, 0, 1};
switch_buffer_t *buffer = context->nalu_buffer;
switch_size_t size = switch_buffer_inuse(buffer);
switch_assert(frame);
+ if (!frame->datalen) { /* one byte NAL unit header */
+ return 1;
+ }
+
+ nalu_hdr = *data;
nalu_idc = (nalu_hdr & 0x60) >> 5;
nalu_type = nalu_hdr & 0x1f;
@@ -219,8 +224,14 @@ static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_fram
if ((nalu_type == 7 || nalu_type == 8) && frame->m) frame->m = SWITCH_FALSE;
if (nalu_type == 28) { // 0x1c FU-A
- int start = *(data + 1) & 0x80;
- int end = *(data + 1) & 0x40;
+ int start, end;
+
+ if (frame->datalen < 2) { /* FU indicator + FU header, the payload may be empty */
+ return 1;
+ }
+
+ start = *(data + 1) & 0x80;
+ end = *(data + 1) & 0x40;
nalu_type = *(data + 1) & 0x1f;
@@ -236,6 +247,11 @@ static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_fram
switch_buffer_zero(buffer);
}
} else if (end) {
+ if (!context->nalu_28_start) {
+ /* the start fragment was lost, this tail is undecodable */
+ return 1;
+ }
+
context->nalu_28_start = 0;
} else if (!context->nalu_28_start) {
return 0;
@@ -612,6 +628,10 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
uint32_t error_code;
switch_status_t status = SWITCH_STATUS_SUCCESS;
+ if (!frame->data) {
+ return SWITCH_STATUS_FALSE;
+ }
+
#ifdef DEBUG_H264
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d seq: %u ts: %u mark:%d nalu:0x%x\n", frame->datalen, frame->seq, frame->timestamp, frame->m, *(uint8_t *)frame->data);
#endif
@@ -629,7 +649,8 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
context->last_received_timestamp = frame->timestamp;
context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE;
- size = buffer_h264_nalu(context, frame);
+ /* an empty packet adds nothing, but its marker bit still closes the picture below */
+ size = frame->datalen > 0 ? buffer_h264_nalu(context, frame) : switch_buffer_inuse(context->nalu_buffer);
#ifdef DEBUG_H264
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "READ buf:%ld got_key:%d st:%d m:%d size:%" SWITCH_SIZE_T_FMT "\n", size, context->got_sps, status, frame->m, size);
diff --git a/src/mod/codecs/mod_openh264/test/test_mod_openh264.cpp b/src/mod/codecs/mod_openh264/test/test_mod_openh264.cpp
index 729f77be84..03aea99ed0 100644
--- a/src/mod/codecs/mod_openh264/test/test_mod_openh264.cpp
+++ b/src/mod/codecs/mod_openh264/test/test_mod_openh264.cpp
@@ -43,6 +43,26 @@ static fctcl_init_t my_cl_options[] = {
FCTCL_INIT_NULL /* Sentinel */
};
+/* Feed one RTP payload to the decoder the way the media layer does: a fresh frame per packet,
+ * the marker on the last packet of a picture and the timestamp advancing per picture. */
+static switch_status_t decode_rtp_payload(switch_codec_t *codec, switch_frame_t *frame, void *data, uint32_t datalen, switch_bool_t m)
+{
+ switch_status_t status;
+
+ frame->data = data;
+ frame->datalen = datalen;
+ frame->m = m;
+ frame->flags = 0;
+ frame->img = NULL;
+ frame->seq++;
+
+ status = switch_core_codec_decode_video(codec, frame);
+
+ if (m) frame->timestamp += 3000;
+
+ return status;
+}
+
FST_CORE_BEGIN("conf")
{
fctcl_install(my_cl_options);
@@ -207,11 +227,113 @@ FST_CORE_BEGIN("conf")
FST_TEST_END()
+ /* RTP payloads shorter than the headers their NAL type implies are rejected by the depacketizer,
+ * while FU-A fragments with an empty payload, which RFC 6184 allows, are accepted. Without a
+ * picture to return the decoder reports SWITCH_STATUS_MORE_DATA, so a dropped packet shows as
+ * the key frame request it raises. */
+ FST_TEST_BEGIN(decoder_short_packet_test)
+ {
+ switch_status_t status;
+ switch_codec_t codec = { 0 };
+ switch_codec_settings_t codec_settings = {{ 0 }};
+ switch_frame_t frame = { 0 };
+ /* each packet buffer is exactly the length it claims */
+ uint8_t fu_a_indicator_only[] = { 0x7c };
+ uint8_t fu_a_start_empty[] = { 0x7c, 0x85 };
+ uint8_t fu_a_middle_empty[] = { 0x7c, 0x05 };
+ uint8_t fu_a_end_empty[] = { 0x7c, 0x45 };
+ uint8_t fu_a_start[] = { 0x7c, 0x85, 0x01, 0x02 };
+ uint8_t fu_a_end[] = { 0x7c, 0x45, 0x03, 0x04 };
+ uint8_t stap_a_indicator_only[] = { 0x78 };
+ uint8_t stap_a_half_size[] = { 0x78, 0x00 };
+ uint8_t single_nalu_header_only[] = { 0x65 };
+
+ switch_set_string(codec_settings.video.config_profile_name, "test-decoder-1");
+
+ status = switch_core_codec_init(&codec,
+ "H264",
+ NULL,
+ NULL,
+ 0,
+ 0,
+ 1, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE,
+ &codec_settings, fst_pool);
+ fst_check(status == SWITCH_STATUS_SUCCESS);
+
+ frame.payload = 96;
+ frame.timestamp = 3000;
+
+ /* an empty payload adds nothing */
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, 0, SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(frame.img == NULL);
+
+ /* an FU-A without the FU header is dropped and a key frame requested */
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, sizeof(fu_a_indicator_only), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, sizeof(fu_a_indicator_only), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+
+ /* an FU-A start fragment with an empty payload is accepted */
+ status = decode_rtp_payload(&codec, &frame, fu_a_start_empty, sizeof(fu_a_start_empty), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(!switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+
+ /* a fragment without the FU header in the middle of a reassembly is dropped, then empty middle and end fragments complete one */
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, sizeof(fu_a_indicator_only), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(!switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ status = decode_rtp_payload(&codec, &frame, fu_a_middle_empty, sizeof(fu_a_middle_empty), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(!switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ status = decode_rtp_payload(&codec, &frame, fu_a_end_empty, sizeof(fu_a_end_empty), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(frame.img == NULL);
+
+ /* an end fragment whose start fragment was lost is dropped */
+ status = decode_rtp_payload(&codec, &frame, fu_a_end, sizeof(fu_a_end), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+ fst_check(frame.img == NULL);
+
+ /* an empty packet carrying the marker closes the picture: the fragment started before it is
+ * handed to the decoder, so a later end fragment has no start to attach to */
+ status = decode_rtp_payload(&codec, &frame, fu_a_start, sizeof(fu_a_start), SWITCH_FALSE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ status = decode_rtp_payload(&codec, &frame, fu_a_indicator_only, 0, SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(frame.img == NULL);
+ status = decode_rtp_payload(&codec, &frame, fu_a_end, sizeof(fu_a_end), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(switch_test_flag(&frame, SFF_WAIT_KEY_FRAME));
+
+ /* a STAP-A with no room for a NALU size, and a single NALU of only a header, are accepted */
+ status = decode_rtp_payload(&codec, &frame, stap_a_indicator_only, sizeof(stap_a_indicator_only), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ status = decode_rtp_payload(&codec, &frame, stap_a_half_size, sizeof(stap_a_half_size), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ status = decode_rtp_payload(&codec, &frame, single_nalu_header_only, sizeof(single_nalu_header_only), SWITCH_TRUE);
+ fst_check(status == SWITCH_STATUS_MORE_DATA);
+ fst_check(frame.img == NULL);
+
+ switch_core_codec_destroy(&codec);
+ }
+ FST_TEST_END()
+
FST_TEARDOWN_BEGIN()
{
- const char *err = NULL;
+ /* the module stays loaded across the tests, FST_MODULE_END unloads it after the last one */
switch_sleep(1000000);
- fst_check(switch_loadable_module_unload_module(SWITCH_GLOBAL_dirs.mod_dir, (char *)"mod_openh264", SWITCH_TRUE, &err) == SWITCH_STATUS_SUCCESS);
}
FST_TEARDOWN_END()
}