commit 71943544885ff364a10bcc5ffc62d0e651c9a021
Author: Nikola Pajkovsky <nikolap@openssl.org>
Date: Mon May 18 09:48:59 2026 +0200
fix UB in priority_queue
clang-22 reports UB during test make test V=0 TESTS=test_priority_queue
ssl/priority_queue.c:159:13: runtime error: call to function size_t_compare through pointer to incorrect function type 'int (*)(const void *, const void *)'
/home/npajkovsky/openssl/openssl/test/priority_queue_test.c:28: note: size_t_compare defined here
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ssl/priority_queue.c:159:13
Fixes: https://github.com/openssl/project/issues/1953
Signed-off-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Norbert Pocs <norbertp@openssl.org>
MergeDate: Tue May 19 10:43:59 2026
(Merged from https://github.com/openssl/openssl/pull/31209)
diff --git a/include/internal/priority_queue.h b/include/internal/priority_queue.h
index 067c881596..9f4a636165 100644
--- a/include/internal/priority_queue.h
+++ b/include/internal/priority_queue.h
@@ -16,56 +16,56 @@
#define PRIORITY_QUEUE_OF(type) OSSL_PRIORITY_QUEUE_##type
-#define DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, ctype) \
- typedef struct ossl_priority_queue_st_##type PRIORITY_QUEUE_OF(type); \
- static ossl_unused ossl_inline PRIORITY_QUEUE_OF(type) * ossl_pqueue_##type##_new(int (*compare)(const ctype *, const ctype *)) \
- { \
- return (PRIORITY_QUEUE_OF(type) *)ossl_pqueue_new( \
- (int (*)(const void *, const void *))compare); \
- } \
- static ossl_unused ossl_inline void \
- ossl_pqueue_##type##_free(PRIORITY_QUEUE_OF(type) * pq) \
- { \
- ossl_pqueue_free((OSSL_PQUEUE *)pq); \
- } \
- static ossl_unused ossl_inline void \
- ossl_pqueue_##type##_pop_free(PRIORITY_QUEUE_OF(type) * pq, \
- void (*freefunc)(ctype *)) \
- { \
- ossl_pqueue_pop_free((OSSL_PQUEUE *)pq, (void (*)(void *))freefunc); \
- } \
- static ossl_unused ossl_inline int \
- ossl_pqueue_##type##_reserve(PRIORITY_QUEUE_OF(type) * pq, size_t n) \
- { \
- return ossl_pqueue_reserve((OSSL_PQUEUE *)pq, n); \
- } \
- static ossl_unused ossl_inline size_t \
- ossl_pqueue_##type##_num(const PRIORITY_QUEUE_OF(type) * pq) \
- { \
- return ossl_pqueue_num((OSSL_PQUEUE *)pq); \
- } \
- static ossl_unused ossl_inline int \
- ossl_pqueue_##type##_push(PRIORITY_QUEUE_OF(type) * pq, \
- ctype * data, size_t *elem) \
- { \
- return ossl_pqueue_push((OSSL_PQUEUE *)pq, (void *)data, elem); \
- } \
- static ossl_unused ossl_inline ctype * \
- ossl_pqueue_##type##_peek(const PRIORITY_QUEUE_OF(type) * pq) \
- { \
- return (type *)ossl_pqueue_peek((OSSL_PQUEUE *)pq); \
- } \
- static ossl_unused ossl_inline ctype * \
- ossl_pqueue_##type##_pop(PRIORITY_QUEUE_OF(type) * pq) \
- { \
- return (type *)ossl_pqueue_pop((OSSL_PQUEUE *)pq); \
- } \
- static ossl_unused ossl_inline ctype * \
- ossl_pqueue_##type##_remove(PRIORITY_QUEUE_OF(type) * pq, \
- size_t elem) \
- { \
- return (type *)ossl_pqueue_remove((OSSL_PQUEUE *)pq, elem); \
- } \
+#define DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, ctype) \
+ typedef struct ossl_priority_queue_st_##type PRIORITY_QUEUE_OF(type); \
+ static ossl_unused ossl_inline PRIORITY_QUEUE_OF(type) * ossl_pqueue_##type##_new(int (*compare)(const void *, const void *)) \
+ { \
+ return (PRIORITY_QUEUE_OF(type) *)ossl_pqueue_new( \
+ compare); \
+ } \
+ static ossl_unused ossl_inline void \
+ ossl_pqueue_##type##_free(PRIORITY_QUEUE_OF(type) * pq) \
+ { \
+ ossl_pqueue_free((OSSL_PQUEUE *)pq); \
+ } \
+ static ossl_unused ossl_inline void \
+ ossl_pqueue_##type##_pop_free(PRIORITY_QUEUE_OF(type) * pq, \
+ void (*freefunc)(void *)) \
+ { \
+ ossl_pqueue_pop_free((OSSL_PQUEUE *)pq, freefunc); \
+ } \
+ static ossl_unused ossl_inline int \
+ ossl_pqueue_##type##_reserve(PRIORITY_QUEUE_OF(type) * pq, size_t n) \
+ { \
+ return ossl_pqueue_reserve((OSSL_PQUEUE *)pq, n); \
+ } \
+ static ossl_unused ossl_inline size_t \
+ ossl_pqueue_##type##_num(const PRIORITY_QUEUE_OF(type) * pq) \
+ { \
+ return ossl_pqueue_num((OSSL_PQUEUE *)pq); \
+ } \
+ static ossl_unused ossl_inline int \
+ ossl_pqueue_##type##_push(PRIORITY_QUEUE_OF(type) * pq, \
+ ctype * data, size_t *elem) \
+ { \
+ return ossl_pqueue_push((OSSL_PQUEUE *)pq, (void *)data, elem); \
+ } \
+ static ossl_unused ossl_inline ctype * \
+ ossl_pqueue_##type##_peek(const PRIORITY_QUEUE_OF(type) * pq) \
+ { \
+ return (type *)ossl_pqueue_peek((OSSL_PQUEUE *)pq); \
+ } \
+ static ossl_unused ossl_inline ctype * \
+ ossl_pqueue_##type##_pop(PRIORITY_QUEUE_OF(type) * pq) \
+ { \
+ return (type *)ossl_pqueue_pop((OSSL_PQUEUE *)pq); \
+ } \
+ static ossl_unused ossl_inline ctype * \
+ ossl_pqueue_##type##_remove(PRIORITY_QUEUE_OF(type) * pq, \
+ size_t elem) \
+ { \
+ return (type *)ossl_pqueue_remove((OSSL_PQUEUE *)pq, elem); \
+ } \
struct ossl_priority_queue_st_##type
#define DEFINE_PRIORITY_QUEUE_OF(type) \
diff --git a/ssl/quic/quic_rcidm.c b/ssl/quic/quic_rcidm.c
index 0d5cb0337b..9a92ec6e35 100644
--- a/ssl/quic/quic_rcidm.c
+++ b/ssl/quic/quic_rcidm.c
@@ -270,8 +270,11 @@ static void rcidm_check_rcid(QUIC_RCIDM *rcidm, RCID *rcid)
assert(rcid->state != RCID_STATE_RETIRING || rcidm->num_retiring > 0);
}
-static int rcid_cmp(const RCID *a, const RCID *b)
+static int rcid_cmp(const void *av, const void *bv)
{
+ const RCID *a = av;
+ const RCID *b = bv;
+
if (a->seq_num < b->seq_num)
return -1;
if (a->seq_num > b->seq_num)
diff --git a/test/priority_queue_test.c b/test/priority_queue_test.c
index 8c09b50418..9cf8557594 100644
--- a/test/priority_queue_test.c
+++ b/test/priority_queue_test.c
@@ -24,8 +24,11 @@ DEFINE_PRIORITY_QUEUE_OF(size_t);
static size_t num_rec_freed;
-static int size_t_compare(const size_t *a, const size_t *b)
+static int size_t_compare(const void *av, const void *bv)
{
+ const size_t *a = av;
+ const size_t *b = bv;
+
if (*a < *b)
return -1;
if (*a > *b)
@@ -33,17 +36,12 @@ static int size_t_compare(const size_t *a, const size_t *b)
return 0;
}
-static int qsort_size_t_compare(const void *a, const void *b)
-{
- return size_t_compare((const size_t *)a, (const size_t *)b);
-}
-
static int qsort_size_t_compare_rev(const void *a, const void *b)
{
- return size_t_compare((const size_t *)b, (const size_t *)a);
+ return size_t_compare(b, a);
}
-static void free_checker(ossl_unused size_t *p)
+static void free_checker(ossl_unused void *p)
{
num_rec_freed++;
}
@@ -72,7 +70,7 @@ static int test_size_t_priority_queue_int(int reserve, int order, int count,
for (i = 0; i < count; i++)
values[i] = random ? test_random() : (size_t)(count - i);
memcpy(sorted, values, sizeof(*sorted) * count);
- qsort(sorted, count, sizeof(*sorted), &qsort_size_t_compare);
+ qsort(sorted, count, sizeof(*sorted), &size_t_compare);
if (order == 1)
memcpy(values, sorted, sizeof(*values) * count);
@@ -105,7 +103,7 @@ static int test_size_t_priority_queue_int(int reserve, int order, int count,
}
}
memcpy(sorted, values, sizeof(*sorted) * count);
- qsort(sorted, count, sizeof(*sorted), &qsort_size_t_compare);
+ qsort(sorted, count, sizeof(*sorted), &size_t_compare);
}
for (i = 0; ossl_pqueue_size_t_peek(pq) != NULL; i++)
if (!TEST_size_t_eq(*ossl_pqueue_size_t_peek(pq), sorted[i])
@@ -164,8 +162,11 @@ typedef struct info_st {
DEFINE_PRIORITY_QUEUE_OF(INFO);
-static int cmp(const INFO *a, const INFO *b)
+static int cmp(const void *av, const void *bv)
{
+ const INFO *a = av;
+ const INFO *b = bv;
+
if (a->seq_num < b->seq_num)
return -1;
if (a->seq_num > b->seq_num)
diff --git a/test/quic_cc_test.c b/test/quic_cc_test.c
index abe10780fc..b4ce5caa1b 100644
--- a/test/quic_cc_test.c
+++ b/test/quic_cc_test.c
@@ -86,8 +86,11 @@ typedef struct net_pkt_st {
DEFINE_PRIORITY_QUEUE_OF(NET_PKT);
-static int net_pkt_cmp(const NET_PKT *a, const NET_PKT *b)
+static int net_pkt_cmp(const void *av, const void *bv)
{
+ const NET_PKT *a = av;
+ const NET_PKT *b = bv;
+
return ossl_time_compare(a->next_time, b->next_time);
}
@@ -125,7 +128,7 @@ static int net_sim_init(struct net_sim *s,
return 1;
}
-static void do_free(NET_PKT *pkt)
+static void do_free(void *pkt)
{
OPENSSL_free(pkt);
}