Commit e8dbf3dd84 for asterisk.org
commit e8dbf3dd84d72c37c9b238d10bbb4ea3cb00e1a0
Author: phoneben <3232963@gmail.com>
Date: Sun Nov 15 23:26:49 2026 +0200
stasis_state: Don't hold the states container lock while running callbacks
stasis_state_callback_all() and stasis_state_callback_subscribed() used
ao2_callback_data(), which keeps the manager's states container locked
for the whole traversal. Handlers can be slow. With pollmailboxes=yes
and ODBC voicemail storage, app_voicemail runs several queries per
subscribed mailbox, so the lock was held for the full sweep. Any thread
that needed the container (subscribe, publish by id, state removal)
waited for the sweep to end.
A backtrace of one such wait shows a taskpool worker in
sorcery_observers_notify_create() -> mwi_contact_added() ->
mwi_contact_changed() -> create_unsolicited_mwi_subscriptions() ->
ast_mwi_subscribe_pool() -> stasis_state_add_subscriber(), blocked
until the sweep finished. mwi_contact_changed() holds the
unsolicited_mwi container lock at that point, so MWI setup for other
endpoints waits as well.
Use an ao2_iterator instead, so the container is only locked while
advancing. A state added during a traversal may not be visited until
the next one. Traversals are still serialized against each other, now by the
manager's own lock, because handlers may depend on that
(app_voicemail's IMAP polling and logout handlers must not overlap).
Nothing else takes that lock, so other threads are not blocked by it.
Measured on 20.22.0-rc2, ODBC voicemail, pollfreq=30, with one endpoint
registering and unregistering every 1.5 seconds for 10 minutes:
before: 18 sweeps of 5.0-9.5 s, 9 waits for the lock of 3.5-5.2 s
after: 18 sweeps of 5.0-9.5 s, no wait of 50 ms or more
AI assistance: Claude Fable 5.2
Resolves: #2018
diff --git a/include/asterisk/mwi.h b/include/asterisk/mwi.h
index 44439173f3..3eb368b180 100644
--- a/include/asterisk/mwi.h
+++ b/include/asterisk/mwi.h
@@ -306,6 +306,11 @@ typedef int (*on_mwi_state)(struct ast_mwi_state *mwi_state, void *data);
* \param handler The mwi state handler to call for each managed mailbox
* \param data User to data to pass on to the handler
*
+ * \note The handler runs without the mailbox states locked. A mailbox added
+ * while the traversal is running may not be passed to the handler.
+ * \note Only one traversal of the mailboxes runs at a time, whether started
+ * by ast_mwi_state_callback_all() or ast_mwi_state_callback_subscribed().
+ *
* \since 13.28.0
* \since 16.5.0
*/
@@ -317,6 +322,11 @@ void ast_mwi_state_callback_all(on_mwi_state handler, void *data);
* \param handler The mwi state handler to call for each managed mailbox
* \param data User to data to pass on to the handler
*
+ * \note The handler runs without the mailbox states locked. A mailbox added
+ * while the traversal is running may not be passed to the handler.
+ * \note Only one traversal of the mailboxes runs at a time, whether started
+ * by ast_mwi_state_callback_all() or ast_mwi_state_callback_subscribed().
+ *
* \since 13.28.0
* \since 16.5.0
*/
diff --git a/include/asterisk/stasis_state.h b/include/asterisk/stasis_state.h
index a1d36253ad..9a86414103 100644
--- a/include/asterisk/stasis_state.h
+++ b/include/asterisk/stasis_state.h
@@ -527,6 +527,16 @@ typedef int (*on_stasis_state)(const char *id, struct stasis_message *msg, void
* \param handler The handler to call for each managed state
* \param data User to data to pass on to the handler
*
+ * \note The states are not locked while the handler runs, so the handler may
+ * block or call other stasis_state functions without blocking other
+ * threads that use this manager.
+ * \note A state added while the traversal is running may not be passed to
+ * the handler. It will be seen by the next traversal.
+ * \note Only one traversal runs at a time for a given manager. This applies
+ * to both stasis_state_callback_all() and
+ * stasis_state_callback_subscribed(): a call to either waits until any
+ * traversal of the same manager that is already running has finished.
+ *
* \since 13.28.0
* \since 16.5.0
*/
@@ -540,6 +550,16 @@ void stasis_state_callback_all(struct stasis_state_manager *manager, on_stasis_s
* \param handler The handler to call for each managed state
* \param data User to data to pass on to the handler
*
+ * \note The states are not locked while the handler runs, so the handler may
+ * block or call other stasis_state functions without blocking other
+ * threads that use this manager.
+ * \note A state added while the traversal is running may not be passed to
+ * the handler. It will be seen by the next traversal.
+ * \note Only one traversal runs at a time for a given manager. This applies
+ * to both stasis_state_callback_all() and
+ * stasis_state_callback_subscribed(): a call to either waits until any
+ * traversal of the same manager that is already running has finished.
+ *
* \since 13.28.0
* \since 16.5.0
*/
diff --git a/main/stasis_state.c b/main/stasis_state.c
index decf228dbe..de4d01ca3e 100644
--- a/main/stasis_state.c
+++ b/main/stasis_state.c
@@ -326,8 +326,9 @@ struct stasis_state_manager *stasis_state_manager_create(const char *topic_name)
{
struct stasis_state_manager *manager;
+ /* The manager's own lock serializes the stasis_state_callback_*() traversals */
manager = ao2_alloc_options(sizeof(*manager), state_manager_dtor,
- AO2_ALLOC_OPT_LOCK_NOLOCK);
+ AO2_ALLOC_OPT_LOCK_MUTEX);
if (!manager) {
return NULL;
}
@@ -738,13 +739,55 @@ static int handle_stasis_state_proxy(void *obj, void *arg, void *data, int flags
return 0;
}
-void stasis_state_callback_all(struct stasis_state_manager *manager, on_stasis_state handler,
- void *data)
+/*!
+ * \internal
+ * \brief Run a callback on each state proxy without holding the container lock
+ *
+ * \note ao2_callback_data() is not used here on purpose. It keeps the states
+ * container locked for the whole traversal, and a handler may block for a
+ * long time (storage or network access, for example). Any thread that adds,
+ * finds or removes a state would then wait until the traversal ends. An
+ * iterator only locks the container while it advances.
+ *
+ * A state added while the traversal is running may not be visited.
+ *
+ * The manager's own lock is held instead, so only one traversal runs at a
+ * time per manager, as was the case when the container lock was held. Handlers
+ * may depend on that (app_voicemail's IMAP polling and logout handlers must
+ * not overlap). Nothing else takes the manager lock, so other threads are not
+ * blocked by it.
+ *
+ * \param manager The state manager
+ * \param cb Callback to run on each proxy, returns CMP_STOP to end the traversal
+ * \param handler The caller's handler, passed to cb
+ * \param data The caller's data, passed to cb
+ */
+static void state_callback(struct stasis_state_manager *manager, ao2_callback_data_fn *cb,
+ on_stasis_state handler, void *data)
{
+ struct ao2_iterator iter;
+ void *proxy;
+
ast_assert(handler != NULL);
- ao2_callback_data(manager->states, OBJ_MULTIPLE | OBJ_NODATA,
- handle_stasis_state_proxy, handler, data);
+ ao2_lock(manager);
+ iter = ao2_iterator_init(manager->states, 0);
+ while ((proxy = ao2_iterator_next(&iter))) {
+ int res = cb(proxy, handler, data, 0);
+
+ ao2_ref(proxy, -1);
+ if (res & CMP_STOP) {
+ break;
+ }
+ }
+ ao2_iterator_destroy(&iter);
+ ao2_unlock(manager);
+}
+
+void stasis_state_callback_all(struct stasis_state_manager *manager, on_stasis_state handler,
+ void *data)
+{
+ state_callback(manager, handle_stasis_state_proxy, handler, data);
}
static int handle_stasis_state_subscribed(void *obj, void *arg, void *data, int flags)
@@ -764,8 +807,5 @@ static int handle_stasis_state_subscribed(void *obj, void *arg, void *data, int
void stasis_state_callback_subscribed(struct stasis_state_manager *manager, on_stasis_state handler,
void *data)
{
- ast_assert(handler != NULL);
-
- ao2_callback_data(manager->states, OBJ_MULTIPLE | OBJ_NODATA,
- handle_stasis_state_subscribed, handler, data);
+ state_callback(manager, handle_stasis_state_subscribed, handler, data);
}