Commit 78783f19 for openh264
commit 78783f195ce116ecbaea83f7fac0ea8a355518b5
Author: Erik Språng <sprang@google.com>
Date: Fri Nov 20 06:21:29 2026 +0100
Synchronize task completion with CWelsTaskManageBase teardown (#4046)
When a worker thread finishes the last task in a batch, OnTaskMinusOne()
signals m_hTaskEvent via WelsEventSignal(), waking the encoder thread
waiting in ExecuteTaskList(). Previously, the worker unlocked
m_cWaitTaskNumLock (owned by CWelsTaskManageBase) after signaling the
event, and CWelsThreadPool::RemoveInstance() did not wait for in-flight
OnTaskExecuted() callbacks when other instances kept the shared thread
pool alive. If the encoder tore down the task manager immediately after
ExecuteTaskList() returned, the worker could access task manager members
that had already been destroyed.
To fix this:
- Hold CWelsThreadPool::m_cLockBusyTasks across OnTaskExecuted() in
OnTaskStop(), and synchronize on m_cLockBusyTasks in RemoveInstance()
(and clear m_pThreadPool) before CWelsTaskManageBase::Uninit() destroys
its tasks and event primitives.
- Protect the condition variable in WelsEventSignal() (and CWelsThread)
with pMutex on all platforms, unlock pMutex before sem_post / SetEvent,
and remove the now-redundant m_cWaitTaskNumLock.
Co-authored-by: Erik Språng <sprang@webrtc.org>
diff --git a/codec/common/inc/WelsThreadPool.h b/codec/common/inc/WelsThreadPool.h
index a24309aa..8ae6cc39 100644
--- a/codec/common/inc/WelsThreadPool.h
+++ b/codec/common/inc/WelsThreadPool.h
@@ -59,6 +59,9 @@ class CWelsThreadPool : public CWelsThread, public IWelsTaskThreadSink {
static WELS_THREAD_ERROR_CODE SetThreadNum (int32_t iMaxThreadNum);
static CWelsThreadPool* AddReference();
+ // After RemoveInstance() returns, the pool will not call into any sink or
+ // touch any task that was queued by the caller. Sinks must therefore not
+ // call AddReference()/RemoveInstance()/IsReferenced() from within callbacks.
void RemoveInstance();
static bool IsReferenced();
@@ -84,7 +87,6 @@ class CWelsThreadPool : public CWelsThread, public IWelsTaskThreadSink {
void DestroyThread (CWelsTaskThread* pThread);
WELS_THREAD_ERROR_CODE AddThreadToIdleQueue (CWelsTaskThread* pThread);
WELS_THREAD_ERROR_CODE AddThreadToBusyList (CWelsTaskThread* pThread);
- WELS_THREAD_ERROR_CODE RemoveThreadFromBusyList (CWelsTaskThread* pThread);
bool AddTaskToWaitedList (IWelsTask* pTask);
CWelsTaskThread* GetIdleThread();
IWelsTask* GetWaitedTask();
@@ -110,7 +112,7 @@ class CWelsThreadPool : public CWelsThread, public IWelsTaskThreadSink {
CWelsLock m_cLockPool;
CWelsLock m_cLockWaitedTasks;
CWelsLock m_cLockIdleTasks;
- CWelsLock m_cLockBusyTasks;
+ CWelsLock m_cLockBusyTasks; // guards m_cBusyThreads and serializes sink callbacks
DISALLOW_COPY_AND_ASSIGN (CWelsThreadPool);
};
diff --git a/codec/common/src/WelsThread.cpp b/codec/common/src/WelsThread.cpp
index 37ad319c..19dcfa4c 100644
--- a/codec/common/src/WelsThread.cpp
+++ b/codec/common/src/WelsThread.cpp
@@ -66,7 +66,9 @@ void CWelsThread::Thread() {
break;
}
+ WelsMutexLock (&m_hMutex);
m_iConVar = 1;
+ WelsMutexUnlock (&m_hMutex);
ExecuteTask();//in ExecuteTask there will be OnTaskStop which opens the potential new Signaling of next run, so the setting of m_iConVar = 1 should be before ExecuteTask()
}
diff --git a/codec/common/src/WelsThreadLib.cpp b/codec/common/src/WelsThreadLib.cpp
index 4e44f248..a4669edf 100644
--- a/codec/common/src/WelsThreadLib.cpp
+++ b/codec/common/src/WelsThreadLib.cpp
@@ -131,9 +131,14 @@ WELS_THREAD_ERROR_CODE WelsEventOpen (WELS_EVENT* event, const char* event_na
}
WELS_THREAD_ERROR_CODE WelsEventSignal (WELS_EVENT* event, WELS_MUTEX *pMutex, int* iCondition) {
- (*iCondition) --;
- if ((*iCondition) <= 0) {
- if (SetEvent (*event)) {
+ WelsMutexLock (pMutex);
+ const bool bSignal = (-- (*iCondition) <= 0);
+ // Unlock pMutex before signaling hEvent, as the woken thread may immediately
+ // destroy the object that owns pMutex, event, and iCondition.
+ WELS_EVENT hEvent = *event;
+ WelsMutexUnlock (pMutex);
+ if (bSignal && hEvent != NULL) {
+ if (SetEvent (hEvent)) {
return WELS_THREAD_ERROR_OK;
}
}
@@ -312,26 +317,29 @@ void WelsSleep (uint32_t dwMilliSecond) {
WELS_THREAD_ERROR_CODE WelsEventSignal (WELS_EVENT* event, WELS_MUTEX *pMutex, int* iCondition) {
WELS_THREAD_ERROR_CODE err = 0;
//fprintf( stderr, "before signal it, event=%x iCondition= %d..\n", event, *iCondition );
-#ifdef __APPLE__
WelsMutexLock (pMutex);
- (*iCondition) --;
- WelsMutexUnlock (pMutex);
- if ((*iCondition) <= 0) {
- err = pthread_cond_signal (event);
- //fprintf( stderr, "signal it, event=%x iCondition= %d..\n",event, *iCondition );
-
+ const bool bSignal = (-- (*iCondition) <= 0);
+#ifdef __APPLE__
+ // On macOS, WELS_EVENT is a pthread_cond_t and WelsEventWait re-acquires pMutex
+ // before returning, so keep pMutex locked across the condition update and signal.
+ if (bSignal) {
+ err = pthread_cond_signal (event);
+ //fprintf( stderr, "signal it, event=%x iCondition= %d..\n",event, *iCondition );
}
+ WelsMutexUnlock (pMutex);
#else
- (*iCondition) --;
- if ((*iCondition) <= 0) {
+ // Unlock pMutex before sem_post, as the woken thread waiting in sem_wait
+ // does not acquire pMutex and may immediately destroy the owner of pMutex.
+ WELS_EVENT hEvent = *event;
+ WelsMutexUnlock (pMutex);
+ if (bSignal && hEvent != NULL) {
// int32_t val = 0;
// sem_getvalue(event, &val);
// fprintf( stderr, "before signal it, val= %d..\n",val );
- if (event != NULL)
- err = sem_post (*event);
+ err = sem_post (hEvent);
// sem_getvalue(event, &val);
//fprintf( stderr, "signal it, event=%x iCondition= %d..\n",event, *iCondition );
- }
+ }
#endif
//fprintf( stderr, "after signal it, event=%x iCondition= %d..\n",event, *iCondition );
return err;
diff --git a/codec/common/src/WelsThreadPool.cpp b/codec/common/src/WelsThreadPool.cpp
index 2b140019..dfa9d385 100644
--- a/codec/common/src/WelsThreadPool.cpp
+++ b/codec/common/src/WelsThreadPool.cpp
@@ -121,6 +121,12 @@ void CWelsThreadPool::RemoveInstance() {
m_pThreadPoolSelf = NULL;
}
//fprintf(stdout, "m_iRefCount=%d, IdleThreadNum=%d, BusyThreadNum=%d, WaitedTask=%d\n", m_iRefCount, GetIdleThreadNum(), GetBusyThreadNum(), GetWaitedTaskNum());
+ } else {
+ // Synchronize with OnTaskStop(): if a worker thread just signaled the last task
+ // of the caller being torn down, it still holds m_cLockBusyTasks until
+ // OnTaskExecuted() returns. Acquiring the lock here ensures the worker has
+ // exited OnTaskExecuted() before the caller destroys the task sink and tasks.
+ CWelsAutoLock cBusyLock (m_cLockBusyTasks);
}
}
@@ -140,13 +146,23 @@ WELS_THREAD_ERROR_CODE CWelsThreadPool::OnTaskStart (CWelsTaskThread* pThread, I
WELS_THREAD_ERROR_CODE CWelsThreadPool::OnTaskStop (CWelsTaskThread* pThread, IWelsTask* pTask) {
//fprintf(stdout, "CWelsThreadPool::OnTaskStop 0: Task %x at Thread %x Finished\n", pTask, pThread);
- RemoveThreadFromBusyList (pThread);
+ // Return pThread to the idle queue before removing it from m_cBusyThreads so
+ // StopAllRunning() sees it in m_cIdleThreads as soon as GetBusyThreadNum() == 0.
+ // The pool thread cannot re-dispatch pThread until CWelsTaskThread::ExecuteTask()
+ // releases m_cLockTask after OnTaskStop() returns.
AddThreadToIdleQueue (pThread);
- if (pTask && pTask->GetSink()) {
- //fprintf(stdout, "CWelsThreadPool::OnTaskStop 1: Task %x at Thread %x Finished, m_pSink=%x\n", pTask, pThread, pTask->GetSink());
- pTask->GetSink()->OnTaskExecuted();
- ////fprintf(stdout, "CWelsThreadPool::OnTaskStop 1: Task %x at Thread %x Finished, m_pSink=%x\n", pTask, pThread, pTask->GetSink());
+ {
+ // Hold the pool-owned m_cLockBusyTasks across OnTaskExecuted() and busy-list
+ // removal so that RemoveInstance() / StopAllRunning() will wait until the
+ // worker is completely done using pTask and its sink.
+ CWelsAutoLock cLock (m_cLockBusyTasks);
+ if (pTask && pTask->GetSink()) {
+ //fprintf(stdout, "CWelsThreadPool::OnTaskStop 1: Task %x at Thread %x Finished, m_pSink=%x\n", pTask, pThread, pTask->GetSink());
+ pTask->GetSink()->OnTaskExecuted();
+ ////fprintf(stdout, "CWelsThreadPool::OnTaskStop 1: Task %x at Thread %x Finished, m_pSink=%x\n", pTask, pThread, pTask->GetSink());
+ }
+ m_cBusyThreads->erase (pThread);
}
//if (m_pSink) {
// m_pSink->OnTaskExecuted (pTask);
@@ -306,15 +322,6 @@ WELS_THREAD_ERROR_CODE CWelsThreadPool::AddThreadToBusyList (CWelsTaskThread* pT
return WELS_THREAD_ERROR_OK;
}
-WELS_THREAD_ERROR_CODE CWelsThreadPool::RemoveThreadFromBusyList (CWelsTaskThread* pThread) {
- CWelsAutoLock cLock (m_cLockBusyTasks);
- if (m_cBusyThreads->erase (pThread)) {
- return WELS_THREAD_ERROR_OK;
- } else {
- return WELS_THREAD_ERROR_GENERAL;
- }
-}
-
bool CWelsThreadPool::AddTaskToWaitedList (IWelsTask* pTask) {
CWelsAutoLock cLock (m_cLockWaitedTasks);
@@ -336,6 +343,7 @@ CWelsTaskThread* CWelsThreadPool::GetIdleThread() {
}
int32_t CWelsThreadPool::GetBusyThreadNum() {
+ CWelsAutoLock cLock (m_cLockBusyTasks);
return (m_cBusyThreads?m_cBusyThreads->size():0);
}
@@ -370,6 +378,7 @@ void CWelsThreadPool::ClearWaitedTasks() {
while (0 != m_cWaitedTasks->size()) {
pTask = m_cWaitedTasks->begin();
if (pTask->GetSink()) {
+ CWelsAutoLock cBusyLock (m_cLockBusyTasks);
pTask->GetSink()->OnTaskCancelled();
}
m_cWaitedTasks->pop_front();
diff --git a/codec/encoder/core/inc/wels_task_management.h b/codec/encoder/core/inc/wels_task_management.h
index acd61263..602e3af7 100644
--- a/codec/encoder/core/inc/wels_task_management.h
+++ b/codec/encoder/core/inc/wels_task_management.h
@@ -106,7 +106,6 @@ class CWelsTaskManageBase : public IWelsTaskManage, public WelsCommon::IWelsTas
int32_t m_iWaitTaskNum;
WELS_EVENT m_hTaskEvent;
WELS_MUTEX m_hEventMutex;
- WelsCommon::CWelsLock m_cWaitTaskNumLock;
private:
DISALLOW_COPY_AND_ASSIGN (CWelsTaskManageBase);
diff --git a/codec/encoder/core/src/wels_task_management.cpp b/codec/encoder/core/src/wels_task_management.cpp
index cc53d8d3..0d5839ab 100644
--- a/codec/encoder/core/src/wels_task_management.cpp
+++ b/codec/encoder/core/src/wels_task_management.cpp
@@ -124,10 +124,14 @@ WelsErrorType CWelsTaskManageBase::Init (sWelsEncCtx* pEncCtx) {
}
void CWelsTaskManageBase::Uninit() {
- DestroyTasks();
//fprintf(stdout, "m_pThreadPool = m_pThreadPool->RemoveInstance\n");
- if (m_pThreadPool)
+ if (m_pThreadPool) {
+ // Release the thread pool reference first to ensure no worker thread is
+ // still inside OnTaskExecuted() before destroying tasks, events, and mutexes.
m_pThreadPool->RemoveInstance();
+ m_pThreadPool = NULL;
+ }
+ DestroyTasks();
//WELS_DELETE_OP (m_pThreadPool);
//fprintf(stdout, "m_pThreadPool = m_pThreadPool->RemoveInstance2\n");
@@ -200,7 +204,6 @@ void CWelsTaskManageBase::DestroyTasks() {
void CWelsTaskManageBase::OnTaskMinusOne() {
//fprintf(stdout, "OnTaskMinusOne event %x m_iWaitTaskNum=%d\n", &m_hEventMutex, m_iWaitTaskNum);
- WelsCommon::CWelsAutoLock cAutoLock (m_cWaitTaskNumLock);
WelsEventSignal (&m_hTaskEvent, &m_hEventMutex, &m_iWaitTaskNum);
/*WelsMutexLock(&m_hEventMutex);
m_iWaitTaskNum --;
diff --git a/test/common/WelsThreadPoolTest.cpp b/test/common/WelsThreadPoolTest.cpp
index c4052a85..5d427f1e 100644
--- a/test/common/WelsThreadPoolTest.cpp
+++ b/test/common/WelsThreadPoolTest.cpp
@@ -145,3 +145,85 @@ TEST_F(CThreadPoolTestFixture, PartialInitLeakUAF) {
WelsThreadPoolTestUtil::SendSignalToOtherThreads();
}
+
+namespace {
+
+class CSyncEvent {
+ public:
+ CSyncEvent() : m_iCond (1) {
+ WelsMutexInit (&m_hMutex);
+ WelsEventOpen (&m_hEvent);
+ }
+ ~CSyncEvent() {
+ WelsEventClose (&m_hEvent);
+ WelsMutexDestroy (&m_hMutex);
+ }
+ void Signal() {
+ WelsEventSignal (&m_hEvent, &m_hMutex, &m_iCond);
+ }
+ void Wait() {
+ WelsEventWait (&m_hEvent, &m_hMutex, m_iCond);
+ }
+
+ private:
+ WELS_EVENT m_hEvent;
+ WELS_MUTEX m_hMutex;
+ int m_iCond;
+};
+
+class CImmediateTask : public IWelsTask {
+ public:
+ explicit CImmediateTask (WelsCommon::IWelsTaskSink* pSink) : IWelsTask (pSink) {}
+ virtual ~CImmediateTask() {}
+ virtual int32_t Execute() {
+ return cmResultSuccess;
+ }
+};
+
+// Sink whose OnTaskExecuted() signals the test thread upon entry and waits
+// until the test thread is ready to call RemoveInstance() before lingering.
+class CLingeringSink : public CThreadPoolTest {
+ public:
+ CLingeringSink() : m_iCallbacksReturned (0) {}
+
+ virtual int32_t OnTaskExecuted() {
+ int32_t iRet = CThreadPoolTest::OnTaskExecuted();
+ m_cCallbackEntered.Signal();
+ m_cRemoveStarted.Wait();
+ WelsSleep (5);
+ m_iCallbacksReturned ++;
+ return iRet;
+ }
+
+ CSyncEvent m_cCallbackEntered;
+ CSyncEvent m_cRemoveStarted;
+ volatile int32_t m_iCallbacksReturned;
+};
+
+} // namespace
+
+// Once RemoveInstance() has returned, the pool must no longer be inside any
+// callback of the caller's sink, even when other references keep the pool and
+// its worker threads alive.
+TEST (CThreadPoolTest, NoCallbackInFlightAfterRemoveInstance) {
+ CWelsThreadPool* pKeepAlive = CWelsThreadPool::AddReference();
+ ASSERT_TRUE (pKeepAlive != NULL);
+
+ for (int32_t i = 0; i < 5; i++) {
+ CLingeringSink cSink;
+ CImmediateTask cTask (&cSink);
+ CWelsThreadPool* pPool = CWelsThreadPool::AddReference();
+ ASSERT_TRUE (pPool != NULL);
+ ASSERT_EQ (WELS_THREAD_ERROR_OK, pPool->QueueTask (&cTask));
+ cSink.m_cCallbackEntered.Wait();
+ cSink.m_cRemoveStarted.Signal();
+ pPool->RemoveInstance();
+ EXPECT_EQ (1, cSink.m_iCallbacksReturned) << "iteration " << i;
+ while (cSink.m_iCallbacksReturned < 1) {
+ WelsSleep (1); // keep cSink alive until the worker is done with it
+ }
+ }
+
+ pKeepAlive->RemoveInstance();
+}
+
diff --git a/test/encoder/EncUT_EncoderTaskManagement.cpp b/test/encoder/EncUT_EncoderTaskManagement.cpp
index f0d499a0..a1a55599 100644
--- a/test/encoder/EncUT_EncoderTaskManagement.cpp
+++ b/test/encoder/EncUT_EncoderTaskManagement.cpp
@@ -46,4 +46,151 @@ TEST (EncoderTaskManagement, CWelsTaskManageMultiD) {
ASSERT_TRUE (NULL != pTaskManage);
delete pTaskManage;
+}
+
+namespace {
+
+class CSyncEvent {
+ public:
+ CSyncEvent() : m_iCond (1) {
+ WelsMutexInit (&m_hMutex);
+ WelsEventOpen (&m_hEvent);
+ }
+ ~CSyncEvent() {
+ WelsEventClose (&m_hEvent);
+ WelsMutexDestroy (&m_hMutex);
+ }
+ void Signal() {
+ WelsEventSignal (&m_hEvent, &m_hMutex, &m_iCond);
+ }
+ void Wait() {
+ WelsEventWait (&m_hEvent, &m_hMutex, m_iCond);
+ }
+
+ private:
+ WELS_EVENT m_hEvent;
+ WELS_MUTEX m_hMutex;
+ int m_iCond;
+};
+
+class CNoopTask : public CWelsBaseTask {
+ public:
+ explicit CNoopTask (WelsCommon::IWelsTaskSink* pSink) : CWelsBaseTask (pSink) {}
+ virtual ~CNoopTask() {}
+ virtual WelsErrorType Execute() {
+ return ENC_RETURN_SUCCESS;
+ }
+ virtual uint32_t GetTaskType() const {
+ return CWelsBaseTask::WELS_ENC_TASK_ENCODING;
+ }
+};
+
+// Number of OnTaskExecuted() callbacks that have run to completion.
+static WelsCommon::CWelsLock g_cCallbackLock;
+static int32_t g_iCallbacksDone = 0;
+
+static int32_t GetCallbacksDone() {
+ WelsCommon::CWelsAutoLock cLock (g_cCallbackLock);
+ return g_iCallbacksDone;
+}
+
+// Task manager with two trivial tasks whose final OnTaskExecuted() callback
+// waits until the owner thread has entered ~CLingeringTaskManage() (right
+// before ~CWelsTaskManageBase() calls Uninit()) and then lingers briefly.
+class CLingeringTaskManage : public CWelsTaskManageBase {
+ public:
+ explicit CLingeringTaskManage (CSyncEvent* pDestroyStarted)
+ : m_pDestroyStarted (pDestroyStarted), m_iCallbackOrder (0) {}
+ virtual ~CLingeringTaskManage() {
+ m_pDestroyStarted->Signal();
+ }
+
+ virtual WelsErrorType OnTaskExecuted() {
+ CSyncEvent* pDestroyStarted = m_pDestroyStarted;
+ int32_t iOrder;
+ {
+ WelsCommon::CWelsAutoLock cLock (g_cCallbackLock);
+ iOrder = ++m_iCallbackOrder;
+ }
+ WelsErrorType eRet = CWelsTaskManageBase::OnTaskExecuted();
+ if (iOrder == 2) {
+ pDestroyStarted->Wait();
+ WelsSleep (5);
+ }
+ WelsCommon::CWelsAutoLock cLock (g_cCallbackLock);
+ g_iCallbacksDone ++;
+ return eRet;
+ }
+
+ protected:
+ virtual WelsErrorType CreateTasks (sWelsEncCtx* pEncCtx, const int32_t kiCurDid) {
+ if (kiCurDid > 0) {
+ return ENC_RETURN_SUCCESS;
+ }
+ m_iTaskNum[0] = 2;
+ for (int32_t idx = 0; idx < m_iTaskNum[0]; ++idx) {
+ CWelsBaseTask* pTask = WELS_NEW_OP (CNoopTask (this), CNoopTask);
+ WELS_VERIFY_RETURN_IF (ENC_RETURN_MEMALLOCERR, NULL == pTask)
+ WELS_VERIFY_RETURN_IF (ENC_RETURN_MEMALLOCERR, true != m_cEncodingTaskList[0]->push_back (pTask))
+ }
+ return ENC_RETURN_SUCCESS;
+ }
+
+ private:
+ CSyncEvent* m_pDestroyStarted;
+ int32_t m_iCallbackOrder;
+};
+
+} // namespace
+
+// Destroying a task manager right after ExecuteTasks() returns must not leave a
+// pool worker still inside one of its callbacks.
+TEST (EncoderTaskManagement, NoCallbackInFlightAfterDestroy) {
+ // A second task manager keeps the shared thread pool alive, so that tearing
+ // down the task manager under test does not stop and join the workers.
+ sWelsEncCtx sKeepAliveCtx;
+ SWelsSvcCodingParam sKeepAliveParam;
+ memset (&sKeepAliveCtx, 0, sizeof (sKeepAliveCtx));
+ sKeepAliveCtx.pSvcParam = &sKeepAliveParam;
+ sKeepAliveParam.iMultipleThreadIdc = 2;
+ IWelsTaskManage* pKeepAlive = IWelsTaskManage::CreateTaskManage (&sKeepAliveCtx, 1, false);
+ ASSERT_TRUE (NULL != pKeepAlive);
+
+ sWelsEncCtx sCtx;
+ SWelsSvcCodingParam sParam;
+ SDqLayer sCurDqLayer;
+ memset (&sCtx, 0, sizeof (sCtx));
+ memset (&sCurDqLayer, 0, sizeof (sCurDqLayer));
+ sCtx.pSvcParam = &sParam;
+ sCtx.pCurDqLayer = &sCurDqLayer;
+ sParam.iMultipleThreadIdc = 2;
+
+ {
+ WelsCommon::CWelsAutoLock cLock (g_cCallbackLock);
+ g_iCallbacksDone = 0;
+ }
+
+ const int32_t kiIterations = 5;
+ for (int32_t i = 0; i < kiIterations; ++i) {
+ CSyncEvent cDestroyStarted;
+ CLingeringTaskManage* pTaskManage = new CLingeringTaskManage (&cDestroyStarted);
+ ASSERT_EQ (ENC_RETURN_SUCCESS, pTaskManage->Init (&sCtx));
+ pTaskManage->InitFrame (0);
+ ASSERT_EQ (ENC_RETURN_SUCCESS, pTaskManage->ExecuteTasks());
+ delete pTaskManage;
+
+ // Both callbacks must have completed before the destructor returned.
+ const int32_t iExpected = 2 * (i + 1);
+ const int32_t iDone = GetCallbacksDone();
+ EXPECT_EQ (iExpected, iDone) << "iteration " << i;
+ if (iExpected != iDone) {
+ // Let the lingering worker finish before leaving the test.
+ while (GetCallbacksDone() < iExpected) {
+ WelsSleep (1);
+ }
+ break;
+ }
+ }
+
+ delete pKeepAlive;
}
\ No newline at end of file