Commit 1189afa8 for openh264
commit 1189afa8d6f243a544fb511cbae3043ce79e6c84
Author: Erik Språng <sprang@google.com>
Date: Wed Jul 15 10:58:53 2026 +0200
Fix thread cleanup and memory leaks during partial initialization (#3959)
If CWelsThreadPool::Init() fails halfway through creating worker threads, StopAllRunning() returns an error code because the number of idle threads does not match m_iMaxThreadNum. Previously, Uninit() would return early on this error code without terminating or destroying the worker threads that were already successfully started, leaving them running after the pool was destroyed.
This commit updates Uninit() to systematically terminate, join, and clean up all worker threads across both idle and busy lists regardless of StopAllRunning()'s return value. Additionally, if pThread->Start() fails in CreateIdleThread(), the allocated thread object is now freed before returning.
Co-authored-by: Erik Språng <sprang@webrtc.org>
diff --git a/codec/common/src/WelsThreadPool.cpp b/codec/common/src/WelsThreadPool.cpp
index 302643cc..2b140019 100644
--- a/codec/common/src/WelsThreadPool.cpp
+++ b/codec/common/src/WelsThreadPool.cpp
@@ -206,12 +206,10 @@ WELS_THREAD_ERROR_CODE CWelsThreadPool::Uninit() {
CWelsAutoLock cLock (m_cLockPool);
iReturn = StopAllRunning();
- if (WELS_THREAD_ERROR_OK != iReturn) {
- return iReturn;
- }
+ assert (0 == GetBusyThreadNum());
m_cLockIdleTasks.Lock();
- while (m_cIdleThreads->size() > 0) {
+ while (m_cIdleThreads && m_cIdleThreads->size() > 0) {
DestroyThread (m_cIdleThreads->begin());
m_cIdleThreads->pop_front();
}
@@ -280,6 +278,7 @@ WELS_THREAD_ERROR_CODE CWelsThreadPool::CreateIdleThread() {
}
if (WELS_THREAD_ERROR_OK != pThread->Start()) {
+ WELS_DELETE_OP (pThread);
return WELS_THREAD_ERROR_GENERAL;
}
//fprintf(stdout, "ThreadPool: AddThreadToIdleQueue: %x\n", pThread);
diff --git a/test/common/WelsThreadPoolTest.cpp b/test/common/WelsThreadPoolTest.cpp
index 6c3609cd..f3e39286 100644
--- a/test/common/WelsThreadPoolTest.cpp
+++ b/test/common/WelsThreadPoolTest.cpp
@@ -1,14 +1,17 @@
+#include "WelsThreadPoolTest.h"
+
#include <gtest/gtest.h>
#include <string.h>
-#include <string>
+
#include <list>
#include <map>
+#include <string>
-#include "typedefs.h"
+#include "WelsTask.h"
#include "WelsThreadLib.h"
#include "WelsThreadPool.h"
-#include "WelsTask.h"
-#include "WelsThreadPoolTest.h"
+#include "WelsThreadPoolTestUtil.h"
+#include "typedefs.h"
#define TEST_TASK_NUM 30
@@ -104,3 +107,37 @@ TEST (CThreadPoolTest, CThreadPoolTestMulti) {
EXPECT_FALSE (CWelsThreadPool::IsReferenced());
}
+class CThreadPoolTestFixture : public ::testing::Test {
+ protected:
+ virtual void SetUp() { WelsThreadPoolTestUtil::SetupSignalHandler(); }
+
+ virtual void TearDown() {
+ WelsThreadPoolTestUtil::RemoveThreadLimit();
+ WelsThreadPoolTestUtil::RestoreSignalHandler();
+ }
+};
+
+TEST_F(CThreadPoolTestFixture, PartialInitLeakUAF) {
+ WelsThreadPoolTestUtil::SThreadLimitResult sLimit =
+ WelsThreadPoolTestUtil::FindSingleThreadLimit();
+ if (!sLimit.bSupported) {
+ printf("Thread limit manipulation is not supported, skipping test.\n");
+ return;
+ }
+
+ WelsThreadPoolTestUtil::CThreadLimitGuard cLimitGuard;
+
+ ASSERT_TRUE(WelsThreadPoolTestUtil::SetThreadLimit(sLimit.uiLimit));
+
+ // Request 2 threads: the first worker thread (i = 0) succeeds, while
+ // attempting to create the second (i = 1) fails due to our 1-thread limit,
+ // triggering a controlled partial initialization failure.
+ CWelsThreadPool::SetThreadNum(2);
+ CWelsThreadPool* pPool = CWelsThreadPool::AddReference();
+ EXPECT_EQ(NULL, pPool);
+
+ // Restore old rlimit immediately so we can create threads/signals normally.
+ WelsThreadPoolTestUtil::RemoveThreadLimit();
+
+ WelsThreadPoolTestUtil::SendSignalToOtherThreads();
+}
diff --git a/test/common/WelsThreadPoolTestUtil.h b/test/common/WelsThreadPoolTestUtil.h
new file mode 100644
index 00000000..ea34173e
--- /dev/null
+++ b/test/common/WelsThreadPoolTestUtil.h
@@ -0,0 +1,208 @@
+/*!
+ * \copy
+ * Copyright (c) 2009-2026, Cisco Systems
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ * \file WelsThreadPoolTestUtil.h
+ *
+ * \brief OS-agnostic utilities for thread pool testing (thread limits and signal handling)
+ *
+ */
+
+#ifndef _WELS_THREAD_POOL_TEST_UTIL_H_
+#define _WELS_THREAD_POOL_TEST_UTIL_H_
+
+#include "typedefs.h"
+#include <stdio.h>
+
+#if defined(__linux__)
+#include <sys/resource.h>
+#include <signal.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <dirent.h>
+#include <sys/syscall.h>
+#include <stdlib.h>
+#include <string.h>
+#endif
+
+namespace WelsThreadPoolTestUtil {
+
+struct SThreadLimitResult {
+ bool bSupported;
+ size_t uiLimit;
+};
+
+#if defined(__linux__)
+
+static struct rlimit g_sOriginalRlimit;
+static bool g_bOriginalRlimitSaved = false;
+static struct sigaction g_sOriginalSigaction;
+static bool g_bOriginalSigactionSaved = false;
+
+inline void* DummyEmptyThreadRoutine (void*) {
+ return NULL;
+}
+
+inline void DummySignalHandler (int) {
+ // Do nothing, just interrupt system calls like sem_wait
+}
+
+inline SThreadLimitResult FindSingleThreadLimit() {
+ SThreadLimitResult sRes = { false, 0 };
+ if (0 != getrlimit (RLIMIT_NPROC, &g_sOriginalRlimit)) {
+ return sRes;
+ }
+ g_bOriginalRlimitSaved = true;
+
+ struct rlimit rl = g_sOriginalRlimit;
+ int low = 1, high = (int)g_sOriginalRlimit.rlim_cur;
+ while (low < high) {
+ int mid = (low + high) / 2;
+ rl.rlim_cur = mid;
+ if (setrlimit (RLIMIT_NPROC, &rl) != 0) {
+ low = mid + 1;
+ continue;
+ }
+ pthread_t th;
+ int res = pthread_create (&th, NULL, DummyEmptyThreadRoutine, NULL);
+ if (res == 0) {
+ pthread_join (th, NULL);
+ high = mid;
+ } else {
+ low = mid + 1;
+ }
+ }
+
+ sRes.bSupported = true;
+ sRes.uiLimit = (size_t)low;
+ return sRes;
+}
+
+inline bool SetThreadLimit (size_t uiLimit) {
+ if (!g_bOriginalRlimitSaved) {
+ if (0 != getrlimit (RLIMIT_NPROC, &g_sOriginalRlimit)) {
+ return false;
+ }
+ g_bOriginalRlimitSaved = true;
+ }
+ struct rlimit rl = g_sOriginalRlimit;
+ rl.rlim_cur = uiLimit;
+ return (0 == setrlimit (RLIMIT_NPROC, &rl));
+}
+
+inline bool RemoveThreadLimit() {
+ if (!g_bOriginalRlimitSaved) {
+ return true;
+ }
+ return (0 == setrlimit (RLIMIT_NPROC, &g_sOriginalRlimit));
+}
+
+inline void SetupSignalHandler() {
+ struct sigaction sa;
+ memset (&sa, 0, sizeof (sa));
+ sa.sa_handler = DummySignalHandler;
+ sa.sa_flags = 0; // Ensure SA_RESTART is NOT set so sem_wait returns EINTR
+ sigemptyset (&sa.sa_mask);
+ if (!g_bOriginalSigactionSaved) {
+ sigaction (SIGUSR1, &sa, &g_sOriginalSigaction);
+ g_bOriginalSigactionSaved = true;
+ } else {
+ sigaction (SIGUSR1, &sa, NULL);
+ }
+}
+
+inline void RestoreSignalHandler() {
+ if (g_bOriginalSigactionSaved) {
+ sigaction (SIGUSR1, &g_sOriginalSigaction, NULL);
+ g_bOriginalSigactionSaved = false;
+ }
+}
+
+inline void SendSignalToOtherThreads() {
+ for (int i = 0; i < 10; i++) {
+ usleep (10000);
+ DIR* dir = opendir ("/proc/self/task");
+ if (dir) {
+ struct dirent* entry;
+ pid_t my_tid = syscall (SYS_gettid);
+ pid_t my_pid = getpid();
+ while ((entry = readdir (dir)) != NULL) {
+ if (entry->d_name[0] >= '0' && entry->d_name[0] <= '9') {
+ pid_t tid = atoi (entry->d_name);
+ if (tid != my_tid) {
+ syscall (SYS_tgkill, my_pid, tid, SIGUSR1);
+ }
+ }
+ }
+ closedir (dir);
+ }
+ }
+}
+
+#else
+
+inline SThreadLimitResult FindSingleThreadLimit() {
+ SThreadLimitResult sRes = { false, 0 };
+ return sRes;
+}
+
+inline bool SetThreadLimit (size_t uiLimit) {
+ return false;
+}
+
+inline bool RemoveThreadLimit() {
+ return false;
+}
+
+inline void SetupSignalHandler() {
+}
+
+inline void RestoreSignalHandler() {
+}
+
+inline void SendSignalToOtherThreads() {
+}
+
+#endif
+
+class CTestStateGuard {
+ public:
+ CTestStateGuard() {}
+ ~CTestStateGuard() {
+ RemoveThreadLimit();
+ RestoreSignalHandler();
+ }
+};
+
+typedef CTestStateGuard CThreadLimitGuard;
+
+} // namespace WelsThreadPoolTestUtil
+
+#endif