Commit b8e4a4df for guacamole.apache.org

commit b8e4a4df5516c9d249ce2de30832e861ebb1407f
Author: Stephen Schiffli <sschiffli@keepersecurity.com>
Date:   Fri Mar 13 07:31:27 2026 -0700

    GUACAMOLE-1998: Refactor rwlock TLS tracking to use a single global key with per-thread slot array.

diff --git a/src/libguac/guacamole/rwlock.h b/src/libguac/guacamole/rwlock.h
index 88d0029e..90a4546d 100644
--- a/src/libguac/guacamole/rwlock.h
+++ b/src/libguac/guacamole/rwlock.h
@@ -39,27 +39,11 @@
  */

 /**
- * A structure packaging together a pthread rwlock along with a key to a
- * thread-local property to keep track of the current status of the lock,
- * allowing the functions defined in this header to provide reentrant behavior.
- * Note that both the lock and key must be initialized before being provided
- * to any of these functions.
+ * A reentrant read-write lock. Callers must use only the guac_rwlock_*
+ * functions to acquire and release this lock. Using pthread rwlock functions
+ * directly will break the reentrant tracking.
  */
-typedef struct guac_rwlock {
-
-    /**
-     * A non-reentrant pthread rwlock to be wrapped by the local lock,
-     * functions providing reentrant behavior.
-     */
-    pthread_rwlock_t lock;
-
-    /**
-     * A key to access a thread-local property tracking any ownership of the
-     * lock by the current thread.
-     */
-    pthread_key_t key;
-
-} guac_rwlock;
+typedef pthread_rwlock_t guac_rwlock;

 /**
  * Initialize the provided guac reentrant rwlock. The lock will be configured to be
@@ -79,11 +63,9 @@ void guac_rwlock_init(guac_rwlock* lock);
 void guac_rwlock_destroy(guac_rwlock* lock);

 /**
- * Acquire the write lock for the provided guac reentrant rwlock, if the key does not
- * indicate that the write lock is already acquired. If the key indicates that
- * the read lock is already acquired, the read lock will be dropped before the
- * write lock is acquired. The thread local property associated with the key
- * will be updated as necessary to track the thread's ownership of the lock.
+ * Acquires the write lock for the provided guac reentrant rwlock, or increments
+ * its hold count if the current thread already holds it. If the current thread
+ * holds a read lock, it will be released and a write lock acquired.
  *
  * If an error occurs while attempting to acquire the lock, a non-zero value is
  * returned, and guac_error is set appropriately.
@@ -99,10 +81,8 @@ void guac_rwlock_destroy(guac_rwlock* lock);
 int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock);

 /**
- * Acquire the read lock for the provided guac reentrant rwlock, if the key does not
- * indicate that the read or write lock is already acquired. The thread local
- * property associated with the key will be updated as necessary to track the
- * thread's ownership of the lock.
+ * Acquires the read lock for the provided guac reentrant rwlock, or increments
+ * its hold count if the current thread already holds a read or write lock.
  *
  * If an error occurs while attempting to acquire the lock, a non-zero value is
  * returned, and guac_error is set appropriately.
@@ -118,10 +98,9 @@ int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock);
 int guac_rwlock_acquire_read_lock(guac_rwlock* reentrant_rwlock);

 /**
- * Release the rwlock associated with the provided guac reentrant rwlock if this
- * is the last level of the lock held by this thread. Otherwise, the thread
- * local property associated with the key will be updated as needed to ensure
- * that the correct number of release requests will finally release the lock.
+ * Releases the rwlock associated with the provided guac reentrant rwlock if
+ * this is the last level held by the current thread. Otherwise, decrements the
+ * hold count to reflect the pending release.
  *
  * If an error occurs while attempting to release the lock, a non-zero value is
  * returned, and guac_error is set appropriately.
diff --git a/src/libguac/rwlock.c b/src/libguac/rwlock.c
index 1ed71766..925a2c14 100644
--- a/src/libguac/rwlock.c
+++ b/src/libguac/rwlock.c
@@ -17,9 +17,12 @@
  * under the License.
  */

+#include <limits.h>
 #include <pthread.h>
 #include <stdint.h>
+
 #include "guacamole/error.h"
+#include "guacamole/mem.h"
 #include "guacamole/rwlock.h"

 /**
@@ -38,123 +41,192 @@
  */
 #define GUAC_REENTRANT_LOCK_WRITE_LOCK 2

-void guac_rwlock_init(guac_rwlock* lock) {
+/**
+ * The maximum number of distinct guac_rwlock instances a single thread may
+ * hold simultaneously.
+ */
+#define GUAC_RWLOCK_MAX_HELD 16

-    /* Configure to allow sharing this lock with child processes */
-    pthread_rwlockattr_t lock_attributes;
-    pthread_rwlockattr_init(&lock_attributes);
-    pthread_rwlockattr_setpshared(&lock_attributes, PTHREAD_PROCESS_SHARED);
+/**
+ * Per-lock state tracked for a single thread. One slot is allocated per
+ * distinct lock that the thread currently holds.
+ */
+typedef struct guac_rwlock_thread_state {

-    /* Initialize the rwlock */
-    pthread_rwlock_init(&(lock->lock), &lock_attributes);
+    /**
+     * The lock this slot describes. NULL indicates the slot is empty.
+     */
+    guac_rwlock* lock;

-    /* Initialize the  flags to 0, as threads won't have acquired it yet */
-    pthread_key_create(&(lock->key), (void *) 0);
+    /**
+     * Which lock the current thread holds: GUAC_REENTRANT_LOCK_NO_LOCK,
+     * GUAC_REENTRANT_LOCK_READ_LOCK, or GUAC_REENTRANT_LOCK_WRITE_LOCK.
+     */
+    int flag;

-}
+    /**
+     * The reentrant depth, representing the number of times the current thread
+     * has acquired this lock without a corresponding release.
+     */
+    unsigned int count;

-void guac_rwlock_destroy(guac_rwlock* lock) {
+} guac_rwlock_thread_state;

-    /* Destroy the rwlock */
-    pthread_rwlock_destroy(&(lock->lock));
+/**
+ * A single process wide key whose per-thread value is a pointer to that
+ * thread's array of guac_rwlock_thread_state entries. Created exactly once
+ * via pthread_once.
+ */
+static pthread_key_t guac_rwlock_key;
+static pthread_once_t guac_rwlock_key_init = PTHREAD_ONCE_INIT;

-    /* Destroy the thread-local key */
-    pthread_key_delete(lock->key);
+/**
+ * Destructor registered with pthread_key_create that frees the per-thread
+ * guac_rwlock_thread_state array when a thread exits.
+ *
+ * @param pointer
+ *     Pointer to the per-thread state array to free.
+ */
+static void guac_rwlock_free_pointer(void* pointer) {
+
+    guac_mem_free(pointer);

 }

 /**
- * Clean up and destroy the provided guac reentrant rwlock.
- *
- * @param lock
- *     The guac reentrant rwlock to be destroyed.
+ * Creates the single global thread-local key. Invoked exactly once via
+ * pthread_once.
  */
-void guac_rwlock_destroy(guac_rwlock* lock);
+static void guac_rwlock_create_key(void) {
+
+    pthread_key_create(&guac_rwlock_key, guac_rwlock_free_pointer);
+
+}

 /**
- * Extract and return the flag indicating which lock is held, if any, from the
- * provided key value. The flag is always stored in the least-significant
- * nibble of the value.
- *
- * @param value
- *     The key value containing the flag.
+ * Returns the per-thread state array, allocating and registering it on first
+ * use.
  *
  * @return
- *     The flag indicating which lock is held, if any.
+ *     A pointer to the calling thread's guac_rwlock_thread_state array, or
+ *     NULL if allocation fails.
  */
-static uintptr_t get_lock_flag(uintptr_t value) {
-    return value & 0xF;
+static guac_rwlock_thread_state* guac_rwlock_get_thread_states(void) {
+
+    pthread_once(&guac_rwlock_key_init, guac_rwlock_create_key);
+
+    guac_rwlock_thread_state* states = pthread_getspecific(guac_rwlock_key);
+    if (states == NULL) {
+        states = guac_mem_zalloc(sizeof(guac_rwlock_thread_state), GUAC_RWLOCK_MAX_HELD);
+        pthread_setspecific(guac_rwlock_key, states);
+    }
+
+    return states;
+
 }

 /**
- * Extract and return the lock count from the provided key. This returned value
- * is the difference between the number of lock and unlock requests made by the
- * current thread. This count is always stored in the remaining value after the
- * least-significant nibble where the flag is stored.
+ * Returns the state slot for the given lock for the current thread, or NULL
+ * if the current thread does not hold that lock.
  *
- * @param value
- *     The key value containing the count.
+ * @param lock
+ *     The lock whose state slot should be retrieved.
  *
  * @return
- *     The difference between the number of lock and unlock requests made by
- *     the current thread.
+ *     The state slot for the given lock, or NULL if the current thread does
+ *     not hold that lock.
  */
-static uintptr_t get_lock_count(uintptr_t value) {
-    return value >> 4;
+static guac_rwlock_thread_state* guac_rwlock_state_get(guac_rwlock* lock) {
+
+    guac_rwlock_thread_state* states = guac_rwlock_get_thread_states();
+
+    for (int i = 0; i < GUAC_RWLOCK_MAX_HELD; i++) {
+        if (states[i].lock == lock)
+            return &states[i];
+    }
+
+    return NULL;
+
 }

 /**
- * Given a flag indicating if and how the current thread controls a lock, and
- * a count of the depth of lock requests, return a value containing the flag
- * in the least-significant nibble, and the count in the rest.
- *
- * @param flag
- *     A flag indicating which lock, if any, is held by the current thread.
+ * Returns the state slot for the given lock for the current thread, creating
+ * and initializing a new slot if one does not already exist. Returns NULL if
+ * all slots are in use.
  *
- * @param count
- *     The depth of the lock attempt by the current thread, i.e. the number of
- *     lock requests minus unlock requests.
+ * @param lock
+ *     The lock whose state slot should be retrieved or created.
  *
  * @return
- *     A value containing the flag in the least-significant nibble, and the
- *     count in the rest, cast to a void* for thread-local storage.
+ *     The state slot for the given lock, or NULL if all slots are in use.
  */
-static void* get_value_from_flag_and_count(
-        uintptr_t flag, uintptr_t count) {
-    return (void*) ((flag & 0xF) | count << 4);
+static guac_rwlock_thread_state* guac_rwlock_state_get_or_create(guac_rwlock* lock) {
+
+    guac_rwlock_thread_state* states = guac_rwlock_get_thread_states();
+
+    guac_rwlock_thread_state* empty = NULL;
+    for (int i = 0; i < GUAC_RWLOCK_MAX_HELD; i++) {
+        if (states[i].lock == lock)
+            return &states[i];
+        if (empty == NULL && states[i].lock == NULL)
+            empty = &states[i];
+    }
+
+    if (empty != NULL) {
+        empty->lock = lock;
+        empty->flag = GUAC_REENTRANT_LOCK_NO_LOCK;
+        empty->count = 0;
+    }
+
+    return empty;
+
 }

 /**
- * Return zero if adding one to the current count would overflow the storage
- * allocated to the count, or a non-zero value otherwise.
- *
- * @param current_count
- *     The current count for a lock that the current thread is trying to
- *     reentrantly acquire.
+ * Clears a state slot, marking it as empty so it can be reused.
  *
- * @return
- *     Zero if adding one to the current count would overflow the storage
- *     allocated to the count, or a non-zero value otherwise.
+ * @param state
+ *     The state slot to clear.
  */
-static int would_overflow_count(uintptr_t current_count) {
+static void guac_rwlock_state_clear(guac_rwlock_thread_state* state) {
+    state->lock = NULL;
+    state->flag = GUAC_REENTRANT_LOCK_NO_LOCK;
+    state->count = 0;
+}

-    /**
-     * The count will overflow if it's already equal or greater to the maximum
-     * possible value that can be stored in a uintptr_t excluding the first nibble.
-     */
-    return current_count >= (UINTPTR_MAX >> 4);
+void guac_rwlock_init(guac_rwlock* lock) {
+
+    /* Configure to allow sharing this lock with child processes */
+    pthread_rwlockattr_t lock_attributes;
+    pthread_rwlockattr_init(&lock_attributes);
+    pthread_rwlockattr_setpshared(&lock_attributes, PTHREAD_PROCESS_SHARED);
+
+    /* Initialize the rwlock */
+    pthread_rwlock_init(lock, &lock_attributes);
+    pthread_rwlockattr_destroy(&lock_attributes);
+
+}
+
+void guac_rwlock_destroy(guac_rwlock* lock) {
+
+    /* Destroy the rwlock */
+    pthread_rwlock_destroy(lock);

 }

 int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock) {

-    uintptr_t key_value = (uintptr_t) pthread_getspecific(reentrant_rwlock->key);
-    uintptr_t flag = get_lock_flag(key_value);
-    uintptr_t count = get_lock_count(key_value);
+    guac_rwlock_thread_state* state = guac_rwlock_state_get_or_create(reentrant_rwlock);

-    /* If acquiring this lock again would overflow the counter storage */
-    if (would_overflow_count(count)) {
+    if (state == NULL) {
+        guac_error = GUAC_STATUS_TOO_MANY;
+        guac_error_message = "Unable to acquire write lock because there's"
+                " too many locks held simultaneously by this thread";
+        return 1;
+    }

+    /* If acquiring this lock again would overflow the counter storage */
+    if (state->count >= UINT_MAX) {
         guac_error = GUAC_STATUS_TOO_MANY;
         guac_error_message = "Unable to acquire write lock because there's"
                 " insufficient space to store another level of lock depth";
@@ -164,9 +236,8 @@ int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock) {
     }

     /* If the current thread already holds the write lock, increment the count */
-    if (flag == GUAC_REENTRANT_LOCK_WRITE_LOCK) {
-        pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-                flag, count + 1));
+    if (state->flag == GUAC_REENTRANT_LOCK_WRITE_LOCK) {
+        state->count++;

         /* This thread already has the lock */
         return 0;
@@ -179,15 +250,32 @@ int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock) {
      * write lock by another function without the caller knowing about it. This
      * shouldn't cause any issues, however.
      */
-    if (flag == GUAC_REENTRANT_LOCK_READ_LOCK)
-        pthread_rwlock_unlock(&(reentrant_rwlock->lock));
+    if (state->flag == GUAC_REENTRANT_LOCK_READ_LOCK) {
+        int unlock_err = pthread_rwlock_unlock(reentrant_rwlock);
+        if (unlock_err) {
+            guac_error = GUAC_STATUS_SEE_ERRNO;
+            guac_error_message = "Unable to release read lock for write lock upgrade";
+            return 1;
+        }
+    }

     /* Acquire the write lock */
-    pthread_rwlock_wrlock(&(reentrant_rwlock->lock));
+    int err = pthread_rwlock_wrlock(reentrant_rwlock);
+    if (err) {
+
+        /* The read lock was released above but the write lock was not acquired,
+         * so the current thread no longer holds any lock */
+        if (state->flag == GUAC_REENTRANT_LOCK_READ_LOCK)
+            guac_rwlock_state_clear(state);

-    /* Mark that the current thread has the lock, and increment the count */
-    pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-            GUAC_REENTRANT_LOCK_WRITE_LOCK, count + 1));
+        guac_error = GUAC_STATUS_SEE_ERRNO;
+        guac_error_message = "Unable to acquire write lock";
+        return 1;
+
+    }
+
+    state->flag = GUAC_REENTRANT_LOCK_WRITE_LOCK;
+    state->count++;

     return 0;

@@ -195,12 +283,20 @@ int guac_rwlock_acquire_write_lock(guac_rwlock* reentrant_rwlock) {

 int guac_rwlock_acquire_read_lock(guac_rwlock* reentrant_rwlock) {

-    uintptr_t key_value = (uintptr_t) pthread_getspecific(reentrant_rwlock->key);
-    uintptr_t flag = get_lock_flag(key_value);
-    uintptr_t count = get_lock_count(key_value);
+    guac_rwlock_thread_state* state = guac_rwlock_state_get_or_create(reentrant_rwlock);
+
+    if (state == NULL) {
+
+        guac_error = GUAC_STATUS_TOO_MANY;
+        guac_error_message = "Unable to acquire read lock because there's"
+                " too many locks held simultaneously by this thread";
+
+        return 1;
+
+    }

     /* If acquiring this lock again would overflow the counter storage */
-    if (would_overflow_count(count)) {
+    if (state->count >= UINT_MAX) {

         guac_error = GUAC_STATUS_TOO_MANY;
         guac_error_message = "Unable to acquire read lock because there's"
@@ -212,24 +308,28 @@ int guac_rwlock_acquire_read_lock(guac_rwlock* reentrant_rwlock) {

     /* The current thread may read if either the read or write lock is held */
     if (
-            flag == GUAC_REENTRANT_LOCK_READ_LOCK ||
-            flag == GUAC_REENTRANT_LOCK_WRITE_LOCK
+            state->flag == GUAC_REENTRANT_LOCK_READ_LOCK ||
+            state->flag == GUAC_REENTRANT_LOCK_WRITE_LOCK
     ) {

         /* Increment the depth counter */
-        pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-                flag, count + 1));
+        state->count++;

         /* This thread already has the lock */
         return 0;
     }

     /* Acquire the lock */
-    pthread_rwlock_rdlock(&(reentrant_rwlock->lock));
+    int err = pthread_rwlock_rdlock(reentrant_rwlock);
+    if (err) {
+        guac_error = GUAC_STATUS_SEE_ERRNO;
+        guac_error_message = "Unable to acquire read lock";
+        return 1;
+    }

     /* Set the flag that the current thread has the read lock */
-    pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-                GUAC_REENTRANT_LOCK_READ_LOCK, 1));
+    state->flag  = GUAC_REENTRANT_LOCK_READ_LOCK;
+    state->count = 1;

     return 0;

@@ -237,15 +337,13 @@ int guac_rwlock_acquire_read_lock(guac_rwlock* reentrant_rwlock) {

 int guac_rwlock_release_lock(guac_rwlock* reentrant_rwlock) {

-    uintptr_t key_value = (uintptr_t) pthread_getspecific(reentrant_rwlock->key);
-    uintptr_t flag = get_lock_flag(key_value);
-    uintptr_t count = get_lock_count(key_value);
+    guac_rwlock_thread_state* state = guac_rwlock_state_get(reentrant_rwlock);

     /*
      * Return an error if an attempt is made to release a lock that the current
      * thread does not control.
      */
-    if (count <= 0) {
+    if (state == NULL || state->count == 0) {

         guac_error = GUAC_STATUS_INVALID_ARGUMENT;
         guac_error_message = "Unable to free rwlock because it's not held by"
@@ -256,20 +354,18 @@ int guac_rwlock_release_lock(guac_rwlock* reentrant_rwlock) {
     }

     /* Release the lock if this is the last locked level */
-    if (count == 1) {
+    if (state->count == 1) {

-        pthread_rwlock_unlock(&(reentrant_rwlock->lock));
+        pthread_rwlock_unlock(reentrant_rwlock);

         /* Set the flag that the current thread holds no locks */
-        pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-                GUAC_REENTRANT_LOCK_NO_LOCK, 0));
+        guac_rwlock_state_clear(state);

         return 0;
     }

     /* Do not release the lock since it's still in use - just decrement */
-    pthread_setspecific(reentrant_rwlock->key, get_value_from_flag_and_count(
-            flag, count - 1));
+    state->count--;

     return 0;