Commit 0b0284eb16d for php.net
commit 0b0284eb16dba22bcb6a56d8da9604b8ec4228cc
Author: henderkes <m@pyc.ac>
Date: Sat Jul 4 21:25:52 2026 +0700
perf: move AG and SCNG into native __thread storage
Replay of GH-22231
Closes GH-22595
diff --git a/NEWS b/NEWS
index 35eb80d675c..8c29466dcc6 100644
--- a/NEWS
+++ b/NEWS
@@ -12,6 +12,7 @@ PHP NEWS
. Lock unmodified readonly properties for modification after clone-with.
(NickSdot)
. abort() instead of exit() on hard OOM. (realFlowControl)
+ . perf: ZTS: move AG and SCNG into native __thread storage. (henderkes)
- Calendar:
. Fixed bug GH-22602 (gregoriantojd() and juliantojd() integer overflow with
diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c
index 4222e88755d..a5032e456aa 100644
--- a/TSRM/TSRM.c
+++ b/TSRM/TSRM.c
@@ -41,6 +41,8 @@ typedef struct {
ts_allocate_ctor ctor;
ts_allocate_dtor dtor;
ptrdiff_t fast_offset;
+ /* When set, storage comes from __thread memory instead of being allocated by TSRM. */
+ void *(*tls_addr)(void);
int done;
} tsrm_resource_type;
@@ -164,14 +166,20 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb
static void ts_free_resources(tsrm_tls_entry *thread_resources)
{
+ bool own_thread = thread_resources->thread_id == tsrm_thread_id();
+
/* Need to destroy in reverse order to respect dependencies. */
for (int i = thread_resources->count - 1; i >= 0; i--) {
if (!resource_types_table[i].done) {
+ /* A __thread block of a foreign thread is inaccessible. */
+ if (resource_types_table[i].tls_addr && !own_thread) {
+ continue;
+ }
if (resource_types_table[i].dtor) {
resource_types_table[i].dtor(thread_resources->storage[i]);
}
- if (!resource_types_table[i].fast_offset) {
+ if (!resource_types_table[i].fast_offset && !resource_types_table[i].tls_addr) {
free(thread_resources->storage[i]);
}
}
@@ -180,7 +188,9 @@ static void ts_free_resources(tsrm_tls_entry *thread_resources)
free(thread_resources->storage);
}
-/* Shutdown TSRM (call once for the entire process) */
+/* Shutdown TSRM (call once for the entire process). Tears down every thread left
+ * in the table. For resources allocated with ts_allocate_tls_id(), only the dtor
+ * of the calling thread is invoked. */
TSRM_API void tsrm_shutdown(void)
{/*{{{*/
if (is_thread_shutdown) {
@@ -258,7 +268,10 @@ static void tsrm_update_active_threads(void)
p->storage = (void *) realloc(p->storage, sizeof(void *)*id_count);
for (j=p->count; j<id_count; j++) {
- if (resource_types_table[j].fast_offset) {
+ if (resource_types_table[j].tls_addr) {
+ TSRM_ASSERT(p->thread_id == tsrm_thread_id());
+ p->storage[j] = resource_types_table[j].tls_addr();
+ } else if (resource_types_table[j].fast_offset) {
p->storage[j] = (void *) (((char*)p) + resource_types_table[j].fast_offset);
} else {
p->storage[j] = (void *) malloc(resource_types_table[j].size);
@@ -303,6 +316,7 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = 0;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = NULL;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
tsrm_update_active_threads();
@@ -381,6 +395,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset,
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = *offset;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = NULL;
resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
tsrm_update_active_threads();
@@ -390,6 +405,41 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset,
return *rsrc_id;
}
+/* allocates a resource id whose per-thread storage is a native __thread block */
+TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
+{
+ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new TLS resource id, %d bytes", size));
+
+ tsrm_mutex_lock(tsmm_mutex);
+
+ *rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++);
+
+ if (resource_types_table_size < id_count) {
+ tsrm_resource_type *_tmp;
+ _tmp = (tsrm_resource_type *) realloc(resource_types_table, sizeof(tsrm_resource_type)*id_count);
+ if (!_tmp) {
+ TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate storage for resource"));
+ *rsrc_id = 0;
+ tsrm_mutex_unlock(tsmm_mutex);
+ return 0;
+ }
+ resource_types_table = _tmp;
+ resource_types_table_size = id_count;
+ }
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].size = size;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = 0;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = tls_addr;
+ resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
+
+ tsrm_update_active_threads();
+ tsrm_mutex_unlock(tsmm_mutex);
+
+ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new TLS resource id %d", *rsrc_id));
+ return *rsrc_id;
+}
+
static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource)
{
tsrm_tls_set(thread_resource);
@@ -422,7 +472,9 @@ static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_
if (resource_types_table[i].done) {
(*thread_resources_ptr)->storage[i] = NULL;
} else {
- if (resource_types_table[i].fast_offset) {
+ if (resource_types_table[i].tls_addr) {
+ (*thread_resources_ptr)->storage[i] = resource_types_table[i].tls_addr();
+ } else if (resource_types_table[i].fast_offset) {
(*thread_resources_ptr)->storage[i] = (void *) (((char*)(*thread_resources_ptr)) + resource_types_table[i].fast_offset);
} else {
(*thread_resources_ptr)->storage[i] = (void *) malloc(resource_types_table[i].size);
@@ -510,7 +562,9 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id)
/* In case that extensions don't use the pointer passed from the dtor, but incorrectly
* use the global pointer, we need to setup the global pointer temporarily here. */
set_thread_local_storage_resource_to(thread_resources);
- /* Free up the old resource from the old thread instance */
+ /* Dead thread with a recycled id: its __thread blocks are gone, and this
+ * thread's blocks were never constructed, so keep tls dtors from running. */
+ thread_resources->thread_id = 0;
ts_free_resources(thread_resources);
free((char *) thread_resources - tsrm_reserved_front);
/* Allocate a new resource at the same point in the linked list, and relink the next pointer */
@@ -584,7 +638,7 @@ void ts_free_id(ts_rsrc_id id)
if (resource_types_table[rsrc_id].dtor) {
resource_types_table[rsrc_id].dtor(p->storage[rsrc_id]);
}
- if (!resource_types_table[rsrc_id].fast_offset) {
+ if (!resource_types_table[rsrc_id].fast_offset && !resource_types_table[rsrc_id].tls_addr) {
free(p->storage[rsrc_id]);
}
}
diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h
index 2e8cbddfcda..752ea1803f9 100644
--- a/TSRM/TSRM.h
+++ b/TSRM/TSRM.h
@@ -100,6 +100,10 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz
TSRM_API void tsrm_reserve_fast_front(size_t size);
TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
+/* Resource whose per-thread storage is a native __thread block.
+ * Must be called at startup before any other thread exists. */
+TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
+
/* fetches the requested resource for the current thread */
TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id);
#define ts_resource(id) ts_resource_ex(id, NULL)
diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS
index 7775c931604..462e567f3f3 100644
--- a/UPGRADING.INTERNALS
+++ b/UPGRADING.INTERNALS
@@ -235,3 +235,11 @@ PHP 8.6 INTERNALS UPGRADE NOTES
========================
5. SAPI changes
========================
+
+- SAPIs should explicitly release a thread's resources by calling
+ ts_free_thread() before terminating it. tsrm_shutdown() can only release the
+ resources of the calling thread, for resources allocated with
+ ts_allocate_tls_id().
+
+- AG and SCNG are now allocated with ts_allocate_tls_id() and live in native
+ __thread storage on ZTS builds.
diff --git a/Zend/zend.c b/Zend/zend.c
index 9411b92a201..07692db8519 100644
--- a/Zend/zend.c
+++ b/Zend/zend.c
@@ -54,6 +54,9 @@ ZEND_API int compiler_globals_id;
ZEND_API int executor_globals_id;
ZEND_API size_t compiler_globals_offset;
ZEND_API size_t executor_globals_offset;
+/* ts_allocate_tls_id takes a callback so each thread resolves its own block.
+ * A plain &language_scanner_globals would capture only the registering thread's address. */
+static void *language_scanner_globals_tls_addr(void) { return &language_scanner_globals; }
static HashTable *global_function_table = NULL;
static HashTable *global_class_table = NULL;
static HashTable *global_constants_table = NULL;
@@ -1021,10 +1024,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */
#ifdef ZTS
ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
- ts_allocate_fast_id_at(&language_scanner_globals_id, &language_scanner_globals_offset, ZEND_SCNG_OFFSET, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
+ ts_allocate_tls_id(&language_scanner_globals_id, language_scanner_globals_tls_addr, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET);
ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET);
- ZEND_ASSERT(language_scanner_globals_offset == ZEND_SCNG_OFFSET);
ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL);
compiler_globals = ts_resource(compiler_globals_id);
executor_globals = ts_resource(executor_globals_id);
diff --git a/Zend/zend_alloc.c b/Zend/zend_alloc.c
index 5ff2860d822..fc7bc1f4d9d 100644
--- a/Zend/zend_alloc.c
+++ b/Zend/zend_alloc.c
@@ -2613,13 +2613,12 @@ typedef struct _zend_alloc_globals {
#ifdef ZTS
static int alloc_globals_id;
-static size_t alloc_globals_offset;
-# define ZEND_AG_OFFSET (ZEND_SCNG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_alloc_globals)))
-# define AG(v) ZEND_TSRMG_FAST(ZEND_AG_OFFSET, zend_alloc_globals *, v)
+static TSRM_TLS TSRM_TLS_MODEL_ATTR zend_alloc_globals alloc_globals;
+static void *alloc_globals_tls_addr(void) { return &alloc_globals; }
#else
-# define AG(v) (alloc_globals.v)
static zend_alloc_globals alloc_globals;
#endif
+#define AG(v) (alloc_globals.v)
ZEND_API bool is_zend_mm(void)
{
@@ -3336,8 +3335,7 @@ ZEND_API void start_memory_manager(void)
# endif
#endif
#ifdef ZTS
- ts_allocate_fast_id_at(&alloc_globals_id, &alloc_globals_offset, ZEND_AG_OFFSET, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
- ZEND_ASSERT(alloc_globals_offset == ZEND_AG_OFFSET);
+ ts_allocate_tls_id(&alloc_globals_id, alloc_globals_tls_addr, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
#else
alloc_globals_ctor(&alloc_globals);
#endif
@@ -3583,9 +3581,3 @@ ZEND_API char * __zend_strdup(const char *s)
zend_out_of_memory();
}
-#ifdef ZTS
-size_t zend_mm_globals_size(void)
-{
- return sizeof(zend_alloc_globals);
-}
-#endif
diff --git a/Zend/zend_alloc.h b/Zend/zend_alloc.h
index ff51c4fe865..7e3ea031dc7 100644
--- a/Zend/zend_alloc.h
+++ b/Zend/zend_alloc.h
@@ -403,10 +403,6 @@ static void apc_init_heap(void)
*/
-#ifdef ZTS
-size_t zend_mm_globals_size(void);
-#endif
-
END_EXTERN_C()
#endif
diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h
index 61499c0cc23..f78567cfaa7 100644
--- a/Zend/zend_globals.h
+++ b/Zend/zend_globals.h
@@ -330,11 +330,9 @@ struct _zend_executor_globals {
};
#ifdef ZTS
-/* Compile-time offsets of the hot globals, in a reserved region just before the
- * cache pointer. ZEND_AG_OFFSET is furthest, in zend_alloc.c. */
+/* Compile-time offsets of the hot globals, in a reserved region just before *_tsrm_ls_cache. */
# define ZEND_CG_OFFSET (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)))
# define ZEND_EG_OFFSET (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)))
-# define ZEND_SCNG_OFFSET (ZEND_EG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)))
#endif
#define EG_FLAGS_INITIAL (0)
diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h
index 2d2948e50a8..adb3913ece7 100644
--- a/Zend/zend_globals_macros.h
+++ b/Zend/zend_globals_macros.h
@@ -48,12 +48,20 @@ extern ZEND_API zend_executor_globals executor_globals;
/* Language Scanner */
#ifdef ZTS
-# define LANG_SCNG(v) ZEND_TSRMG_FAST(ZEND_SCNG_OFFSET, zend_php_scanner_globals *, v)
extern ZEND_API ts_rsrc_id language_scanner_globals_id;
-extern ZEND_API size_t language_scanner_globals_offset;
+# if defined(ZEND_WIN32) && !defined(LIBZEND_EXPORTS)
+# define LANG_SCNG(v) TSRMG(language_scanner_globals_id, zend_php_scanner_globals *, v)
+# else
+# ifdef ZEND_WIN32
+extern TSRM_TLS zend_php_scanner_globals language_scanner_globals;
+# else
+extern ZEND_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals;
+# endif
+# define LANG_SCNG(v) (language_scanner_globals.v)
+# endif
#else
-# define LANG_SCNG(v) (language_scanner_globals.v)
extern ZEND_API zend_php_scanner_globals language_scanner_globals;
+# define LANG_SCNG(v) (language_scanner_globals.v)
#endif
diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l
index 07f2d44cb5c..1d7e41a2a7d 100644
--- a/Zend/zend_language_scanner.l
+++ b/Zend/zend_language_scanner.l
@@ -84,7 +84,11 @@
#define SCNG LANG_SCNG
#ifdef ZTS
ZEND_API ts_rsrc_id language_scanner_globals_id;
-ZEND_API size_t language_scanner_globals_offset;
+# ifdef ZEND_WIN32
+TSRM_TLS zend_php_scanner_globals language_scanner_globals;
+# else
+ZEND_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals;
+# endif
#else
ZEND_API zend_php_scanner_globals language_scanner_globals;
#endif
diff --git a/main/main.c b/main/main.c
index 48e4a757513..9e77c99a45d 100644
--- a/main/main.c
+++ b/main/main.c
@@ -2819,13 +2819,11 @@ PHPAPI void php_reserve_tsrm_memory(void)
tsrm_reserve(
TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) +
TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) +
- TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) +
TSRM_ALIGNED_SIZE(sizeof(zend_ini_scanner_globals)) +
TSRM_ALIGNED_SIZE(sizeof(virtual_cwd_globals)) +
#ifdef ZEND_SIGNALS
TSRM_ALIGNED_SIZE(sizeof(zend_signal_globals_t)) +
#endif
- TSRM_ALIGNED_SIZE(zend_mm_globals_size()) +
TSRM_ALIGNED_SIZE(zend_gc_globals_size()) +
TSRM_ALIGNED_SIZE(sizeof(php_core_globals)) +
TSRM_ALIGNED_SIZE(sizeof(sapi_globals_struct)) +
@@ -2845,9 +2843,7 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads)
/* Must cover the total size of every ZEND_*_OFFSET global, or the furthest underflows the block. */
tsrm_reserve_fast_front(
TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) +
- TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) +
- TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) +
- TSRM_ALIGNED_SIZE(zend_mm_globals_size())); // AG size, exposed through function call
+ TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)));
(void)ts_resource(0);
return ret;
}