Commit f785614f516 for php.net
commit f785614f5164ee21128a47760d16bdb377052cc6
Author: henderkes <m@pyc.ac>
Date: Fri Jun 12 16:03:43 2026 +0700
TSRM: make CG, EG, SCNG and AG compile-time offsets
Saves an offset load every time they're accessed
Closes GH-22287
diff --git a/NEWS b/NEWS
index a4e0e3ba4f0..91e7169f98f 100644
--- a/NEWS
+++ b/NEWS
@@ -36,6 +36,7 @@ PHP NEWS
. perf: make all static extensions use TSRMG_STATIC. (henderkes)
. Fixed bug GH-22257 (type confusion in Exception::getTraceAsString()).
(David Carlier)
+ . TSRM: make CG, EG, SCNG and AG compile-time offsets. (henderkes)
- BCMath:
. Added NUL-byte validation to BCMath functions. (jorgsowa)
diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c
index e99993204b6..4222e88755d 100644
--- a/TSRM/TSRM.c
+++ b/TSRM/TSRM.c
@@ -36,12 +36,11 @@ struct _tsrm_tls_entry {
tsrm_tls_entry *next;
};
-
typedef struct {
size_t size;
ts_allocate_ctor ctor;
ts_allocate_dtor dtor;
- size_t fast_offset;
+ ptrdiff_t fast_offset;
int done;
} tsrm_resource_type;
@@ -58,6 +57,7 @@ static int resource_types_table_size;
/* Reserved space for fast globals access */
static size_t tsrm_reserved_pos = 0;
static size_t tsrm_reserved_size = 0;
+static size_t tsrm_reserved_front = 0;
static MUTEX_T tsmm_mutex; /* thread-safe memory manager mutex */
static MUTEX_T tsrm_env_mutex; /* tsrm environ mutex */
@@ -155,6 +155,7 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb
tsrm_reserved_pos = 0;
tsrm_reserved_size = 0;
+ tsrm_reserved_front = 0;
tsrm_env_mutex = tsrm_mutex_alloc();
@@ -205,7 +206,7 @@ TSRM_API void tsrm_shutdown(void)
} else {
free(p->storage);
}
- free(p);
+ free((char *) p - tsrm_reserved_front);
p = next_p;
}
}
@@ -232,6 +233,7 @@ TSRM_API void tsrm_shutdown(void)
tsrm_reserved_pos = 0;
tsrm_reserved_size = 0;
+ tsrm_reserved_front = 0;
}/*}}}*/
/* {{{ */
@@ -319,17 +321,20 @@ TSRM_API void tsrm_reserve(size_t size)
}/*}}}*/
+/* Carve a fixed-offset front region out of the reserved space. It is placed
+ * before the TLS entry, so the hot globals get compile-time-constant negative
+ * offsets from the cache pointer. */
+TSRM_API void tsrm_reserve_fast_front(size_t size)
+{
+ tsrm_reserved_front = TSRM_ALIGNED_SIZE(size);
+ tsrm_reserved_size -= tsrm_reserved_front;
+}
+
+
/* allocates a new fast thread-safe-resource id */
TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
{/*{{{*/
- TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size));
-
tsrm_mutex_lock(tsmm_mutex);
-
- /* obtain a resource id */
- *rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++);
- TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtained resource id %d", *rsrc_id));
-
size = TSRM_ALIGNED_SIZE(size);
if (tsrm_reserved_size - tsrm_reserved_pos < size) {
TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate space for fast resource"));
@@ -338,9 +343,26 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz
tsrm_mutex_unlock(tsmm_mutex);
return 0;
}
-
- *offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos;
+ ptrdiff_t fixed_offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos;
tsrm_reserved_pos += size;
+ tsrm_mutex_unlock(tsmm_mutex);
+
+ return ts_allocate_fast_id_at(rsrc_id, offset, fixed_offset, size, ctor, dtor);
+}/*}}}*/
+
+
+TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
+{
+ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size));
+
+ tsrm_mutex_lock(tsmm_mutex);
+
+ /* obtain a resource id */
+ *rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++);
+ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtained resource id %d", *rsrc_id));
+
+ size = TSRM_ALIGNED_SIZE(size);
+ *offset = (size_t) fixed_offset;
/* store the new resource type in the resource sizes table */
if (resource_types_table_size < id_count) {
@@ -366,7 +388,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new resource id %d", *rsrc_id));
return *rsrc_id;
-}/*}}}*/
+}
static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource)
{
@@ -378,7 +400,10 @@ static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource
static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_T thread_id)
{/*{{{*/
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Creating data structures for thread %x", thread_id));
- (*thread_resources_ptr) = (tsrm_tls_entry *) malloc(TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size);
+ /* The entry follows the fixed-offset front region.
+ * hot globals live at negative offsets from the TLS cache pointer. */
+ char *block = (char *) malloc(tsrm_reserved_front + TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size);
+ (*thread_resources_ptr) = (tsrm_tls_entry *) (block + tsrm_reserved_front);
(*thread_resources_ptr)->storage = NULL;
if (id_count > 0) {
(*thread_resources_ptr)->storage = (void **) malloc(sizeof(void *)*id_count);
@@ -487,7 +512,7 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id)
set_thread_local_storage_resource_to(thread_resources);
/* Free up the old resource from the old thread instance */
ts_free_resources(thread_resources);
- free(thread_resources);
+ free((char *) thread_resources - tsrm_reserved_front);
/* Allocate a new resource at the same point in the linked list, and relink the next pointer */
allocate_new_resource(last_thread_resources, thread_id);
thread_resources = *last_thread_resources;
@@ -529,7 +554,7 @@ void ts_free_thread(void)
tsrm_tls_table[hash_value] = thread_resources->next;
}
tsrm_tls_set(0);
- free(thread_resources);
+ free((char *) thread_resources - tsrm_reserved_front);
break;
}
if (thread_resources->next) {
diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h
index 639b1134ddd..2e8cbddfcda 100644
--- a/TSRM/TSRM.h
+++ b/TSRM/TSRM.h
@@ -20,6 +20,7 @@
# include <main/php_config.h>
#endif
+#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
@@ -94,6 +95,11 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate
TSRM_API void tsrm_reserve(size_t size);
TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
+/* Fast resources at caller-chosen, compile-time-constant offsets. The fixed
+ * front region must be reserved after tsrm_reserve() and before any fast id. */
+TSRM_API void tsrm_reserve_fast_front(size_t size);
+TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
+
/* fetches the requested resource for the current thread */
TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id);
#define ts_resource(id) ts_resource_ex(id, NULL)
diff --git a/Zend/zend.c b/Zend/zend.c
index f16b1a30dbb..9411b92a201 100644
--- a/Zend/zend.c
+++ b/Zend/zend.c
@@ -1019,9 +1019,12 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */
zend_init_rsrc_list_dtors();
#ifdef ZTS
- ts_allocate_fast_id(&compiler_globals_id, &compiler_globals_offset, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
- ts_allocate_fast_id(&executor_globals_id, &executor_globals_offset, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
- ts_allocate_fast_id(&language_scanner_globals_id, &language_scanner_globals_offset, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
+ ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
+ ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
+ ts_allocate_fast_id_at(&language_scanner_globals_id, &language_scanner_globals_offset, ZEND_SCNG_OFFSET, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
+ ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET);
+ ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET);
+ ZEND_ASSERT(language_scanner_globals_offset == ZEND_SCNG_OFFSET);
ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL);
compiler_globals = ts_resource(compiler_globals_id);
executor_globals = ts_resource(executor_globals_id);
diff --git a/Zend/zend_alloc.c b/Zend/zend_alloc.c
index 942a8b8e130..d0f2b221b9a 100644
--- a/Zend/zend_alloc.c
+++ b/Zend/zend_alloc.c
@@ -2614,7 +2614,8 @@ typedef struct _zend_alloc_globals {
#ifdef ZTS
static int alloc_globals_id;
static size_t alloc_globals_offset;
-# define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
+# define ZEND_AG_OFFSET (ZEND_SCNG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_alloc_globals)))
+# define AG(v) ZEND_TSRMG_FAST(ZEND_AG_OFFSET, zend_alloc_globals *, v)
#else
# define AG(v) (alloc_globals.v)
static zend_alloc_globals alloc_globals;
@@ -3335,7 +3336,8 @@ ZEND_API void start_memory_manager(void)
# endif
#endif
#ifdef ZTS
- ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
+ ts_allocate_fast_id_at(&alloc_globals_id, &alloc_globals_offset, ZEND_AG_OFFSET, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
+ ZEND_ASSERT(alloc_globals_offset == ZEND_AG_OFFSET);
#else
alloc_globals_ctor(&alloc_globals);
#endif
diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h
index 8257df32e83..61499c0cc23 100644
--- a/Zend/zend_globals.h
+++ b/Zend/zend_globals.h
@@ -329,6 +329,14 @@ struct _zend_executor_globals {
void *reserved[ZEND_MAX_RESERVED_RESOURCES];
};
+#ifdef ZTS
+/* Compile-time offsets of the hot globals, in a reserved region just before the
+ * cache pointer. ZEND_AG_OFFSET is furthest, in zend_alloc.c. */
+# define ZEND_CG_OFFSET (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)))
+# define ZEND_EG_OFFSET (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)))
+# define ZEND_SCNG_OFFSET (ZEND_EG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)))
+#endif
+
#define EG_FLAGS_INITIAL (0)
#define EG_FLAGS_IN_SHUTDOWN (1<<0)
#define EG_FLAGS_OBJECT_STORE_NO_REUSE (1<<1)
diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h
index bde10a0989d..2d2948e50a8 100644
--- a/Zend/zend_globals_macros.h
+++ b/Zend/zend_globals_macros.h
@@ -30,7 +30,7 @@ BEGIN_EXTERN_C()
/* Compiler */
#ifdef ZTS
-# define CG(v) ZEND_TSRMG_FAST(compiler_globals_offset, zend_compiler_globals *, v)
+# define CG(v) ZEND_TSRMG_FAST(ZEND_CG_OFFSET, zend_compiler_globals *, v)
#else
# define CG(v) (compiler_globals.v)
extern ZEND_API struct _zend_compiler_globals compiler_globals;
@@ -40,7 +40,7 @@ ZEND_API int zendparse(void);
/* Executor */
#ifdef ZTS
-# define EG(v) ZEND_TSRMG_FAST(executor_globals_offset, zend_executor_globals *, v)
+# define EG(v) ZEND_TSRMG_FAST(ZEND_EG_OFFSET, zend_executor_globals *, v)
#else
# define EG(v) (executor_globals.v)
extern ZEND_API zend_executor_globals executor_globals;
@@ -48,7 +48,7 @@ extern ZEND_API zend_executor_globals executor_globals;
/* Language Scanner */
#ifdef ZTS
-# define LANG_SCNG(v) ZEND_TSRMG_FAST(language_scanner_globals_offset, zend_php_scanner_globals *, v)
+# define LANG_SCNG(v) ZEND_TSRMG_FAST(ZEND_SCNG_OFFSET, zend_php_scanner_globals *, v)
extern ZEND_API ts_rsrc_id language_scanner_globals_id;
extern ZEND_API size_t language_scanner_globals_offset;
#else
diff --git a/main/main.c b/main/main.c
index afb9fd410e8..48e4a757513 100644
--- a/main/main.c
+++ b/main/main.c
@@ -2842,6 +2842,12 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads)
{
bool ret = tsrm_startup(expected_threads, 1, 0, NULL);
php_reserve_tsrm_memory();
+ /* Must cover the total size of every ZEND_*_OFFSET global, or the furthest underflows the block. */
+ tsrm_reserve_fast_front(
+ TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) +
+ TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) +
+ TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) +
+ TSRM_ALIGNED_SIZE(zend_mm_globals_size())); // AG size, exposed through function call
(void)ts_resource(0);
return ret;
}