Commit 8a94bb3822b for php
commit 8a94bb3822b9913926b64078b2dfec5e8092ba1b
Merge: 294f50b527c c1a5f468d09
Author: Ilija Tovilo <ilija.tovilo@me.com>
Date: Wed Oct 21 16:27:03 2026 +0200
Merge branch 'PHP-8.3' into PHP-8.4
* PHP-8.3:
Add NEWS entry
Avoid state corruption via bailout
diff --cc NEWS
index 48afd654b2e,91bd9fa4a64..12d15e699f1
--- a/NEWS
+++ b/NEWS
@@@ -1,429 -1,23 +1,432 @@@
PHP NEWS
|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
-?? ??? ????, PHP 8.3.37
+?? ??? ????, PHP 8.4.28
+- Core:
+ . Fixed bug GH-24184 (System ID not being reinitialized after an Apache
+ graceful restart, potentially causing incompatible OPcache file cache
+ entries to be reused). (Levi Morrison)
+ . Fixed memory leak when an exception already contains the pending exception
+ in its previous chain. (Edmond)
+ . Fixed bug GH-23979 (Nullsafe operator must not flush delayed oplines of an
+ enclosing function). (ndossche)
+ . Fixed memory manager keeping a block reallocated to exactly the size of
+ the next smaller bin in its larger bin, which efree_size() then freed
+ into the wrong one. (Marc Bennewitz)
+ . Fixed bug GH-24218 (Invalid opcode for count() of a literal array
+ without SCCP). (lazerg)
+ . Fix throwing behaviour with recursive arrays in VM. (ndossche)
+ . Fixed skipped catch block when switching fibers during exception unwinding
+ (Nicolas Grekas)
+ . Fix escape analysis for objects implementing ArrayAccess. (ndossche, Arnaud)
+ . Fix UAF for resolved callable zval. (EdmondDantes)
+ . Fix memory leak for nested HTTP calls. (EdmondDantes)
+ . Fixed pthread attribute leak in zend_call_stack_get_netbsd_pthread().
+ (David Carlier)
+ . Fixed bug GH-24321 (ZTS: global_map_ptr_last survives zend_shutdown(),
+ so an in-process engine restart keeps growing the map_ptr table).
+ (David Carlier)
+
+- Date:
+ . Fixed bug GH-20503 (Assertion failure with ext/date DateInterval property
+ hash construction). (heathdutton)
+ . Fixed bug GH-24350 (MINIT reads a stale external tz database after an
+ in-process module restart). (Lazizbek Ergashev)
+
+- DOM:
+ . Fixed bug GH-23352 (UAF reading an attribute value node retained across
+ DOMDocument::adoptNode()). (David Carlier)
+ . Fixed bug GH-24260 (querySelectorAll() returns duplicate elements for
+ selector lists). (fivefilters)
+ . Fixed bug GH-23334 (Use-after-free in php_dom_free_templated_content()).
+ (David Carlier, ndossche)
+
+- FPM:
+ . Fixed bug GH-24229 (fcgi_close() passes shutdown() a literal 1, which is
+ SHUT_RD in WASI). (Ryosuke Ishibashi)
+
+- Fileinfo:
+ . Fixed bug GH-24289 (finfo::file() reports the wrong argument number for
+ an invalid $filename). (Lazizbek Ergashev)
+
+- FTP:
+ . Fix double free in ext/ftp data_accept() on accept failure. (alepacheco)
+
+- MBString:
+ . Fixed bug GH-23958 (truncated encoding names resolving to full names). (Kamil Tekiela)
+
+- MySQLnd:
+ . Fixed bug GH-24293 (Segfault on connect with an empty password to a
+ MariaDB PAM account). (lazerg)
+
+- Opcache:
+ . Fixed bug GH-20890 (Segfault in zval_undefined_cv with non-simple property
+ hook with minimal tracing JIT). (ndossche)
+ . Fixed bug GH-17626 (JIT corrupts an opline handler when blacklisting a
+ root trace at the opcache.jit_max_root_traces limit, causing spurious
+ "Too few arguments" errors and crashes). (RV7PR)
+ . Fix type inference of ADD_ARRAY_UNPACK with integer keys. (ndossche)
+ . Fix too wide type inference for ASSIGN_DIM_OP. (ndossche)
+ . Fix OSS-Fuzz #568005340 (FETCH_DIM_FUNC_ARG partial conversion). (ndossche)
+ . Fixed bug GH-24218 (Invalid opcode for count() of a literal array when
+ SCCP is disabled). (Lazizbek Ergashev)
+ . Fixed bug GH-24337 (JIT: segfault when decrementing a non-numeric string
+ property). (Lazizbek Ergashev)
+ . Fix SIGFPE on LONG_MIN % -1 with a constant divisor. (waroad)
+ . Fixed bug GH-24388 (Tracing JIT: segfault when a trace runs out of side
+ exits). (Lazizbek Ergashev)
+ . Fixed bug GH-24391 (JIT switch on constant null takes default instead of
+ case 0). (ndossche)
+ . Fixed bug GH-24206 (Tracing JIT loses loop-carried CV when its loop PHI
+ is reloaded from memory). (ndossche)
+ . Fixed bug GH-24386 (JIT: register operand must be loaded before the typed
+ reference check). (ndossche)
+ . Fixed bug GH-24437 (JIT: Fix OP_DATA read from an unwritten slot in
+ zend_jit_assign_dim_op()). (waroad)
+
+- PDO:
+ . Fixed invalid ATTR_ORACLE_NULLS values being accepted and truncated.
+ (Ilia Alshanetsky)
+ . Fixed a duplicate warning when ATTR_STATEMENT_CLASS is rejected on a
+ persistent connection. (Ilia Alshanetsky)
+
+- Phpdbg:
+ . Fix wrong command names. (ndossche, Weilin Du)
+
+- Readline:
+ . Fixed a crash in readline_info() when completion_append_character
+ contains a non-ASCII byte. (Weilin Du)
+
+- SOAP:
+ . Fixed use of uninitialized func in do_request() on OOM bailout.
+ (David Carlier)
+
+- Session:
+ . Fixed session data loss in the files save handler when writing the session
+ file fails after it was already truncated. (Ilia Alshanetsky)
+
+- Standard:
+ . Fixed bug GH-24399 (usleep() and time_nanosleep() ignore queued APCs on
+ Windows). (Nicolas Grekas)
+ . Fixed chown() and lchown() failing to resolve user names in ZTS builds
+ when getpwnam_r() needs a larger buffer. (Ilia Alshanetsky)
+ . Fixed password_get_info() and password_needs_rehash() accepting malformed
+ bcrypt costs. (Ilia Alshanetsky)
+ . Fixed proc_open() null descriptor not being inheritable on Windows.
+ (Jorg Sowa)
+ . Don't share the unserialize() context with enum autoloaders.
+ (Nicolas Grekas)
+- Tidy:
+ . Fixed tidy_repair_string(), tidy_repair_file(), tidy::repairString() and
+ tidy::repairFile() rejecting null for $encoding. (lacatoire)
+
+- Zip:
+ . Fixed use-after-free when re-entering ZipArchive during destruction or
+ a close warning, and rejected opening streams while closing. (jvoisin)
+ . Fixed a use-after-free when a ZipArchive method that modifies the archive
+ is called from a progress or cancel callback during close().
+ (Ilia Alshanetsky)
-21 Oct 2026, PHP 8.3.36
+- Zlib:
+ . Fixed use-after-free when an error handler removes a failing zlib.inflate
+ filter (GH-24176). (Edmond)
+
+22 Oct 2026, PHP 8.4.27
- - BCMath:
- . Fixed BcMath\Number results that truncate to zero keeping a negative sign
- and comparing less than zero. (Ilia Alshanetsky)
-
- - BZ2:
- . Fixed double close of the descriptor and use-after-free of the inner
- stream when bzopen() is given a stream resource. (Jakub Zelenka)
-
- - CLI
- . Fix GH-22567 (Windows ZTS CLI SAPI should refresh its TSRMLS cache during
- request activation). (matyhtf)
- . Fixed bug GH-23764 (Built-in server leaks a file descriptor on every HEAD
- request for a static file). (jakubskopal)
- . Fixed crash in the built-in server when a client is reset before being
- accepted. (David Carlier)
- . Fixed the built-in server truncating an error page and leaking its
- unsent part after a partial socket write. (Ilia Alshanetsky)
-
- - Core
+ - Core:
+ . Fixed GHSA-w8hw-w9fx-j79w, GHSA-6wmf-4rv5-p3rp (Avoid state corruption via
+ bailout). (CVE-2026-109618, CVE-2026-109619) (Arnaud, ilutov, Jakub Zelenka,
+ ndossche, Nicolas Grekas)
+ . Fix GH-21999: GC inconsistency with lazy object, var_dump(), and object
+ comparison. (Arnaud)
+ . Fixed OSS-Fuzz #536440507 (Immutable class incorrect assertion).
+ (ndossche)
+ . Fixed AVX being reported as supported when the OS has not enabled AVX
+ state. (Ilia Alshanetsky)
+ . Fixed GH-23980 (ZEND_ASSERT violation @ ZEND_INCLUDE_OR_EVAL). (ndossche)
+ . Fixed bug GH-23896 (Assertion failure in zend_call_function() after a
+ throwing deprecation). (lazerg)
+ . Fixed use-after-free when a dl()-loaded extension declares a frameless
+ function and a later request calls it. (Ilia Alshanetsky)
+ . Fixed System V shared memory emulation for Windows attaching segments past
+ the end of their mapping. (Ilia Alshanetsky)
+ . Fixed bug GH-24050 (GC frees an object still held by a resurrected closure
+ when (object) and use share an array). (David Carlier)
+ . Fixed use-after-free when a Fiber callable resolves to $this of the method
+ creating the Fiber ([A::class, 'm'] or "A::m") and that object dies before
+ the Fiber runs. (Edmond Dantes)
+ . Fixed exception thrown by destructor during GC in a Fiber not being rethrown
+ into the frame that triggered the GC. (Nicolas Grekas)
+ . Fixed bug GH-24139 (NULL pointer dereference in php_ini.c when
+ expand_filepath() fails). (lazerg)
+
++- BCMath:
++ . Fixed BcMath\Number results that truncate to zero keeping a negative sign
++ and comparing less than zero. (Ilia Alshanetsky)
+
-24 Sep 2026, PHP 8.3.35
++- BZ2:
++ . Fixed double close of the descriptor and use-after-free of the inner
++ stream when bzopen() is given a stream resource. (Jakub Zelenka)
+
-- Filter:
- . Fixed GHSA-ch8v-r6jh-4vvr (FILTER_SANITIZE_ENCODED does not encode 0xFF).
++- CLI:
++ . Fix GH-22567 (Windows ZTS CLI SAPI should refresh its TSRMLS cache during
++ request activation). (matyhtf)
++ . Fixed bug GH-23764 (Built-in server leaks a file descriptor on every HEAD
++ request for a static file). (jakubskopal)
++ . Fixed crash in the built-in server when a client is reset before being
++ accepted. (David Carlier)
++ . Fixed the built-in server truncating an error page and leaking its
++ unsent part after a partial socket write. (Ilia Alshanetsky)
++
+- DOM:
+ . Fixed use-after-free when re-constructing a DOMXPath whose php:function
+ registrations are freed while still reachable from the cycle collector.
+ (Ilia Alshanetsky)
+ . Fixed bug GH-23365 (DOMNode::insertBefore($n, $n) drops the node and
+ leaves a self-referencing sibling list). (David Carlier)
+ . Fixed Dom\HTMLDocument::getElementById() not finding ids of SVG and
+ MathML elements. (Ilia Alshanetsky)
+ . Fixed Dom\HTMLDocument giving attributes the namespace of their element
+ when a fragment is parsed with an xlink, xml or xmlns context element.
+ (Ilia Alshanetsky)
+ . Fixed bug GH-23729 (DOMXPath::__construct() use-after-free during an
+ evaluation). (David Carlier)
+ . Fixed bug GH-23897 (php:function() assertion failure after a failed
+ registerPHPFunctions()). (David Carlier)
+ . Fixed DOMXPath and Dom\XPath losing the context node and in-scope
+ namespaces of an evaluation after a php:function() callback evaluated
+ another expression on the same object. (Ilia Alshanetsky)
+
+- FFI:
+ . Fixed crashes with FFI callbacks created from __call() trampolines
+ and array callables whose object is released. (Ilia Alshanetsky)
+
+- FTP:
+ . Fixed bug GH-23619 (cryptic error on servers that don't support TLS
+ session resumption on data connection). (ndossche)
+
+- GD:
+ . Fixed undefined behavior with GD2 images with `INT_MAX`-sized chunks.
+ (DanielEScherzer, cmb)
+ . Fixed undefined behavior with GIFs with problematic LZW compression data.
+ (DanielEScherzer, vapier)
+
+- Intl:
+ . Fixed IntlDateFormatter::parse(), localtime() and parseToCalendar()
+ treating the offset as UTF-16 code units instead of bytes.
+ (Ilia Alshanetsky)
+ . Fixed cloning IntlDateFormatter and MessageFormatter losing PHP-side state
+ such as dateType, timeType, calendar and the message pattern.
+ (Ilia Alshanetsky)
+ . Fixed Collator attribute and strength methods not rejecting an
+ unconstructed Collator. (Ilia Alshanetsky)
+ . Fixed grapheme_strstr() and grapheme_stristr() returning incorrect substrings
+ when supplementary characters precede the match. (Weilin Du)
+ . Fixed grapheme_strstr() and grapheme_strpos() missing canonically
+ equivalent matches, and grapheme searches reporting ICU errors for empty
+ haystacks. (Weilin Du)
+ . Fixed grapheme_strrpos() and grapheme_strripos() skipping overlapping matches
+ when using a negative offset. (Weilin Du)
+ . Fixed grapheme_extract() returning an incorrect next offset when starting
+ inside a multibyte UTF-8 character. (Weilin Du, David Carlier)
+
+- Lexbor:
+ . Merge patches lexbor/lexbor@8a14bc0 and lexbor/lexbor@f67ce4b, fixing a
+ heap buffer overflow in :lexbor-contains() parsing and buffer overflows
+ in malformed decode replay. (alexandre-daubois)
+
+- MBString:
+ . Fixed bug GH-23106 (mb_strpos() reads past the end of a haystack ending in
+ a truncated UTF-8 sequence). (Lazizbek Ergashev)
+ . Fixed mbstring functions emitting surrogates in UTF-8 output and flagging
+ it as valid UTF-8. (Nicolas Grekas)
+
+- MySQLi:
+ . Fix GH-22854: Fixed failed assertion when accessing mysqli property after
+ failed reconnection. (Kamil Tekiela)
+
+- MySQLnd:
+ . Fixed field_count not resetting on OK packet. (Kamil Tekiela)
+ . Fixed memory leak when closing a prepared statement after its connection
+ was killed. (Kamil Tekiela)
+
+- Opcache:
+ . Fixed OSS-Fuzz #546798343 (Heap-buffer-overflow in optimizer with
+ FCCs and inlining). (ndossche)
+ . Fixed bug GH-23693 (Tracing JIT produces wrong results for a guard on a
+ loop-invariant addition). (Ilia Alshanetsky)
+ . Fixed OSS-Fuzz #5674034779193344 (Read of uninitialized memory in
+ is_cacheable_stream_path()). (ndossche)
+ . Fix zend_analyze_calls() call_stack buffer overrun. (Mrmaxmeier)
+ . Fixed bug GH-23637 (PHP-FPM worker SIGSEGV: run_time_cache map_ptr offset
+ beyond CG(map_ptr_last) when a class is reached through the CE cache).
+ (David Carlier)
+ . Fix multiple incorrect DCE due to unsound escape analysis. (ndossche,
+ arnaud-lb)
+ . Fixed bug GH-24063 (OPcache optimizer folds `$cond ? -0.0 : 0.0` into a
+ single `-0.0` constant). (lazerg)
+ . Fixed crash when a file cache is reused with a different set of
+ extensions declaring frameless functions. (Ilia Alshanetsky)
+ . Fixed bug GH-23991 (JIT generates invalid IR for multiple recursive calls).
+ (ndossche)
+ . Fix SCCP evaluation of isset()/empty() on dimension of non-escaping object.
+ (ndossche)
+
+- OpenSSL:
+ . Fixed stream_socket_enable_crypto() leaving the socket non-blocking
+ after a handshake timeout. (Ilia Alshanetsky)
+ . Fixed feof() on a TLS stream staying false after a close_notify on Windows.
+ (Jakub Zelenka)
+
+- PCNTL:
+ . Fixed pcntl_signal_dispatch() dropping the queued signals when it runs while
+ an exception is pending. (nicolas-grekas)
+ . Fixed pcntl_signal_dispatch() dropping the signals queued behind a handler
+ that throws. (nicolas-grekas)
+ . Fixed pcntl_setcpuaffinity() error message advertising the cpu count as
+ the inclusive upper bound. (lacatoire)
+ . Fixed pcntl_setqos_class() requiring its optional $qos_class argument.
+ (lacatoire)
+ . Fixed bug GH-23986 (/proc/self paths resolve to the parent process after
+ pcntl_fork()). (Lazizbek Ergashev)
+
+- PDO:
+ . Fixed PDOStatement::getColumnMeta() reading out of bounds for an invalid
+ column index. (Ilia Alshanetsky)
+ . Fixed PDOStatement::bindColumn() registering a binding for a column name
+ that is not in the result set. (Ilia Alshanetsky)
+ . Fixed PDOStatement::execute() leaving the previous result available
+ after a failed execution. (Ilia Alshanetsky)
+ . Fixed a use-after-free when bindValue()/execute()/closeCursor() is called
+ from a bound parameter's __toString() during execute(). (Ilia Alshanetsky)
+ . Fixed bug GH-23962 (Destroying a persistent PDO instance rolls back a
+ transaction still in use by another instance). (Lazizbek Ergashev)
+ . Fixed PDO::setAttribute() installing a PDO::ATTR_STATEMENT_CLASS class
+ whose constructor arguments it rejects. (Ilia Alshanetsky)
+ . Fixed PDO::ATTR_STATEMENT_CLASS constructor_args type errors reporting
+ "array given" regardless of the value passed. (Ilia Alshanetsky)
+ . Fixed PDOStatement::bindParam() and bindColumn() leaking the driver
+ options value. (Ilia Alshanetsky)
+ . Fixed PDOStatement not reporting its bound parameters and columns to the
+ cycle collector. (Ilia Alshanetsky)
+ . Fixed PDOStatement::setFetchMode() not applying changes atomically.
+ (Ilia Alshanetsky)
+
+- PDO_DBLIB:
+ . Fixed bug GH-23741 (segfault after a failed query inside a PDO
+ transaction). Errors raised by beginTransaction(), commit(), rollBack()
+ and lastInsertId() are now reported instead of being dropped.
+ (Ilia Alshanetsky)
+
+- PDO_Firebird:
+ . Fixed bug GH-23758 (PDO_Firebird returns null for non-null empty BLOBs).
+ (Lazizbek Ergashev)
+
+- Readline:
+ . Fixed a heap over-read in the interactive shell prompt when cli.prompt is
+ set to an empty string. (Ilia Alshanetsky)
+
+- Session:
+ . Fixed session_start() continuing after a failed create_sid() when
+ session.use_strict_mode rejects the supplied ID. (Ilia Alshanetsky)
+ . Fixed exceptions from user-defined create_sid() handlers being replaced
+ by return-value validation errors. (Ilia Alshanetsky)
+
+- SimpleXML:
+ . Fixed reconstructing a SimpleXMLElement freeing a child element that
+ another variable still references. (Ilia Alshanetsky)
+
+- SOAP:
+ . Fixed bug GH-22895 (Heap use-after-free while encoding a Traversable with
+ an illegal key). (David Carlier)
+ . Fixed memory leak when load_wsdl_ex() failed to load. (DanielEScherzer)
+
+- Sockets:
+ . Fixed socket_select() silently truncating sets larger than FD_SETSIZE on
+ Windows. (David Carlier)
+
+- SPL:
+ . Fixed SplFixedArray::setSize() doing nothing on subclasses whose
+ constructor does not call parent::__construct(). (Marc Bennewitz)
+ . Fixed memory leak when __construct(), __wakeup() or __unserialize() is
+ called from an element destructor during setSize(0). (Marc Bennewitz)
+
+- SQLite:
+ . Fixed a crash when SQLite3::close() is called from a userland callback.
(Ilia Alshanetsky)
+- Standard:
+ . Fixed sha1_file() returning a digest for incomplete data after a stream
+ read failure. (Ilia Alshanetsky)
+ . Fixed iptcembed() corrupting JPEG headers when called recursively from an
+ output handler. (Ilia Alshanetsky)
+ . Fixed proc_open() leaking descriptor zero when descriptor setup fails.
+ (Ilia Alshanetsky)
+ . Fixed three Windows-only proc_open() defects: an uninitialized
+ PROCESS_INFORMATION, an indeterminate comspec pointer after a failed
+ lookup, and an unchecked CreateFileA() failure. (Ilia Alshanetsky)
+ . Fix persistent stream context lifetime during shutdown (Levi Morrison)
+ . Fix wordwrap() ignoring an existing break at the end of the string
+ (Nicolas Grekas)
+ . Fixed convert.quoted-printable-decode decoding lowercase hex digits to
+ the wrong byte. (Ilia Alshanetsky)
+ . Fixed unserialize() of enums sharing its state with the unserialize()
+ calls made by autoloaders. (nicolas-grekas)
+
+- Tidy:
+ . Fixed a use-after-free when a tidyNode is used after its document is
+ reparsed. (Ilia Alshanetsky)
+
+- XSL:
+ . Fixed bug GH-23730 (use-after-free when XSLTProcessor::importStylesheet()
+ is called during a transformation). (David Carlier)
+
+- Zip:
+ . Fixed ZipArchive::extractTo() ignoring files given in a non-list array.
+ (David Carlier)
+ . Fixed bug GH-23747 (ZipArchive::close() use-after-free from a progress or
+ cancel callback). (David Carlier)
+
+
+24 Sep 2026, PHP 8.4.26
+
+- BCMath:
+ . Fixed out-of-bounds read in bc_is_zero_for_scale() when scale exceeds
+ n_scale. (Ilia Alshanetsky)
+
+- Core:
+ . Fixed out-of-bounds reads during automatic UTF-16/32 encoding detection.
+ (Yudai Takada)
+ . Fixed bug GH-15375 (Nested "yield from" skips items after a valid() or
+ next() call on the inner generator). (iliaal)
+ . Fixed bug GH-23232 (lone namespace separator asks the autoloader for an
+ empty class name). (spawnia)
+ . Fixed bug GH-23301 (Nested "yield from" yields a value twice when the
+ middle generator delegates again). (Lazizbek Ergashev)
+
+- CLI:
+ . Fixed bug GH-23425 (sapi_cli_server_send_headers() does not check the
+ return value of php_cli_server_client_send_through()). (Lazizbek Ergashev)
+
+- DOM:
+ . Fixed NamedNodeMap::getNamedItemNS() with an empty URI not matching
+ the null namespace in spec-following mode. (Ilia Alshanetsky)
+ . Fixed a use-after-free when cloning a DOMNameSpaceNode after
+ DOMDocument::xinclude(). (iliaal)
+ . Fixed bug GH-23331 (UAF when node_list_unlink() skips attribute children
+ that still have a live wrapper). (iliaal)
+ . Fixed a use-after-free when Dom\Element::setAttributeNS() replaces the
+ value of an attribute whose child still has a live wrapper. (iliaal)
+
+- GD:
+ . Fixed imageaffinematrixget() and imageaffinematrixconcat() reporting the
+ wrong argument in error messages. (Weilin Du)
+ . Fixed bug GH-23457 (imagebmp() is extremely slow when writing to a file).
+ (Lazizbek Ergashev)
+
- FPM:
+ . Fixed bug GH-19320 (FPM UID and GID overflow). (Pratik Bhujel)
. Fixed GHSA-62xp-839h-2637 (IPv6 ACL bypass in FastCGI listen.allowed_clients
due to partial address comparison). (CVE-2026-91768) (Alexandre Daubois)
diff --cc Zend/tests/oom_SplFixedArray___wakeup.phpt
index 00000000000,d644acc5cf9..9df4d6b93a9
mode 000000,100644..100644
--- a/Zend/tests/oom_SplFixedArray___wakeup.phpt
+++ b/Zend/tests/oom_SplFixedArray___wakeup.phpt
@@@ -1,0 -1,19 +1,21 @@@
+ --TEST--
+ OOM SplFixedArray::fromArray() 003
+ --INI--
+ memory_limit=16M
+ --FILE--
+ <?php
+
+ $a = new SplFixedArray();
+ for ($i = 0; $i < 128*1024; $i++) {
+ @$a->{$i} = 0;
+ }
+
+ $pad = str_repeat('a', max(16*1024*1024 - memory_get_usage(true) - 512*1024, 1));
+
+ $a->__wakeup();
+
+ ?>
+ --EXPECTF--
++Deprecated: Method SplFixedArray::__wakeup() is deprecated since 8.4, this method is obsolete, as serialization hooks are provided by __unserialize() and __serialize() in %s on line %d
++
+ Fatal error: Allowed memory size of 16777216 bytes exhausted %S(tried to allocate %d bytes) in %s on line 10
diff --cc Zend/zend.c
index da78dc0c2e8,b60214ed24d..32028c9c5a0
--- a/Zend/zend.c
+++ b/Zend/zend.c
@@@ -737,16 -728,16 +737,18 @@@ static void compiler_globals_ctor(zend_
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
compiler_globals->map_ptr_size = 0;
compiler_globals->map_ptr_last = global_map_ptr_last;
- if (compiler_globals->map_ptr_last) {
+ compiler_globals->internal_run_time_cache = NULL;
+ if (compiler_globals->map_ptr_last || zend_map_ptr_static_size) {
/* Allocate map_ptr table */
compiler_globals->map_ptr_size = ZEND_MM_ALIGNED_SIZE_EX(compiler_globals->map_ptr_last, 4096);
- void *base = pemalloc(compiler_globals->map_ptr_size * sizeof(void*), 1);
+ void *base = pemalloc((zend_map_ptr_static_size + compiler_globals->map_ptr_size) * sizeof(void*), 1);
compiler_globals->map_ptr_real_base = base;
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(base);
- memset(base, 0, compiler_globals->map_ptr_last * sizeof(void*));
+ memset(base, 0, (zend_map_ptr_static_size + compiler_globals->map_ptr_last) * sizeof(void*));
}
+ zend_init_internal_run_time_cache();
+
+ compiler_globals->interned_strings_fallible = false;
}
/* }}} */
@@@ -842,10 -829,7 +844,11 @@@ static void executor_globals_ctor(zend_
executor_globals->pid = 0;
executor_globals->oldact = (struct sigaction){0};
#endif
+ memset(executor_globals->strtod_state.freelist, 0,
+ sizeof(executor_globals->strtod_state.freelist));
+ executor_globals->strtod_state.p5s = NULL;
+ executor_globals->strtod_state.result = NULL;
+ memset(&executor_globals->delayed_oom, 0, sizeof(executor_globals->delayed_oom));
}
/* }}} */
@@@ -1926,35 -1873,6 +1931,38 @@@ ZEND_API ZEND_COLD void zend_user_excep
}
} /* }}} */
+ZEND_API zend_result zend_execute_script(int type, zval *retval, zend_file_handle *file_handle)
+{
+ zend_op_array *op_array = zend_compile_file(file_handle, type);
+ if (file_handle->opened_path) {
+ zend_hash_add_empty_element(&EG(included_files), file_handle->opened_path);
+ }
+
+ zend_result ret = SUCCESS;
+ if (op_array) {
+ zend_execute(op_array, retval);
++ if (ZEND_HAS_OOM()) {
++ zend_handle_delayed_oom();
++ }
+ zend_exception_restore();
+ if (UNEXPECTED(EG(exception))) {
+ if (Z_TYPE(EG(user_exception_handler)) != IS_UNDEF) {
+ zend_user_exception_handler();
+ }
+ if (EG(exception)) {
+ ret = zend_exception_error(EG(exception), E_ERROR);
+ }
+ }
+ zend_destroy_static_vars(op_array);
+ destroy_op_array(op_array);
+ efree_size(op_array, sizeof(zend_op_array));
+ } else if (type == ZEND_REQUIRE) {
+ ret = FAILURE;
+ }
+
+ return ret;
+}
+
ZEND_API zend_result zend_execute_scripts(int type, zval *retval, int file_count, ...) /* {{{ */
{
va_list files;
diff --cc Zend/zend_API.h
index d91da91bf29,dcd90556c2b..4d1b3e486c7
--- a/Zend/zend_API.h
+++ b/Zend/zend_API.h
@@@ -536,9 -530,46 +536,47 @@@ ZEND_API const char *zend_get_type_by_c
#define array_init(arg) ZVAL_ARR((arg), zend_new_array(0))
#define array_init_size(arg, size) ZVAL_ARR((arg), zend_new_array(size))
+ #define array_init_singleton(arg) ZVAL_ARR((arg), zend_new_array_singleton())
+
+ /* Fallible array_init()/object_init(): on failure return false and leave arg as
+ * IS_NULL, so arg stays safe to destroy. */
+ static zend_always_inline bool _array_finit(zval *arg, size_t size)
+ {
+ zend_array *ht = zend_fnew_array(size);
+ if (UNEXPECTED(ht == NULL)) {
+ ZVAL_NULL(arg);
+ return false;
+ }
+ ZVAL_ARR(arg, ht);
+ return true;
+ }
+
+ static zend_always_inline bool _array_finit_ex(zval *arg, size_t size, bool packed)
+ {
+ zend_array *ht = zend_fnew_array(size);
+ if (UNEXPECTED(ht == NULL)) {
+ ZVAL_NULL(arg);
+ return false;
+ }
+ if (UNEXPECTED(zend_hash_real_finit(ht, packed, /* fallible */ true) == FAILURE)) {
+ efree(ht);
+ ZVAL_NULL(arg);
+ return false;
+ }
+ ZVAL_ARR(arg, ht);
+ return true;
+ }
+
+ #define array_finit(arg) _array_finit((arg), 0)
+ #define array_finit_size(arg, size) _array_finit((arg), (size))
+ #define array_finit_size_ex(arg, size, packed) _array_finit_ex((arg), (size), (packed))
+ #define array_finit_size_packed(arg, size) _array_finit_ex((arg), (size), /* packed */ true)
+ #define array_finit_size_mixed(arg, size) _array_finit_ex((arg), (size), /* packed */ false)
+
ZEND_API void object_init(zval *arg);
+ ZEND_API bool object_finit(zval *arg);
ZEND_API zend_result object_init_ex(zval *arg, zend_class_entry *ce);
+ZEND_API zend_result object_init_with_constructor(zval *arg, zend_class_entry *class_type, uint32_t param_count, zval *params, HashTable *named_params);
ZEND_API zend_result object_and_properties_init(zval *arg, zend_class_entry *ce, HashTable *properties);
ZEND_API void object_properties_init(zend_object *object, zend_class_entry *class_type);
ZEND_API void object_properties_init_ex(zend_object *object, HashTable *properties);
diff --cc Zend/zend_alloc.c
index 0b0ecb44e23,85eed43812e..3cbdd1e5f5d
--- a/Zend/zend_alloc.c
+++ b/Zend/zend_alloc.c
@@@ -53,13 -53,14 +53,16 @@@
#include "zend.h"
#include "zend_alloc.h"
+ #include "zend_API.h"
#include "zend_globals.h"
+#include "zend_hrtime.h"
#include "zend_operators.h"
#include "zend_multiply.h"
#include "zend_bitset.h"
#include "zend_mmap.h"
+#include "zend_portability.h"
+ #include "zend_string.h"
+ #include "zend_exceptions.h"
#include <signal.h>
#ifdef HAVE_UNISTD_H
@@@ -217,12 -194,10 +220,16 @@@ typedef zend_mm_bitset zend_mm_page_map
#define ZEND_MM_BINS 30
+ #if ZEND_MM_LIMIT
+ # define ZEND_MM_LIMIT_MAX ((size_t)Z_L(-1) >> 1)
+ #endif
+
+#if UINTPTR_MAX == UINT64_MAX
+# define BSWAPPTR(u) ZEND_BYTES_SWAP64(u)
+#else
+# define BSWAPPTR(u) ZEND_BYTES_SWAP32(u)
+#endif
+
typedef struct _zend_mm_page zend_mm_page;
typedef struct _zend_mm_bin zend_mm_bin;
typedef struct _zend_mm_free_slot zend_mm_free_slot;
@@@ -298,17 -276,21 +309,18 @@@ struct _zend_mm_heap
int last_chunks_delete_boundary; /* number of chunks after last deletion */
int last_chunks_delete_count; /* number of deletion over the last boundary */
#if ZEND_MM_CUSTOM
- union {
- struct {
- void *(*_malloc)(size_t);
- void (*_free)(void*);
- void *(*_realloc)(void*, size_t);
- } std;
- struct {
- void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
- void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
- void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
- } debug;
+ struct {
+ void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ size_t (*_gc)(void);
+ void (*_shutdown)(bool full, bool silent);
} custom_heap;
HashTable *tracked_allocs;
+ uint64_t max_allocs;
#endif
+ pid_t pid;
+ zend_random_bytes_insecure_state rand_state;
};
struct _zend_mm_chunk {
@@@ -1713,11 -1739,13 +1849,14 @@@ static zend_always_inline void *zend_mm
/* Check if requested size fits into current bin */
if (size <= old_size) {
- /* Check if truncation is necessary */
- if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
+ /* Check if truncation is necessary. A size that fits the
+ * bin below moves there, as efree_size() maps it to that bin. */
+ if (old_bin_num > 0 && size <= bin_data_size[old_bin_num - 1]) {
/* truncation */
ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
+ if (UNEXPECTED(ret == NULL)) {
+ goto failed;
+ }
copy_size = use_copy_size ? MIN(size, copy_size) : size;
memcpy(ret, ptr, copy_size);
zend_mm_free_small(heap, ptr, old_bin_num);
@@@ -2054,11 -2066,13 +2188,14 @@@ static zend_mm_heap *zend_mm_init(void
heap->size = 0;
heap->peak = 0;
#endif
+ zend_mm_init_key(heap);
#if ZEND_MM_LIMIT
- heap->limit = (size_t)Z_L(-1) >> 1;
+ heap->limit = ZEND_MM_LIMIT_MAX;
+ heap->hard_limit = ZEND_MM_LIMIT_MAX;
+ heap->hard_limit_percent = -1;
heap->overflow = 0;
#endif
+ heap->fallible = false;
#if ZEND_MM_CUSTOM
heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
#endif
@@@ -2399,10 -2400,11 +2536,11 @@@ ZEND_API void zend_mm_shutdown(zend_mm_
#if ZEND_MM_CUSTOM
if (heap->use_custom_heap) {
- if (heap->custom_heap.std._malloc == tracked_malloc) {
+ if (heap->custom_heap._malloc == tracked_malloc) {
if (silent) {
- tracked_free_all();
+ tracked_free_all(heap);
}
+ heap->max_allocs = 0;
zend_hash_clean(heap->tracked_allocs);
if (full) {
zend_hash_destroy(heap->tracked_allocs);
@@@ -2950,26 -3126,50 +3254,50 @@@ static zend_always_inline zval *tracked
return size_zv;
}
- static zend_always_inline void tracked_check_limit(zend_mm_heap *heap, size_t add_size) {
- #if ZEND_MM_STAT
- if (add_size > heap->limit - heap->size && !heap->overflow) {
+ static zend_always_inline zend_result tracked_check_limit(zend_mm_heap *heap, size_t add_size ZEND_FILE_LINE_DC) {
+ return zend_mm_check_limit(heap, add_size, /* gc */ false, NULL ZEND_FILE_LINE_RELAY_CC);
+ }
+
+ /* ZEND_MM_MAX_ALLOCS fails the n-th allocation of the request in the same way
+ * as the memory limit would, so that the OOM handling can be exercised at an
+ * arbitrary allocation rather than only at the ones large enough to reach the
+ * limit. */
+ static zend_never_inline ZEND_COLD zend_result tracked_max_allocs_error(zend_mm_heap *heap, size_t add_size ZEND_FILE_LINE_DC) {
+ zend_mm_max_allocs_reached = true;
+
+ if (!ZEND_HAS_OOM()) {
+ zend_mm_delayed_error(
#if ZEND_DEBUG
- zend_mm_safe_error(heap,
- "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)",
- heap->limit, "file", 0, add_size);
+ "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)",
#else
- zend_mm_safe_error(heap,
- "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)",
- heap->limit, add_size);
+ "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)",
#endif
+ heap->limit
+ ZEND_FILE_LINE_RELAY_CC,
+ add_size);
}
- #endif
+
+ return heap->fallible ? FAILURE : SUCCESS;
+ }
+
+ static zend_always_inline zend_result tracked_check_max_allocs(zend_mm_heap *heap, size_t add_size ZEND_FILE_LINE_DC) {
+ if (UNEXPECTED(heap->max_allocs && !--heap->max_allocs)) {
+ return tracked_max_allocs_error(heap, add_size ZEND_FILE_LINE_RELAY_CC);
+ }
+
+ return SUCCESS;
}
-static void *tracked_malloc(size_t size)
+static void *tracked_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
{
zend_mm_heap *heap = AG(mm_heap);
- tracked_check_limit(heap, size);
+ if (UNEXPECTED(tracked_check_limit(heap, size ZEND_FILE_LINE_CC) == FAILURE)) {
+ return NULL;
+ }
+
+ if (UNEXPECTED(tracked_check_max_allocs(heap, size ZEND_FILE_LINE_CC) == FAILURE)) {
+ return NULL;
+ }
void *ptr = malloc(size);
if (!ptr) {
@@@ -3260,10 -3454,10 +3607,11 @@@ ZEND_API zend_mm_heap *zend_mm_startup_
heap->size = 0;
heap->peak = 0;
#endif
+ zend_mm_init_key(heap);
#if ZEND_MM_LIMIT
- heap->limit = (size_t)Z_L(-1) >> 1;
+ heap->limit = ZEND_MM_LIMIT_MAX;
heap->overflow = 0;
+ heap->fallible = false;
#endif
#if ZEND_MM_CUSTOM
heap->use_custom_heap = 0;
@@@ -3292,7 -3485,61 +3640,61 @@@
#endif
}
+ ZEND_API void ZEND_FASTCALL zend_handle_delayed_oom(void)
+ {
+ zend_oom_info *info = &EG(delayed_oom);
+ zend_execute_data *execute_data = EG(current_execute_data);
+
+ if (info->state != ZEND_OOM_STATE_PENDING) {
+ return;
+ }
+ info->state = ZEND_OOM_STATE_HANDLING;
+
+ AG(mm_heap)->overflow = 1;
+
+ /* message was allocated persistently, use emalloc() now as error_get_last()
+ * may expose it to userland. */
+ zend_string *message = zend_string_init(ZSTR_VAL(info->message), ZSTR_LEN(info->message), 0);
+ zend_string_release(info->message);
+ info->message = message;
+
+ if (execute_data) {
+ /* An exception may already be in flight, and zend_throw_unwind_exit()
+ * does not allow that */
+ EG(exception) = NULL;
+
+ /* Object allocation should succeed unless it trips the hard limit */
+ zend_throw_unwind_exit();
+ }
+
+ zend_try {
+ zend_error_zstr_at(E_ERROR, info->filename, info->lineno, info->message);
+ } zend_catch {
+ } zend_end_try();
+
+ AG(mm_heap)->overflow = 0;
+
+ /* Bailout discards EG(current_execute_data). Would be better to avoid the bailout here. */
+ EG(current_execute_data) = execute_data;
+
+ /* This function may be called from the VM interrupt handler: opline is not
+ * executed yet. Set opline_before_exception to NULL so
+ * ZEND_HANDLE_EXCEPTION does not use it. */
+ EG(opline_before_exception) = NULL;
+
+ zend_mm_oom_info_dtor(info);
+ info->state = ZEND_OOM_STATE_HANDLED;
+
+ /* No userland frame to unwind: there is nothing to restore afterwards
+ * and no exception to throw, so catching the bailout here would just
+ * let the request carry on past a fatal error. Let it propagate and
+ * abort the request, the way an immediate OOM does. */
+ if (!execute_data) {
+ zend_bailout();
+ }
+ }
+
-ZEND_API void * __zend_malloc(size_t len)
+ZEND_API void * __zend_malloc(size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
{
void *tmp = malloc(len);
if (EXPECTED(tmp || !len)) {
@@@ -3301,13 -3548,14 +3703,14 @@@
zend_out_of_memory();
}
-ZEND_API void * __zend_calloc(size_t nmemb, size_t len)
+ZEND_API void * __zend_calloc(size_t nmemb, size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
{
void *tmp;
+ size_t actual_size;
+ ZEND_MM_SAFE_ADDRESS_GUARDED(actual_size, nmemb, len, 0);
- len = zend_safe_address_guarded(nmemb, len, 0);
- tmp = __zend_malloc(len ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
- memset(tmp, 0, len);
- tmp = __zend_malloc(actual_size);
++ tmp = __zend_malloc(actual_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
+ memset(tmp, 0, actual_size);
return tmp;
}
diff --cc Zend/zend_alloc.h
index 541989a2a13,6d1d9c2f17a..315b9f3dcda
--- a/Zend/zend_alloc.h
+++ b/Zend/zend_alloc.h
@@@ -215,7 -230,40 +231,40 @@@ ZEND_API ZEND_ATTRIBUTE_MALLOC char * _
#define perealloc2_recoverable_rel(ptr, size, copy_size, persistent) ((persistent)?realloc((ptr), (size)):erealloc2_recoverable_rel((ptr), (size), (copy_size)))
#define pestrdup_rel(s, persistent) ((persistent)?strdup(s):estrdup_rel(s))
+ /* Fallible allocation.
+ * These functions return NULL if allocation would exceed memory limit. */
+
+ ZEND_API ZEND_ATTRIBUTE_MALLOC void* ZEND_FASTCALL _femalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE(1);
+ ZEND_API ZEND_ATTRIBUTE_MALLOC void* ZEND_FASTCALL _fecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE2(1,2);
+ ZEND_API void* ZEND_FASTCALL _ferealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE(2);
+ ZEND_API void* ZEND_FASTCALL _ferealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE(2);
+ ZEND_API ZEND_ATTRIBUTE_MALLOC void* ZEND_FASTCALL _safe_femalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ ZEND_API void* ZEND_FASTCALL _safe_ferealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+
+ #define femalloc(size, fallible) \
+ ((fallible)?_femalloc((size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_emalloc((size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+ #define fecalloc(nmemb, size, fallible) \
+ ((fallible)?_fecalloc((nmemb), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_ecalloc((nmemb), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+ #define ferealloc(ptr, size, fallible) \
+ ((fallible)?_ferealloc((ptr), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_erealloc((ptr), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+ #define ferealloc2(ptr, size, copy_size, fallible) \
+ ((fallible)?_ferealloc2((ptr), (size), (copy_size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_erealloc2((ptr), (size), (copy_size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+ #define safe_femalloc(nmemb, size, offset, fallible) \
+ ((fallible)?_safe_femalloc((nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_safe_emalloc((nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+ #define safe_ferealloc(ptr, nmemb, size, offset, fallible) \
+ ((fallible)?_safe_ferealloc((ptr), (nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):_safe_erealloc((ptr), (nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC))
+
+ #define fpemalloc(size, persistent, fallible) \
- ((persistent)?__zend_malloc(size):femalloc(size, fallible))
-#define fperealloc(ptr, size, persistent, fallible) ((persistent)?__zend_realloc((ptr), (size)):ferealloc((ptr), (size), (fallible)))
-#define fperealloc2(ptr, size, copy_size, persistent, fallible) ((persistent)?__zend_realloc((ptr), (size)):ferealloc2((ptr), (size), (copy_size), (fallible)))
++ ((persistent)?__zend_malloc(size ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):femalloc(size, fallible))
++#define fperealloc(ptr, size, persistent, fallible) ((persistent)?__zend_realloc((ptr), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):ferealloc((ptr), (size), (fallible)))
++#define fperealloc2(ptr, size, copy_size, persistent, fallible) ((persistent)?__zend_realloc((ptr), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC):ferealloc2((ptr), (size), (copy_size), (fallible)))
+ #define safe_fpemalloc(nmemb, size, offset, persistent, fallible) \
+ ((persistent)?_safe_malloc(nmemb, size, offset):safe_femalloc(nmemb, size, offset, fallible))
+ #define safe_fperealloc(ptr, nmemb, size, offset, persistent, fallible) \
+ ((persistent)?_safe_realloc(ptr, nmemb, size, offset):safe_ferealloc(ptr, nmemb, size, offset, fallible))
+
ZEND_API zend_result zend_set_memory_limit(size_t memory_limit);
+ ZEND_API zend_result zend_set_memory_hard_limit_percent(zend_long percent);
ZEND_API bool zend_alloc_in_memory_limit_error_reporting(void);
ZEND_API void start_memory_manager(void);
diff --cc Zend/zend_builtin_functions.c
index 274fa476d9d,3d49b49ab63..7917f4c98a5
--- a/Zend/zend_builtin_functions.c
+++ b/Zend/zend_builtin_functions.c
@@@ -2146,11 -1980,17 +2177,18 @@@ not_frameless_call
}
ZVAL_ARR(&tmp, stack_frame);
- zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &tmp);
+ if (UNEXPECTED(zend_hash_next_index_finsert_new(Z_ARRVAL_P(return_value), &tmp) == HASH_NOMEM)) {
+ zend_array_destroy(stack_frame);
+ return;
+ }
frameno++;
+ prev_stack_frame = stack_frame;
skip_frame:
+ if (ZEND_HAS_OOM()) {
+ return;
+ }
+
if (UNEXPECTED(ZEND_CALL_KIND(call) == ZEND_CALL_TOP_FUNCTION)
&& !fake_frame
&& prev
diff --cc Zend/zend_execute.c
index ea5f55cf6ef,b5e937928ff..ae0b337ed2a
--- a/Zend/zend_execute.c
+++ b/Zend/zend_execute.c
@@@ -4078,16 -3896,6 +4078,18 @@@ ZEND_API void ZEND_FASTCALL zend_free_c
}
/* }}} */
+ZEND_API ZEND_COLD void ZEND_FASTCALL zend_fcall_interrupt(zend_execute_data *call)
+{
+ zend_atomic_bool_store_ex(&EG(vm_interrupt), false);
+ if (zend_atomic_bool_load_ex(&EG(timed_out))) {
+ zend_timeout();
++ } else if (ZEND_HAS_OOM()) {
++ zend_handle_delayed_oom();
+ } else if (zend_interrupt_function) {
+ zend_interrupt_function(call);
+ }
+}
+
#define ZEND_VM_INTERRUPT_CHECK() do { \
if (UNEXPECTED(zend_atomic_bool_load_ex(&EG(vm_interrupt)))) { \
ZEND_VM_INTERRUPT(); \
@@@ -4673,6 -4472,32 +4675,18 @@@ static void cleanup_unfinished_calls(ze
}
/* }}} */
+ /* Same as cleanup_unfinished_calls(), but only free the call frames. */
+ static void free_unfinished_calls(zend_execute_data *execute_data)
+ {
+ zend_execute_data *call = EX(call);
+
+ while (UNEXPECTED(call != NULL)) {
+ EX(call) = call->prev_execute_data;
+ zend_vm_stack_free_call_frame(call);
+ call = EX(call);
+ }
+ }
+
-static const zend_live_range *find_live_range(const zend_op_array *op_array, uint32_t op_num, uint32_t var_num) /* {{{ */
-{
- int i;
- for (i = 0; i < op_array->last_live_range; i++) {
- const zend_live_range *range = &op_array->live_range[i];
- if (op_num >= range->start && op_num < range->end
- && var_num == (range->var & ~ZEND_LIVE_MASK)) {
- return range;
- }
- }
- return NULL;
-}
-/* }}} */
-
static void cleanup_live_vars(zend_execute_data *execute_data, uint32_t op_num, uint32_t catch_op_num) /* {{{ */
{
int i;
diff --cc Zend/zend_globals.h
index 62a97d75363,1599b40ff4b..5751ceea6c8
--- a/Zend/zend_globals.h
+++ b/Zend/zend_globals.h
@@@ -312,9 -306,9 +314,11 @@@ struct _zend_executor_globals
struct sigaction oldact;
#endif
+ zend_strtod_state strtod_state;
+
void *reserved[ZEND_MAX_RESERVED_RESOURCES];
+
+ zend_oom_info delayed_oom;
};
#define EG_FLAGS_INITIAL (0)
diff --cc Zend/zend_hash.c
index 82d0318428f,d2d39d494db..3d88d179f7c
--- a/Zend/zend_hash.c
+++ b/Zend/zend_hash.c
@@@ -169,12 -182,18 +182,18 @@@ static zend_always_inline zend_result z
void *data;
uint32_t nSize = ht->nTableSize;
- ZEND_ASSERT(HT_SIZE_TO_MASK(nSize));
+ ZEND_ASSERT(HT_SIZE_TO_MASK(nSize) != 0);
if (UNEXPECTED(GC_FLAGS(ht) & IS_ARRAY_PERSISTENT)) {
- data = pemalloc(HT_SIZE_EX(nSize, HT_SIZE_TO_MASK(nSize)), 1);
+ data = fpemalloc(HT_SIZE_EX(nSize, HT_SIZE_TO_MASK(nSize)), true, fallible);
+ if (ZEND_CONST_COND(fallible, true) && UNEXPECTED(!data)) {
+ return FAILURE;
+ }
} else if (EXPECTED(nSize == HT_MIN_SIZE)) {
- data = emalloc(HT_SIZE_EX(HT_MIN_SIZE, HT_SIZE_TO_MASK(HT_MIN_SIZE)));
+ data = femalloc(HT_SIZE_EX(HT_MIN_SIZE, HT_SIZE_TO_MASK(HT_MIN_SIZE)), fallible);
+ if (ZEND_CONST_COND(fallible, true) && UNEXPECTED(!data)) {
+ return FAILURE;
+ }
ht->nTableMask = HT_SIZE_TO_MASK(HT_MIN_SIZE);
HT_SET_DATA_ADDR(ht, data);
/* Don't overwrite iterator count. */
@@@ -397,9 -520,9 +520,9 @@@ static zend_always_inline zend_result Z
{
HT_ASSERT_RC1(ht);
- if (nSize == 0) return;
+ if (nSize == 0) return SUCCESS;
- ZEND_ASSERT(HT_SIZE_TO_MASK(nSize));
+ ZEND_ASSERT(HT_SIZE_TO_MASK(nSize) != 0);
if (UNEXPECTED(HT_FLAGS(ht) & HASH_FLAG_UNINITIALIZED)) {
if (nSize > ht->nTableSize) {
@@@ -1318,9 -1524,12 +1529,12 @@@ static zend_result ZEND_FASTCALL zend_h
uint32_t nSize = ht->nTableSize + ht->nTableSize;
Bucket *old_buckets = ht->arData;
- ZEND_ASSERT(HT_SIZE_TO_MASK(nSize));
+ ZEND_ASSERT(HT_SIZE_TO_MASK(nSize) != 0);
- new_data = pemalloc(HT_SIZE_EX(nSize, HT_SIZE_TO_MASK(nSize)), GC_FLAGS(ht) & IS_ARRAY_PERSISTENT);
+ new_data = fpemalloc(HT_SIZE_EX(nSize, HT_SIZE_TO_MASK(nSize)), GC_FLAGS(ht) & IS_ARRAY_PERSISTENT, fallible);
+ if (ZEND_CONST_COND(fallible, true) && UNEXPECTED(!new_data)) {
+ return FAILURE;
+ }
ht->nTableSize = nSize;
ht->nTableMask = HT_SIZE_TO_MASK(ht->nTableSize);
HT_SET_DATA_ADDR(ht, new_data);
diff --cc Zend/zend_hash.h
index b0cf1818455,a2d47df24b0..3c37cc064e9
--- a/Zend/zend_hash.h
+++ b/Zend/zend_hash.h
@@@ -817,8 -856,15 +859,15 @@@ static zend_always_inline void *zend_ha
zval tmp, *zv;
ZVAL_PTR(&tmp, pData);
- zv = zend_hash_index_update(ht, h, &tmp);
+ if (fallible) {
+ zv = zend_hash_index_fupdate(ht, h, &tmp);
+ if (UNEXPECTED(zv == HASH_NOMEM)) {
+ return HASH_NOMEM;
+ }
+ } else {
+ zv = zend_hash_index_update(ht, h, &tmp);
+ }
- ZEND_ASSUME(Z_PTR_P(zv));
+ ZEND_ASSERT(Z_PTR_P(zv));
return Z_PTR_P(zv);
}
diff --cc Zend/zend_objects.c
index 8a54a839869,a900d85042d..1a3333f637b
--- a/Zend/zend_objects.c
+++ b/Zend/zend_objects.c
@@@ -116,7 -99,7 +116,7 @@@ ZEND_API void zend_objects_destroy_obje
zend_function *destructor = object->ce->destructor;
if (destructor) {
- if (UNEXPECTED(zend_object_is_lazy(object))) {
- if (ZEND_HAS_OOM()) {
++ if (UNEXPECTED(zend_object_is_lazy(object) || ZEND_HAS_OOM())) {
return;
}
diff --cc Zend/zend_smart_str.h
index affbb5c50ac,0626f8d5bb0..33ad81a592e
--- a/Zend/zend_smart_str.h
+++ b/Zend/zend_smart_str.h
@@@ -34,10 -36,9 +36,10 @@@ ZEND_API void smart_str_append_printf(s
ZEND_ATTRIBUTE_FORMAT(printf, 2, 3);
ZEND_API void ZEND_FASTCALL smart_str_append_escaped_truncated(smart_str *str, const zend_string *value, size_t length);
ZEND_API void ZEND_FASTCALL smart_str_append_scalar(smart_str *str, const zval *value, size_t truncate);
+ZEND_API zend_result ZEND_FASTCALL smart_str_append_zval(smart_str *dest, const zval *value, size_t truncate);
END_EXTERN_C()
- static zend_always_inline size_t smart_str_alloc(smart_str *str, size_t len, bool persistent) {
+ static zend_always_inline size_t smart_str_falloc(smart_str *str, size_t len, bool persistent, bool fallible) {
if (UNEXPECTED(!str->s)) {
goto do_smart_str_realloc;
} else {
diff --cc Zend/zend_vm_def.h
index a5605f9a43c,d63afcc71e6..df255c5a868
--- a/Zend/zend_vm_def.h
+++ b/Zend/zend_vm_def.h
@@@ -9657,205 -9581,6 +9660,209 @@@ ZEND_VM_HANDLER(202, ZEND_CALLABLE_CONV
ZEND_VM_NEXT_OPCODE();
}
+ZEND_VM_HANDLER(208, ZEND_JMP_FRAMELESS, CONST, JMP_ADDR, NUM|CACHE_SLOT)
+{
+ USE_OPLINE
+ zend_jmp_fl_result result = (uintptr_t)CACHED_PTR(opline->extended_value);
+ZEND_VM_C_LABEL(try_again):
+ if (EXPECTED(result == ZEND_JMP_FL_HIT)) {
+ OPLINE = OP_JMP_ADDR(opline, opline->op2);
+ ZEND_VM_CONTINUE();
+ } else if (EXPECTED(result == ZEND_JMP_FL_MISS)) {
+ ZEND_VM_NEXT_OPCODE();
+ } else {
+ ZEND_ASSERT(result == ZEND_JMP_FL_UNPRIMED);
+ /* func_name refers to the function in the local namespace, e.g. foo\substr. */
+ zval *func_name = (zval *)RT_CONSTANT(opline, opline->op1);
+ /* If it cannot be found locally, we must be referring to the global function. */
+ zval *func = zend_hash_find_known_hash(EG(function_table), Z_STR_P(func_name));
+ /* ZEND_JMP_FL_MISS = 1, ZEND_JMP_FL_HIT = 2 */
+ result = (func == NULL) + 1;
+ CACHE_PTR(opline->extended_value, (void *)result);
+ ZEND_VM_C_GOTO(try_again);
+ }
+}
+
+ZEND_VM_HANDLER(204, ZEND_FRAMELESS_ICALL_0, UNUSED, UNUSED, SPEC(OBSERVER))
+{
+ USE_OPLINE
+ SAVE_OPLINE();
+
+ zval *result = EX_VAR(opline->result.var);
+ ZVAL_NULL(result);
+
+#if !ZEND_VM_SPEC || ZEND_OBSERVER_ENABLED
+ if (ZEND_OBSERVER_ENABLED && UNEXPECTED(zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(ZEND_FLF_FUNC(opline))) == false)) {
+ zend_frameless_observed_call(execute_data);
+ } else
+#endif
+ {
+ zend_frameless_function_0 function = (zend_frameless_function_0)ZEND_FLF_HANDLER(opline);
+ function(EX_VAR(opline->result.var));
+ }
++ ZEND_VM_FCALL_INTERRUPT_CHECK(execute_data);
+ ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
+}
+
+ZEND_VM_HANDLER(205, ZEND_FRAMELESS_ICALL_1, ANY, UNUSED, SPEC(OBSERVER))
+{
+ USE_OPLINE
+ SAVE_OPLINE();
+
+ zval *result = EX_VAR(opline->result.var);
+ ZVAL_NULL(result);
+ zval *arg1 = GET_OP1_ZVAL_PTR_DEREF(BP_VAR_R);
+ if (EG(exception)) {
+ FREE_OP1();
+ HANDLE_EXCEPTION();
+ }
+
+#if !ZEND_VM_SPEC || ZEND_OBSERVER_ENABLED
+ if (ZEND_OBSERVER_ENABLED && UNEXPECTED(zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(ZEND_FLF_FUNC(opline))) == false)) {
+ zend_frameless_observed_call(execute_data);
+ } else
+#endif
+ {
+ zend_frameless_function_1 function = (zend_frameless_function_1)ZEND_FLF_HANDLER(opline);
+ function(result, arg1);
+ }
+ FREE_OP1();
++ ZEND_VM_FCALL_INTERRUPT_CHECK(execute_data);
+ ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
+}
+
+ZEND_VM_HANDLER(206, ZEND_FRAMELESS_ICALL_2, ANY, ANY, SPEC(OBSERVER))
+{
+ USE_OPLINE
+ SAVE_OPLINE();
+
+ zval *result = EX_VAR(opline->result.var);
+ ZVAL_NULL(result);
+ zval *arg1 = GET_OP1_ZVAL_PTR_DEREF(BP_VAR_R);
+ zval *arg2 = GET_OP2_ZVAL_PTR_DEREF(BP_VAR_R);
+ if (EG(exception)) {
+ FREE_OP1();
+ FREE_OP2();
+ HANDLE_EXCEPTION();
+ }
+
+#if !ZEND_VM_SPEC || ZEND_OBSERVER_ENABLED
+ if (ZEND_OBSERVER_ENABLED && UNEXPECTED(zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(ZEND_FLF_FUNC(opline))) == false)) {
+ zend_frameless_observed_call(execute_data);
+ } else
+#endif
+ {
+ zend_frameless_function_2 function = (zend_frameless_function_2)ZEND_FLF_HANDLER(opline);
+ function(result, arg1, arg2);
+ }
+
+ FREE_OP1();
+ /* Set OP1 to UNDEF in case FREE_OP2() throws. */
+ if (OP1_TYPE & (IS_VAR|IS_TMP_VAR)) {
+ ZVAL_UNDEF(EX_VAR(opline->op1.var));
+ }
+ FREE_OP2();
++ ZEND_VM_FCALL_INTERRUPT_CHECK(execute_data);
+ ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
+}
+
+ZEND_VM_HANDLER(207, ZEND_FRAMELESS_ICALL_3, ANY, ANY, SPEC(OBSERVER))
+{
+ USE_OPLINE
+ SAVE_OPLINE();
+
+ zval *result = EX_VAR(opline->result.var);
+ ZVAL_NULL(result);
+ zval *arg1 = GET_OP1_ZVAL_PTR_DEREF(BP_VAR_R);
+ zval *arg2 = GET_OP2_ZVAL_PTR_DEREF(BP_VAR_R);
+ zval *arg3 = GET_OP_DATA_ZVAL_PTR_DEREF(BP_VAR_R);
+ if (EG(exception)) {
+ FREE_OP1();
+ FREE_OP2();
+ FREE_OP_DATA();
+ HANDLE_EXCEPTION();
+ }
+
+#if !ZEND_VM_SPEC || ZEND_OBSERVER_ENABLED
+ if (ZEND_OBSERVER_ENABLED && UNEXPECTED(zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(ZEND_FLF_FUNC(opline))) == false)) {
+ zend_frameless_observed_call(execute_data);
+ } else
+#endif
+ {
+ zend_frameless_function_3 function = (zend_frameless_function_3)ZEND_FLF_HANDLER(opline);
+ function(result, arg1, arg2, arg3);
+ }
+
+ FREE_OP1();
+ /* Set to UNDEF in case FREE_OP2() throws. */
+ if (OP1_TYPE & (IS_VAR|IS_TMP_VAR)) {
+ ZVAL_UNDEF(EX_VAR(opline->op1.var));
+ }
+ FREE_OP2();
+ if (OP2_TYPE & (IS_VAR|IS_TMP_VAR)) {
+ ZVAL_UNDEF(EX_VAR(opline->op2.var));
+ }
+ FREE_OP_DATA();
++ ZEND_VM_FCALL_INTERRUPT_CHECK(execute_data);
+ ZEND_VM_NEXT_OPCODE_EX(1, 2);
+}
+
+ZEND_VM_HANDLER(209, ZEND_INIT_PARENT_PROPERTY_HOOK_CALL, CONST, UNUSED|NUM, NUM)
+{
+ USE_OPLINE
+ SAVE_OPLINE();
+
+ zend_class_entry *ce = EX(func)->common.scope;
+ ZEND_ASSERT(ce);
+
+ zend_class_entry *parent_ce = ce->parent;
+ if (!parent_ce) {
+ zend_throw_error(NULL, "Cannot use \"parent\" when current class scope has no parent");
+ UNDEF_RESULT();
+ HANDLE_EXCEPTION();
+ }
+
+ zend_string *property_name = Z_STR_P(RT_CONSTANT(opline, opline->op1));
+ zend_property_hook_kind hook_kind = opline->op2.num;
+
+ zend_property_info *prop_info = zend_hash_find_ptr(&parent_ce->properties_info, property_name);
+ if (!prop_info) {
+ zend_throw_error(NULL, "Undefined property %s::$%s", ZSTR_VAL(parent_ce->name), ZSTR_VAL(property_name));
+ UNDEF_RESULT();
+ HANDLE_EXCEPTION();
+ }
+ if (prop_info->flags & ZEND_ACC_PRIVATE) {
+ zend_throw_error(NULL, "Cannot access private property %s::$%s", ZSTR_VAL(parent_ce->name), ZSTR_VAL(property_name));
+ UNDEF_RESULT();
+ HANDLE_EXCEPTION();
+ }
+
+ zend_function **hooks = prop_info->hooks;
+ zend_function *hook = hooks ? hooks[hook_kind] : NULL;
+
+ zend_execute_data *call;
+ if (hook) {
+ call = zend_vm_stack_push_call_frame(
+ ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_HAS_THIS,
+ hook, opline->extended_value, Z_OBJ_P(ZEND_THIS));
+ if (EXPECTED(hook->type == ZEND_USER_FUNCTION)) {
+ if (UNEXPECTED(!RUN_TIME_CACHE(&hook->op_array))) {
+ init_func_run_time_cache(&hook->op_array);
+ }
+ call->run_time_cache = RUN_TIME_CACHE(&hook->op_array);
+ }
+ } else {
+ zend_function *fbc = zend_get_property_hook_trampoline(prop_info, hook_kind, property_name);
+ call = zend_vm_stack_push_call_frame(
+ ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_HAS_THIS,
+ fbc, opline->extended_value, Z_OBJ_P(ZEND_THIS));
+ }
+
+ call->prev_execute_data = EX(call);
+ EX(call) = call;
+ ZEND_VM_NEXT_OPCODE();
+}
+
ZEND_VM_HOT_TYPE_SPEC_HANDLER(ZEND_JMP, (OP_JMP_ADDR(op, op->op1) > op), ZEND_JMP_FORWARD, JMP_ADDR, ANY)
{
USE_OPLINE
diff --cc Zend/zend_vm_execute.h
index f17f7bc3b31,3f4ead26d27..4b313f20cd3
Binary files differ
diff --cc ext/bz2/bz2.c
index 4ac24931075,972c330e678..f272e6d1741
--- a/ext/bz2/bz2.c
+++ b/ext/bz2/bz2.c
@@@ -560,7 -544,12 +566,11 @@@ PHP_FUNCTION(bzdecompress
/* no reason to continue if we're going to drop it anyway */
break;
}
- dest = zend_string_safe_realloc(dest, 1, bzs.avail_out+1, (size_t) size, 0);
-#endif
+ tmp = zend_string_safe_frealloc(dest, 1, bzs.avail_out+1, (size_t) size, 0, /* fallible */ true);
+ if (!tmp) {
+ goto failure;
+ }
+ dest = tmp;
bzs.next_out = ZSTR_VAL(dest) + size;
}
@@@ -570,8 -560,14 +580,12 @@@
php_error_docref(NULL, E_WARNING, "Decompressed size too big, max is %zd", SIZE_MAX);
zend_string_efree(dest);
RETVAL_LONG(BZ_MEM_ERROR);
- } else
-#endif
- {
+ } else {
- dest = zend_string_safe_realloc(dest, 1, (size_t)size, 1, 0);
+ tmp = zend_string_safe_frealloc(dest, 1, (size_t)size, 1, 0, /* fallible */ true);
+ if (!tmp) {
+ goto failure;
+ }
+ dest = tmp;
ZSTR_LEN(dest) = (size_t)size;
ZSTR_VAL(dest)[(size_t)size] = '\0';
RETVAL_STR(dest);
diff --cc ext/curl/interface.c
index 8f82bd7f285,71d316a9352..8aec63ca501
--- a/ext/curl/interface.c
+++ b/ext/curl/interface.c
@@@ -608,10 -591,16 +608,16 @@@ static size_t curl_write(char *data, si
PHPWRITE(data, length);
break;
case PHP_CURL_FILE:
- return fwrite(data, size, nmemb, t->fp);
+ return fwrite(data, size, nmemb, write_handler->fp);
case PHP_CURL_RETURN:
if (length > 0) {
- smart_str_appendl(&write_handler->buf, data, length);
- if (!smart_str_fappendl_ex(&t->buf, data, (int) length, /* persistent */ false, /* fallible */ true)) {
++ if (!smart_str_fappendl_ex(&write_handler->buf, data, length, /* persistent */ false, /* fallible */ true)) {
+ #ifdef CURL_WRITEFUNC_ERROR
+ return CURL_WRITEFUNC_ERROR;
+ #else
+ return 0;
+ #endif
+ }
}
break;
case PHP_CURL_USER: {
diff --cc ext/date/lib/parse_date.c
index d1578c0e0f1,d8186dd14fa..31103a81fa0
--- a/ext/date/lib/parse_date.c
+++ b/ext/date/lib/parse_date.c
@@@ -1,4 -1,4 +1,4 @@@
- /* Generated by re2c 1.0.3 on Sun Oct 4 18:25:15 2026 */
-/* Generated by re2c 1.0.3 on Wed Sep 30 15:20:23 2026 */
++/* Generated by re2c 1.0.3 on Wed Sep 30 23:15:59 2026 */
#line 1 "ext/date/lib/parse_date.re"
/*
* The MIT License (MIT)
@@@ -1028,11 -1038,11 +1045,11 @@@ static int scan(Scanner *s, timelib_tz_
std:
s->tok = cursor;
s->len = 0;
- #line 1161 "ext/date/lib/parse_date.re"
-#line 1171 "ext/date/lib/parse_date.re"
++#line 1178 "ext/date/lib/parse_date.re"
- #line 1036 "ext/date/lib/parse_date.c"
-#line 1046 "ext/date/lib/parse_date.c"
++#line 1053 "ext/date/lib/parse_date.c"
{
YYCTYPE yych;
unsigned int yyaccept = 0;
@@@ -1213,23 -1223,23 +1230,23 @@@ yy2
YYDEBUG(2, *YYCURSOR);
++YYCURSOR;
YYDEBUG(3, *YYCURSOR);
- #line 1994 "ext/date/lib/parse_date.re"
-#line 2004 "ext/date/lib/parse_date.re"
++#line 2011 "ext/date/lib/parse_date.re"
{
s->pos = cursor; s->line++;
goto std;
}
- #line 1222 "ext/date/lib/parse_date.c"
-#line 1232 "ext/date/lib/parse_date.c"
++#line 1239 "ext/date/lib/parse_date.c"
yy4:
YYDEBUG(4, *YYCURSOR);
++YYCURSOR;
yy5:
YYDEBUG(5, *YYCURSOR);
- #line 2000 "ext/date/lib/parse_date.re"
-#line 2010 "ext/date/lib/parse_date.re"
++#line 2017 "ext/date/lib/parse_date.re"
{
add_error(s, TIMELIB_ERR_UNEXPECTED_CHARACTER, "Unexpected character");
goto std;
}
- #line 1233 "ext/date/lib/parse_date.c"
-#line 1243 "ext/date/lib/parse_date.c"
++#line 1250 "ext/date/lib/parse_date.c"
yy6:
YYDEBUG(6, *YYCURSOR);
yyaccept = 0;
@@@ -1244,11 -1254,11 +1261,11 @@@
if (yych <= '9') goto yy58;
yy8:
YYDEBUG(8, *YYCURSOR);
- #line 1989 "ext/date/lib/parse_date.re"
-#line 1999 "ext/date/lib/parse_date.re"
++#line 2006 "ext/date/lib/parse_date.re"
{
goto std;
}
- #line 1252 "ext/date/lib/parse_date.c"
-#line 1262 "ext/date/lib/parse_date.c"
++#line 1269 "ext/date/lib/parse_date.c"
yy9:
YYDEBUG(9, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -1282,11 -1292,11 +1299,11 @@@ yy11
YYDEBUG(11, *YYCURSOR);
++YYCURSOR;
YYDEBUG(12, *YYCURSOR);
- #line 1984 "ext/date/lib/parse_date.re"
-#line 1994 "ext/date/lib/parse_date.re"
++#line 2001 "ext/date/lib/parse_date.re"
{
goto std;
}
- #line 1290 "ext/date/lib/parse_date.c"
-#line 1300 "ext/date/lib/parse_date.c"
++#line 1307 "ext/date/lib/parse_date.c"
yy13:
YYDEBUG(13, *YYCURSOR);
yyaccept = 1;
@@@ -1787,7 -1797,7 +1804,7 @@@ yy19
}
yy20:
YYDEBUG(20, *YYCURSOR);
- #line 1899 "ext/date/lib/parse_date.re"
-#line 1909 "ext/date/lib/parse_date.re"
++#line 1916 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("tzcorrection | tz");
@@@ -1801,7 -1811,7 +1818,7 @@@
TIMELIB_DEINIT;
return TIMELIB_TIMEZONE;
}
- #line 1805 "ext/date/lib/parse_date.c"
-#line 1815 "ext/date/lib/parse_date.c"
++#line 1822 "ext/date/lib/parse_date.c"
yy21:
YYDEBUG(21, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -3606,7 -3616,7 +3623,7 @@@ yy80
}
yy81:
YYDEBUG(81, *YYCURSOR);
- #line 1646 "ext/date/lib/parse_date.re"
-#line 1656 "ext/date/lib/parse_date.re"
++#line 1663 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("datenoyearrev");
TIMELIB_INIT;
@@@ -3617,7 -3627,7 +3634,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_TEXT;
}
- #line 3621 "ext/date/lib/parse_date.c"
-#line 3631 "ext/date/lib/parse_date.c"
++#line 3638 "ext/date/lib/parse_date.c"
yy82:
YYDEBUG(82, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -4132,7 -4142,7 +4149,7 @@@ yy112
}
if (yych == '.') goto yy289;
YYDEBUG(114, *YYCURSOR);
- #line 1221 "ext/date/lib/parse_date.re"
-#line 1231 "ext/date/lib/parse_date.re"
++#line 1238 "ext/date/lib/parse_date.re"
{
timelib_ull i;
@@@ -4157,7 -4167,7 +4174,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 4161 "ext/date/lib/parse_date.c"
-#line 4171 "ext/date/lib/parse_date.c"
++#line 4178 "ext/date/lib/parse_date.c"
yy115:
YYDEBUG(115, *YYCURSOR);
++YYCURSOR;
@@@ -5883,7 -5893,7 +5900,7 @@@ yy176
}
yy177:
YYDEBUG(177, *YYCURSOR);
- #line 1387 "ext/date/lib/parse_date.re"
-#line 1397 "ext/date/lib/parse_date.re"
++#line 1404 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("timetiny24 | timeshort24 | timelong24 | iso8601long");
@@@ -5910,7 -5920,7 +5927,7 @@@
TIMELIB_DEINIT;
return TIMELIB_TIME24_WITH_ZONE;
}
- #line 5914 "ext/date/lib/parse_date.c"
-#line 5924 "ext/date/lib/parse_date.c"
++#line 5931 "ext/date/lib/parse_date.c"
yy178:
YYDEBUG(178, *YYCURSOR);
yyaccept = 4;
@@@ -6939,7 -6949,7 +6956,7 @@@ yy223
}
yy224:
YYDEBUG(224, *YYCURSOR);
- #line 1481 "ext/date/lib/parse_date.re"
-#line 1491 "ext/date/lib/parse_date.re"
++#line 1498 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("americanshort | american");
@@@ -6954,7 -6964,7 +6971,7 @@@
TIMELIB_DEINIT;
return TIMELIB_AMERICAN;
}
- #line 6958 "ext/date/lib/parse_date.c"
-#line 6968 "ext/date/lib/parse_date.c"
++#line 6975 "ext/date/lib/parse_date.c"
yy225:
YYDEBUG(225, *YYCURSOR);
yyaccept = 5;
@@@ -7197,7 -7207,7 +7214,7 @@@ yy250
if (yych <= '9') goto yy431;
yy251:
YYDEBUG(251, *YYCURSOR);
- #line 1563 "ext/date/lib/parse_date.re"
-#line 1573 "ext/date/lib/parse_date.re"
++#line 1580 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("datefull");
@@@ -7211,7 -7221,7 +7228,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_FULL;
}
- #line 7215 "ext/date/lib/parse_date.c"
-#line 7225 "ext/date/lib/parse_date.c"
++#line 7232 "ext/date/lib/parse_date.c"
yy252:
YYDEBUG(252, *YYCURSOR);
yyaccept = 3;
@@@ -7325,7 -7335,7 +7342,7 @@@ yy259
if (yych == 'e') goto yy440;
yy260:
YYDEBUG(260, *YYCURSOR);
- #line 1968 "ext/date/lib/parse_date.re"
-#line 1978 "ext/date/lib/parse_date.re"
++#line 1985 "ext/date/lib/parse_date.re"
{
timelib_ull i;
DEBUG_OUTPUT("relative");
@@@ -7340,7 -7350,7 +7357,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 7344 "ext/date/lib/parse_date.c"
-#line 7354 "ext/date/lib/parse_date.c"
++#line 7361 "ext/date/lib/parse_date.c"
yy261:
YYDEBUG(261, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -7786,7 -7796,7 +7803,7 @@@ yy289
if (yych <= '9') goto yy471;
yy290:
YYDEBUG(290, *YYCURSOR);
- #line 1247 "ext/date/lib/parse_date.re"
-#line 1257 "ext/date/lib/parse_date.re"
++#line 1264 "ext/date/lib/parse_date.re"
{
timelib_sll i;
timelib_ull us;
@@@ -7825,7 -7835,7 +7842,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 7829 "ext/date/lib/parse_date.c"
-#line 7839 "ext/date/lib/parse_date.c"
++#line 7846 "ext/date/lib/parse_date.c"
yy291:
YYDEBUG(291, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -7850,7 -7860,7 +7867,7 @@@ yy292
}
yy293:
YYDEBUG(293, *YYCURSOR);
- #line 1809 "ext/date/lib/parse_date.re"
-#line 1819 "ext/date/lib/parse_date.re"
++#line 1826 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("ago");
TIMELIB_INIT;
@@@ -7870,7 -7880,7 +7887,7 @@@
TIMELIB_DEINIT;
return TIMELIB_AGO;
}
- #line 7874 "ext/date/lib/parse_date.c"
-#line 7884 "ext/date/lib/parse_date.c"
++#line 7891 "ext/date/lib/parse_date.c"
yy294:
YYDEBUG(294, *YYCURSOR);
yyaccept = 7;
@@@ -7909,7 -7919,7 +7926,7 @@@
}
yy295:
YYDEBUG(295, *YYCURSOR);
- #line 1889 "ext/date/lib/parse_date.re"
-#line 1899 "ext/date/lib/parse_date.re"
++#line 1906 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("monthtext");
TIMELIB_INIT;
@@@ -7918,7 -7928,7 +7935,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_TEXT;
}
- #line 7922 "ext/date/lib/parse_date.c"
-#line 7932 "ext/date/lib/parse_date.c"
++#line 7939 "ext/date/lib/parse_date.c"
yy296:
YYDEBUG(296, *YYCURSOR);
yyaccept = 7;
@@@ -8493,7 -8503,7 +8510,7 @@@ yy314
}
yy315:
YYDEBUG(315, *YYCURSOR);
- #line 1830 "ext/date/lib/parse_date.re"
-#line 1840 "ext/date/lib/parse_date.re"
++#line 1847 "ext/date/lib/parse_date.re"
{
const timelib_relunit* relunit;
DEBUG_OUTPUT("daytext");
@@@ -8510,7 -8520,7 +8527,7 @@@
TIMELIB_DEINIT;
return TIMELIB_WEEKDAY;
}
- #line 8514 "ext/date/lib/parse_date.c"
-#line 8524 "ext/date/lib/parse_date.c"
++#line 8531 "ext/date/lib/parse_date.c"
yy316:
YYDEBUG(316, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -8778,7 -8788,7 +8795,7 @@@ yy324
}
yy325:
YYDEBUG(325, *YYCURSOR);
- #line 1632 "ext/date/lib/parse_date.re"
-#line 1642 "ext/date/lib/parse_date.re"
++#line 1649 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("datetextual | datenoyear");
@@@ -8791,7 -8801,7 +8808,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_TEXT;
}
- #line 8795 "ext/date/lib/parse_date.c"
-#line 8805 "ext/date/lib/parse_date.c"
++#line 8812 "ext/date/lib/parse_date.c"
yy326:
YYDEBUG(326, *YYCURSOR);
yyaccept = 10;
@@@ -9485,7 -9495,7 +9502,7 @@@ yy350
}
yy351:
YYDEBUG(351, *YYCURSOR);
- #line 1178 "ext/date/lib/parse_date.re"
-#line 1188 "ext/date/lib/parse_date.re"
++#line 1195 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("now");
TIMELIB_INIT;
@@@ -9493,7 -9503,7 +9510,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 9497 "ext/date/lib/parse_date.c"
-#line 9507 "ext/date/lib/parse_date.c"
++#line 9514 "ext/date/lib/parse_date.c"
yy352:
YYDEBUG(352, *YYCURSOR);
yyaccept = 2;
@@@ -10996,7 -11006,7 +11013,7 @@@ yy419
}
yy420:
YYDEBUG(420, *YYCURSOR);
- #line 1415 "ext/date/lib/parse_date.re"
-#line 1425 "ext/date/lib/parse_date.re"
++#line 1432 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("gnunocolon");
TIMELIB_INIT;
@@@ -11018,7 -11028,7 +11035,7 @@@
TIMELIB_DEINIT;
return TIMELIB_GNU_NOCOLON;
}
- #line 11022 "ext/date/lib/parse_date.c"
-#line 11032 "ext/date/lib/parse_date.c"
++#line 11039 "ext/date/lib/parse_date.c"
yy421:
YYDEBUG(421, *YYCURSOR);
yyaccept = 13;
@@@ -11099,7 -11109,7 +11116,7 @@@
}
yy422:
YYDEBUG(422, *YYCURSOR);
- #line 1800 "ext/date/lib/parse_date.re"
-#line 1810 "ext/date/lib/parse_date.re"
++#line 1817 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("year4");
TIMELIB_INIT;
@@@ -11107,7 -11117,7 +11124,7 @@@
TIMELIB_DEINIT;
return TIMELIB_CLF;
}
- #line 11111 "ext/date/lib/parse_date.c"
-#line 11121 "ext/date/lib/parse_date.c"
++#line 11128 "ext/date/lib/parse_date.c"
yy423:
YYDEBUG(423, *YYCURSOR);
yyaccept = 3;
@@@ -11714,7 -11724,7 +11731,7 @@@ yy456
YYDEBUG(456, *YYCURSOR);
++YYCURSOR;
YYDEBUG(457, *YYCURSOR);
- #line 1349 "ext/date/lib/parse_date.re"
-#line 1359 "ext/date/lib/parse_date.re"
++#line 1366 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("timetiny12 | timeshort12 | timelong12");
TIMELIB_INIT;
@@@ -11731,7 -11741,7 +11748,7 @@@
TIMELIB_DEINIT;
return TIMELIB_TIME12;
}
- #line 11735 "ext/date/lib/parse_date.c"
-#line 11745 "ext/date/lib/parse_date.c"
++#line 11752 "ext/date/lib/parse_date.c"
yy458:
YYDEBUG(458, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -13058,7 -13068,7 +13075,7 @@@ yy525
}
yy526:
YYDEBUG(526, *YYCURSOR);
- #line 1187 "ext/date/lib/parse_date.re"
-#line 1197 "ext/date/lib/parse_date.re"
++#line 1204 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("noon");
TIMELIB_INIT;
@@@ -13069,7 -13079,7 +13086,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 13073 "ext/date/lib/parse_date.c"
-#line 13083 "ext/date/lib/parse_date.c"
++#line 13090 "ext/date/lib/parse_date.c"
yy527:
YYDEBUG(527, *YYCURSOR);
yyaccept = 2;
@@@ -14115,7 -14125,7 +14132,7 @@@ yy566
}
yy567:
YYDEBUG(567, *YYCURSOR);
- #line 1549 "ext/date/lib/parse_date.re"
-#line 1559 "ext/date/lib/parse_date.re"
++#line 1566 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("gnudateshort");
@@@ -14128,7 -14138,7 +14145,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_DATE;
}
- #line 14132 "ext/date/lib/parse_date.c"
-#line 14142 "ext/date/lib/parse_date.c"
++#line 14149 "ext/date/lib/parse_date.c"
yy568:
YYDEBUG(568, *YYCURSOR);
yyaccept = 15;
@@@ -14579,7 -14589,7 +14596,7 @@@ yy599
}
yy600:
YYDEBUG(600, *YYCURSOR);
- #line 1618 "ext/date/lib/parse_date.re"
-#line 1628 "ext/date/lib/parse_date.re"
++#line 1635 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("datenodayrev");
@@@ -14592,7 -14602,7 +14609,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_NO_DAY;
}
- #line 14596 "ext/date/lib/parse_date.c"
-#line 14606 "ext/date/lib/parse_date.c"
++#line 14613 "ext/date/lib/parse_date.c"
yy601:
YYDEBUG(601, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -15967,7 -15977,7 +15984,7 @@@ yy696
YYDEBUG(696, *YYCURSOR);
++YYCURSOR;
YYDEBUG(697, *YYCURSOR);
- #line 1604 "ext/date/lib/parse_date.re"
-#line 1614 "ext/date/lib/parse_date.re"
++#line 1621 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("datenoday");
@@@ -15980,7 -15990,7 +15997,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_NO_DAY;
}
- #line 15984 "ext/date/lib/parse_date.c"
-#line 15994 "ext/date/lib/parse_date.c"
++#line 16001 "ext/date/lib/parse_date.c"
yy698:
YYDEBUG(698, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -16541,7 -16551,7 +16558,7 @@@ yy721
}
yy722:
YYDEBUG(722, *YYCURSOR);
- #line 1199 "ext/date/lib/parse_date.re"
-#line 1209 "ext/date/lib/parse_date.re"
++#line 1216 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("midnight | today");
TIMELIB_INIT;
@@@ -16550,7 -16560,7 +16567,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 16554 "ext/date/lib/parse_date.c"
-#line 16564 "ext/date/lib/parse_date.c"
++#line 16571 "ext/date/lib/parse_date.c"
yy723:
YYDEBUG(723, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -16860,7 -16870,7 +16877,7 @@@ yy738
if (yych <= '9') goto yy897;
yy739:
YYDEBUG(739, *YYCURSOR);
- #line 1590 "ext/date/lib/parse_date.re"
-#line 1600 "ext/date/lib/parse_date.re"
++#line 1607 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("pointed date YY");
@@@ -16873,7 -16883,7 +16890,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_FULL_POINTED;
}
- #line 16877 "ext/date/lib/parse_date.c"
-#line 16887 "ext/date/lib/parse_date.c"
++#line 16894 "ext/date/lib/parse_date.c"
yy740:
YYDEBUG(740, *YYCURSOR);
yyaccept = 15;
@@@ -16985,7 -16995,7 +17002,7 @@@ yy751
}
yy752:
YYDEBUG(752, *YYCURSOR);
- #line 1535 "ext/date/lib/parse_date.re"
-#line 1545 "ext/date/lib/parse_date.re"
++#line 1552 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("gnudateshorter");
@@@ -16998,7 -17008,7 +17015,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_DATE;
}
- #line 17002 "ext/date/lib/parse_date.c"
-#line 17012 "ext/date/lib/parse_date.c"
++#line 17019 "ext/date/lib/parse_date.c"
yy753:
YYDEBUG(753, *YYCURSOR);
yyaccept = 18;
@@@ -17247,7 -17257,7 +17264,7 @@@ yy776
}
yy777:
YYDEBUG(777, *YYCURSOR);
- #line 1461 "ext/date/lib/parse_date.re"
-#line 1471 "ext/date/lib/parse_date.re"
++#line 1478 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("iso8601nocolon");
@@@ -17266,7 -17276,7 +17283,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_NOCOLON;
}
- #line 17270 "ext/date/lib/parse_date.c"
-#line 17280 "ext/date/lib/parse_date.c"
++#line 17287 "ext/date/lib/parse_date.c"
yy778:
YYDEBUG(778, *YYCURSOR);
yyaccept = 19;
@@@ -18494,7 -18504,7 +18511,7 @@@ yy848
}
yy849:
YYDEBUG(849, *YYCURSOR);
- #line 1938 "ext/date/lib/parse_date.re"
-#line 1948 "ext/date/lib/parse_date.re"
++#line 1955 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("dateshortwithtimeshort | dateshortwithtimelong | dateshortwithtimelongtz");
@@@ -18523,7 -18533,7 +18540,7 @@@
TIMELIB_DEINIT;
return TIMELIB_SHORTDATE_WITH_TIME;
}
- #line 18527 "ext/date/lib/parse_date.c"
-#line 18537 "ext/date/lib/parse_date.c"
++#line 18544 "ext/date/lib/parse_date.c"
yy850:
YYDEBUG(850, *YYCURSOR);
yyaccept = 20;
@@@ -19567,7 -19577,7 +19584,7 @@@ yy925
}
yy926:
YYDEBUG(926, *YYCURSOR);
- #line 1696 "ext/date/lib/parse_date.re"
-#line 1706 "ext/date/lib/parse_date.re"
++#line 1713 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("pgydotd");
@@@ -19580,7 -19590,7 +19597,7 @@@
TIMELIB_DEINIT;
return TIMELIB_PG_YEARDAY;
}
- #line 19584 "ext/date/lib/parse_date.c"
-#line 19594 "ext/date/lib/parse_date.c"
++#line 19601 "ext/date/lib/parse_date.c"
yy927:
YYDEBUG(927, *YYCURSOR);
yyaccept = 21;
@@@ -19834,7 -19844,7 +19851,7 @@@ yy941
if (yych <= '7') goto yy1059;
yy942:
YYDEBUG(942, *YYCURSOR);
- #line 1729 "ext/date/lib/parse_date.re"
-#line 1739 "ext/date/lib/parse_date.re"
++#line 1746 "ext/date/lib/parse_date.re"
{
timelib_sll w, d;
DEBUG_OUTPUT("isoweek");
@@@ -19852,7 -19862,7 +19869,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_WEEK;
}
- #line 19856 "ext/date/lib/parse_date.c"
-#line 19866 "ext/date/lib/parse_date.c"
++#line 19873 "ext/date/lib/parse_date.c"
yy943:
YYDEBUG(943, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -20328,7 -20338,7 +20345,7 @@@ yy981
if (yych == 'e') goto yy1094;
yy982:
YYDEBUG(982, *YYCURSOR);
- #line 1872 "ext/date/lib/parse_date.re"
-#line 1882 "ext/date/lib/parse_date.re"
++#line 1889 "ext/date/lib/parse_date.re"
{
timelib_sll i;
int behavior = 0;
@@@ -20344,7 -20354,7 +20361,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 20348 "ext/date/lib/parse_date.c"
-#line 20358 "ext/date/lib/parse_date.c"
++#line 20365 "ext/date/lib/parse_date.c"
yy983:
YYDEBUG(983, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -20691,7 -20701,7 +20708,7 @@@ yy1020
YYDEBUG(1020, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1021, *YYCURSOR);
- #line 1578 "ext/date/lib/parse_date.re"
-#line 1588 "ext/date/lib/parse_date.re"
++#line 1595 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("pointed date YYYY");
TIMELIB_INIT;
@@@ -20702,7 -20712,7 +20719,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_FULL_POINTED;
}
- #line 20706 "ext/date/lib/parse_date.c"
-#line 20716 "ext/date/lib/parse_date.c"
++#line 20723 "ext/date/lib/parse_date.c"
yy1022:
YYDEBUG(1022, *YYCURSOR);
++YYCURSOR;
@@@ -20731,7 -20741,7 +20748,7 @@@ yy1024
}
yy1025:
YYDEBUG(1025, *YYCURSOR);
- #line 1509 "ext/date/lib/parse_date.re"
-#line 1519 "ext/date/lib/parse_date.re"
++#line 1526 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("iso8601date2");
@@@ -20744,7 -20754,7 +20761,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_DATE;
}
- #line 20748 "ext/date/lib/parse_date.c"
-#line 20758 "ext/date/lib/parse_date.c"
++#line 20765 "ext/date/lib/parse_date.c"
yy1026:
YYDEBUG(1026, *YYCURSOR);
yyaccept = 15;
@@@ -20964,7 -20974,7 +20981,7 @@@ yy1042
}
yy1043:
YYDEBUG(1043, *YYCURSOR);
- #line 1497 "ext/date/lib/parse_date.re"
-#line 1507 "ext/date/lib/parse_date.re"
++#line 1514 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("iso8601date4 | iso8601date2 | iso8601dateslash | dateslash");
TIMELIB_INIT;
@@@ -20975,7 -20985,7 +20992,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_DATE;
}
- #line 20979 "ext/date/lib/parse_date.c"
-#line 20989 "ext/date/lib/parse_date.c"
++#line 20996 "ext/date/lib/parse_date.c"
yy1044:
YYDEBUG(1044, *YYCURSOR);
yyaccept = 26;
@@@ -21090,7 -21100,7 +21107,7 @@@ yy1047
}
yy1048:
YYDEBUG(1048, *YYCURSOR);
- #line 1658 "ext/date/lib/parse_date.re"
-#line 1668 "ext/date/lib/parse_date.re"
++#line 1675 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("datenocolon");
TIMELIB_INIT;
@@@ -21101,7 -21111,7 +21118,7 @@@
TIMELIB_DEINIT;
return TIMELIB_DATE_NOCOLON;
}
- #line 21105 "ext/date/lib/parse_date.c"
-#line 21115 "ext/date/lib/parse_date.c"
++#line 21122 "ext/date/lib/parse_date.c"
yy1049:
YYDEBUG(1049, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -21171,7 -21181,7 +21188,7 @@@ yy1059
YYDEBUG(1059, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1060, *YYCURSOR);
- #line 1710 "ext/date/lib/parse_date.re"
-#line 1720 "ext/date/lib/parse_date.re"
++#line 1727 "ext/date/lib/parse_date.re"
{
timelib_sll w, d;
DEBUG_OUTPUT("isoweekday");
@@@ -21189,7 -21199,7 +21206,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_WEEK;
}
- #line 21193 "ext/date/lib/parse_date.c"
-#line 21203 "ext/date/lib/parse_date.c"
++#line 21210 "ext/date/lib/parse_date.c"
yy1061:
YYDEBUG(1061, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -21252,7 -21262,7 +21269,7 @@@ yy1069
if (yych <= '9') goto yy1143;
yy1070:
YYDEBUG(1070, *YYCURSOR);
- #line 1748 "ext/date/lib/parse_date.re"
-#line 1758 "ext/date/lib/parse_date.re"
++#line 1765 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("pgtextshort");
@@@ -21265,7 -21275,7 +21282,7 @@@
TIMELIB_DEINIT;
return TIMELIB_PG_TEXT;
}
- #line 21269 "ext/date/lib/parse_date.c"
-#line 21279 "ext/date/lib/parse_date.c"
++#line 21286 "ext/date/lib/parse_date.c"
yy1071:
YYDEBUG(1071, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -21738,7 -21748,7 +21755,7 @@@ yy1106
++YYCURSOR;
yy1107:
YYDEBUG(1107, *YYCURSOR);
- #line 1209 "ext/date/lib/parse_date.re"
-#line 1219 "ext/date/lib/parse_date.re"
++#line 1226 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("tomorrow");
TIMELIB_INIT;
@@@ -21749,7 -21759,7 +21766,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 21753 "ext/date/lib/parse_date.c"
-#line 21763 "ext/date/lib/parse_date.c"
++#line 21770 "ext/date/lib/parse_date.c"
yy1108:
YYDEBUG(1108, *YYCURSOR);
yyaccept = 28;
@@@ -22086,7 -22096,7 +22103,7 @@@ yy1140
YYDEBUG(1140, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1141, *YYCURSOR);
- #line 1762 "ext/date/lib/parse_date.re"
-#line 1772 "ext/date/lib/parse_date.re"
++#line 1779 "ext/date/lib/parse_date.re"
{
int length = 0;
DEBUG_OUTPUT("pgtextreverse");
@@@ -22099,7 -22109,7 +22116,7 @@@
TIMELIB_DEINIT;
return TIMELIB_PG_TEXT;
}
- #line 22103 "ext/date/lib/parse_date.c"
-#line 22113 "ext/date/lib/parse_date.c"
++#line 22120 "ext/date/lib/parse_date.c"
yy1142:
YYDEBUG(1142, *YYCURSOR);
++YYCURSOR;
@@@ -22143,7 -22153,7 +22160,7 @@@ yy1144
}
yy1145:
YYDEBUG(1145, *YYCURSOR);
- #line 1304 "ext/date/lib/parse_date.re"
-#line 1314 "ext/date/lib/parse_date.re"
++#line 1321 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("backof | frontof");
TIMELIB_INIT;
@@@ -22165,7 -22175,7 +22182,7 @@@
TIMELIB_DEINIT;
return TIMELIB_LF_DAY_OF_MONTH;
}
- #line 22169 "ext/date/lib/parse_date.c"
-#line 22179 "ext/date/lib/parse_date.c"
++#line 22186 "ext/date/lib/parse_date.c"
yy1146:
YYDEBUG(1146, *YYCURSOR);
yyaccept = 29;
@@@ -22489,7 -22499,7 +22506,7 @@@ yy1171
}
yy1172:
YYDEBUG(1172, *YYCURSOR);
- #line 1848 "ext/date/lib/parse_date.re"
-#line 1858 "ext/date/lib/parse_date.re"
++#line 1865 "ext/date/lib/parse_date.re"
{
timelib_sll i;
int behavior = 0;
@@@ -22512,7 -22522,7 +22529,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 22516 "ext/date/lib/parse_date.c"
-#line 22526 "ext/date/lib/parse_date.c"
++#line 22533 "ext/date/lib/parse_date.c"
yy1173:
YYDEBUG(1173, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -22524,7 -22534,7 +22541,7 @@@ yy1174
++YYCURSOR;
yy1175:
YYDEBUG(1175, *YYCURSOR);
- #line 1166 "ext/date/lib/parse_date.re"
-#line 1176 "ext/date/lib/parse_date.re"
++#line 1183 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("yesterday");
TIMELIB_INIT;
@@@ -22535,7 -22545,7 +22552,7 @@@
TIMELIB_DEINIT;
return TIMELIB_RELATIVE;
}
- #line 22539 "ext/date/lib/parse_date.c"
-#line 22549 "ext/date/lib/parse_date.c"
++#line 22556 "ext/date/lib/parse_date.c"
yy1176:
YYDEBUG(1176, *YYCURSOR);
yyaccept = 31;
@@@ -23028,7 -23038,7 +23045,7 @@@ yy1222
YYDEBUG(1222, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1223, *YYCURSOR);
- #line 1914 "ext/date/lib/parse_date.re"
-#line 1924 "ext/date/lib/parse_date.re"
++#line 1931 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("dateshortwithtimeshort12 | dateshortwithtimelong12");
TIMELIB_INIT;
@@@ -23051,7 -23061,7 +23068,7 @@@
TIMELIB_DEINIT;
return TIMELIB_SHORTDATE_WITH_TIME;
}
- #line 23055 "ext/date/lib/parse_date.c"
-#line 23065 "ext/date/lib/parse_date.c"
++#line 23072 "ext/date/lib/parse_date.c"
yy1224:
YYDEBUG(1224, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -23553,7 -23563,7 +23570,7 @@@ yy1268
YYDEBUG(1268, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1269, *YYCURSOR);
- #line 1327 "ext/date/lib/parse_date.re"
-#line 1337 "ext/date/lib/parse_date.re"
++#line 1344 "ext/date/lib/parse_date.re"
{
timelib_sll i;
int behavior = 0;
@@@ -23574,7 -23584,7 +23591,7 @@@
TIMELIB_DEINIT;
return TIMELIB_WEEK_DAY_OF_MONTH;
}
- #line 23578 "ext/date/lib/parse_date.c"
-#line 23588 "ext/date/lib/parse_date.c"
++#line 23595 "ext/date/lib/parse_date.c"
yy1270:
YYDEBUG(1270, *YYCURSOR);
yyaccept = 24;
@@@ -23621,7 -23631,7 +23638,7 @@@ yy1273
YYDEBUG(1273, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1274, *YYCURSOR);
- #line 1287 "ext/date/lib/parse_date.re"
-#line 1297 "ext/date/lib/parse_date.re"
++#line 1304 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("firstdayof | lastdayof");
TIMELIB_INIT;
@@@ -23637,12 -23647,12 +23654,12 @@@
TIMELIB_DEINIT;
return TIMELIB_LF_DAY_OF_MONTH;
}
- #line 23641 "ext/date/lib/parse_date.c"
-#line 23651 "ext/date/lib/parse_date.c"
++#line 23658 "ext/date/lib/parse_date.c"
yy1275:
YYDEBUG(1275, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1276, *YYCURSOR);
- #line 1523 "ext/date/lib/parse_date.re"
-#line 1533 "ext/date/lib/parse_date.re"
++#line 1540 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("iso8601datex");
TIMELIB_INIT;
@@@ -23653,7 -23663,7 +23670,7 @@@
TIMELIB_DEINIT;
return TIMELIB_ISO_DATE;
}
- #line 23657 "ext/date/lib/parse_date.c"
-#line 23667 "ext/date/lib/parse_date.c"
++#line 23674 "ext/date/lib/parse_date.c"
yy1277:
YYDEBUG(1277, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -23756,7 -23766,7 +23773,7 @@@ yy1290
YYDEBUG(1290, *YYCURSOR);
++YYCURSOR;
YYDEBUG(1291, *YYCURSOR);
- #line 1367 "ext/date/lib/parse_date.re"
-#line 1377 "ext/date/lib/parse_date.re"
++#line 1384 "ext/date/lib/parse_date.re"
{
DEBUG_OUTPUT("mssqltime");
TIMELIB_INIT;
@@@ -23775,7 -23785,7 +23792,7 @@@
TIMELIB_DEINIT;
return TIMELIB_TIME24_WITH_ZONE;
}
- #line 23779 "ext/date/lib/parse_date.c"
-#line 23789 "ext/date/lib/parse_date.c"
++#line 23796 "ext/date/lib/parse_date.c"
yy1292:
YYDEBUG(1292, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -24199,7 -24209,7 +24216,7 @@@ yy1328
if (yych <= '9') goto yy1331;
yy1329:
YYDEBUG(1329, *YYCURSOR);
- #line 1670 "ext/date/lib/parse_date.re"
-#line 1680 "ext/date/lib/parse_date.re"
++#line 1687 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("xmlrpc | xmlrpcnocolon | soap | wddx | exif");
@@@ -24224,7 -24234,7 +24241,7 @@@
TIMELIB_DEINIT;
return TIMELIB_XMLRPC_SOAP;
}
- #line 24228 "ext/date/lib/parse_date.c"
-#line 24238 "ext/date/lib/parse_date.c"
++#line 24245 "ext/date/lib/parse_date.c"
yy1330:
YYDEBUG(1330, *YYCURSOR);
yych = *++YYCURSOR;
@@@ -24594,7 -24604,7 +24611,7 @@@ yy1374
if (yych <= ':') goto yy1383;
yy1375:
YYDEBUG(1375, *YYCURSOR);
- #line 1776 "ext/date/lib/parse_date.re"
-#line 1786 "ext/date/lib/parse_date.re"
++#line 1793 "ext/date/lib/parse_date.re"
{
int tz_not_found;
DEBUG_OUTPUT("clf");
@@@ -24617,7 -24627,7 +24634,7 @@@
TIMELIB_DEINIT;
return TIMELIB_CLF;
}
- #line 24621 "ext/date/lib/parse_date.c"
-#line 24631 "ext/date/lib/parse_date.c"
++#line 24638 "ext/date/lib/parse_date.c"
yy1376:
YYDEBUG(1376, *YYCURSOR);
yyaccept = 33;
@@@ -24849,7 -24859,7 +24866,7 @@@ yy1406
if (yych == ':') goto yy1286;
goto yy1329;
}
- #line 2004 "ext/date/lib/parse_date.re"
-#line 2014 "ext/date/lib/parse_date.re"
++#line 2021 "ext/date/lib/parse_date.re"
}
diff --cc ext/date/tests/oom_dateperiod_construct.phpt
index 00000000000,e7eda58d756..f917827f57a
mode 000000,100644..100644
--- a/ext/date/tests/oom_dateperiod_construct.phpt
+++ b/ext/date/tests/oom_dateperiod_construct.phpt
@@@ -1,0 -1,18 +1,19 @@@
+ --TEST--
+ DatePeriod::__construct() doesn't reach hard limit
+ --INI--
+ memory_limit=64M
+ --SKIPIF--
+ <?php
+ if (getenv('SKIP_ASAN')) die("skip ASan-instrumented realloc() is slow");
+ ?>
+ --FILE--
+ <?php
+ try {
+ new DatePeriod(str_repeat("\x01", 4 * 1024 * 1024));
+ } catch (\Throwable $e) {
+ echo $e::class, ': ', $e->getMessage(), "\n";
+ }
+ ?>
+ --EXPECTF--
++Deprecated: Calling DatePeriod::__construct(string $isostr, int $options = 0) is deprecated, use DatePeriod::createFromISO8601String() instead in %s on line %d
+ DateMalformedPeriodStringException: Unknown or bad format (%s)
diff --cc ext/dom/xpath.c
index 64161da640b,9ff273cd50e..c3d15fe44dd
--- a/ext/dom/xpath.c
+++ b/ext/dom/xpath.c
@@@ -344,13 -419,16 +344,15 @@@ static void php_xpath_eval(INTERNAL_FUN
case XPATH_NODESET:
{
- int i;
xmlNodeSetPtr nodesetp;
+ zval retval;
if (xpathobjp->type == XPATH_NODESET && NULL != (nodesetp = xpathobjp->nodesetval) && nodesetp->nodeNr) {
-
array_init_size(&retval, nodesetp->nodeNr);
- zend_hash_real_init_packed(Z_ARRVAL_P(&retval));
+ if (zend_hash_real_finit_packed(Z_ARRVAL(retval), /* fallible */ true) != SUCCESS) {
+ goto oom_nodeset;
+ }
- for (i = 0; i < nodesetp->nodeNr; i++) {
+ for (int i = 0; i < nodesetp->nodeNr; i++) {
xmlNodePtr node = nodesetp->nodeTab[i];
zval child;
diff --cc ext/exif/exif.c
index 1b57377ffb8,571e3234c84..a72acb4f2c5
--- a/ext/exif/exif.c
+++ b/ext/exif/exif.c
@@@ -2260,9 -2267,13 +2263,13 @@@ static void exif_iif_add_value(image_in
case TAG_FMT_DOUBLE:
if (length==0) {
break;
- } else
+ }
if (length>1) {
- info_value->list = safe_emalloc(length, sizeof(image_info_value), 0);
+ info_value->list = safe_femalloc(length, sizeof(image_info_value), 0, /* fallible */ true);
+ if (!info_value->list) {
+ info_data->length = 0;
+ return;
+ }
} else {
info_value = &info_data->value;
}
diff --cc ext/fileinfo/libmagic.patch
index 49ba17b76c3,efed868b885..ed95a40221c
--- a/ext/fileinfo/libmagic.patch
+++ b/ext/fileinfo/libmagic.patch
@@@ -1,6 -1,6 +1,6 @@@
diff -u libmagic.orig/apprentice.c libmagic/apprentice.c
---- libmagic.orig/apprentice.c 2022-09-13 20:46:07.000000000 +0200
-+++ libmagic/apprentice.c 2026-09-30 14:44:16.056910654 +0200
+--- libmagic.orig/apprentice.c 2023-07-17 16:38:35.000000000 +0200
- +++ libmagic/apprentice.c 2024-06-09 00:31:40.345830732 +0200
+++++ libmagic/apprentice.c 2026-09-30 22:55:28.668770846 +0200
@@ -48,7 +48,9 @@
#ifdef QUICK
#include <sys/mman.h>
@@@ -11,27 -11,33 +11,33 @@@
#include <limits.h>
#ifdef HAVE_BYTESWAP_H
#include <byteswap.h>
-@@ -147,10 +149,7 @@
+@@ -146,10 +148,7 @@
#endif
- private char *mkdbname(struct magic_set *, const char *, int);
--private struct magic_map *apprentice_buf(struct magic_set *, struct magic *,
+ file_private char *mkdbname(struct magic_set *, const char *, int);
+-file_private struct magic_map *apprentice_buf(struct magic_set *, struct magic *,
- size_t);
- private struct magic_map *apprentice_map(struct magic_set *, const char *);
--private int check_buffer(struct magic_set *, struct magic_map *, const char *);
- private void apprentice_unmap(struct magic_map *);
- private int apprentice_compile(struct magic_set *, struct magic_map *,
+ file_private struct magic_map *apprentice_map(struct magic_set *, const char *);
+-file_private int check_buffer(struct magic_set *, struct magic_map *, const char *);
+ file_private void apprentice_unmap(struct magic_map *);
+ file_private int apprentice_compile(struct magic_set *, struct magic_map *,
const char *);
- @@ -185,6 +184,8 @@
-@@ -186,6 +185,14 @@
++@@ -185,6 +184,14 @@
{ NULL, 0, NULL }
};
-+#if __has_attribute(nonstring)
+++#if __has_attribute(nonstring) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 15
+ +# define ZEND_NONSTRING __attribute__((nonstring))
+ +#else
+ +# define ZEND_NONSTRING
+ +#endif
+ +
+#include "../data_file.c"
+
#ifdef COMPILE_ONLY
int main(int, char *[]);
- @@ -445,21 +446,13 @@
-@@ -446,21 +453,13 @@
++@@ -445,21 +452,13 @@
struct mlist *ml;
mlp->map = NULL;
@@@ -55,7 -61,7 +61,7 @@@
mlp->prev->next = ml;
ml->prev = mlp->prev;
ml->next = mlp;
- @@ -537,13 +530,19 @@
-@@ -539,13 +538,19 @@
++@@ -537,13 +536,19 @@
return;
for (i = 0; i < MAGIC_SETS; i++)
mlist_free(ms->mlist[i]);
@@@ -78,17 -84,17 +84,17 @@@
+ efree(ms);
}
- protected struct magic_set *
-@@ -554,7 +559,7 @@
+ file_protected struct magic_set *
- @@ -552,7 +551,7 @@
++@@ -552,7 +557,7 @@
struct magic_set *ms;
size_t i, len;
- if ((ms = CAST(struct magic_set *, calloc(CAST(size_t, 1u),
+ if ((ms = CAST(struct magic_set *, ecalloc(CAST(size_t, 1u),
- sizeof(struct magic_set)))) == NULL)
+ sizeof(*ms)))) == NULL)
return NULL;
- @@ -565,7 +564,7 @@
-@@ -567,7 +572,7 @@
++@@ -565,7 +570,7 @@
ms->o.blen = 0;
len = (ms->c.len = 10) * sizeof(*ms->c.li);
@@@ -97,16 -103,8 +103,16 @@@
goto free;
ms->event_flags = 0;
- @@ -589,49 +588,35 @@
-@@ -597,42 +602,28 @@
- private void
++@@ -589,49 +594,35 @@
+ #endif
+ return ms;
+ free:
+- free(ms);
++ efree(ms);
+ return NULL;
+ }
+
+ file_private void
apprentice_unmap(struct magic_map *map)
{
- size_t i;
@@@ -160,8 -158,8 +166,8 @@@
return NULL;
}
mlist->next = mlist->prev = mlist;
- @@ -652,21 +637,9 @@
-@@ -653,21 +644,9 @@
- private void
++@@ -652,21 +643,9 @@
+ file_private void
mlist_free_one(struct mlist *ml)
{
- size_t i;
@@@ -182,8 -180,8 +188,8 @@@
+ efree(ml);
}
- private void
-@@ -686,51 +665,6 @@
+ file_private void
- @@ -685,50 +658,6 @@
++@@ -685,50 +664,6 @@
mlist_free_one(mlist);
}
@@@ -232,9 -231,9 +238,9 @@@
-#endif
-
/* const char *fn: list of magic files and directories */
- protected int
+ file_protected int
file_apprentice(struct magic_set *ms, const char *fn, int action)
- @@ -739,12 +668,28 @@
-@@ -741,12 +675,28 @@
++@@ -739,12 +674,28 @@
(void)file_reset(ms, 0);
@@@ -264,7 -263,7 +270,7 @@@
file_oomem(ms, strlen(fn));
return -1;
}
- @@ -757,7 +702,7 @@
-@@ -759,7 +709,7 @@
++@@ -757,7 +708,7 @@
mlist_free(ms->mlist[j]);
ms->mlist[j] = NULL;
}
@@@ -273,7 -272,7 +279,7 @@@
return -1;
}
}
- @@ -774,7 +719,7 @@
-@@ -776,7 +726,7 @@
++@@ -774,7 +725,7 @@
fn = p;
}
@@@ -282,7 -281,7 +288,7 @@@
if (errs == -1) {
for (i = 0; i < MAGIC_SETS; i++) {
- @@ -1289,7 +1234,7 @@
-@@ -1289,7 +1239,7 @@
++@@ -1289,7 +1240,7 @@
size_t incr = mset[i].max + ALLOC_INCR;
if ((mp = CAST(struct magic_entry *,
@@@ -291,7 -290,7 +297,7 @@@
NULL) {
file_oomem(ms, sizeof(*mp) * incr);
return -1;
- @@ -1312,13 +1257,19 @@
-@@ -1312,13 +1262,19 @@
++@@ -1312,13 +1263,19 @@
load_1(struct magic_set *ms, int action, const char *fn, int *errs,
struct magic_entry_set *mset)
{
@@@ -315,7 -314,7 +321,7 @@@
if (errno != ENOENT)
file_error(ms, errno, "cannot read magic file `%s'",
fn);
- @@ -1328,8 +1279,7 @@
-@@ -1328,8 +1284,7 @@
++@@ -1328,8 +1285,7 @@
memset(&me, 0, sizeof(me));
/* read and parse this file */
@@@ -325,7 -324,7 +331,7 @@@
if (len == 0) /* null line, garbage, etc */
continue;
if (line[len - 1] == '\n') {
- @@ -1388,8 +1338,8 @@
-@@ -1388,8 +1343,8 @@
++@@ -1388,8 +1344,8 @@
}
if (me.mp)
(void)addentry(ms, &me, mset);
@@@ -336,7 -335,7 +342,7 @@@
}
/*
- @@ -1474,7 +1424,7 @@
-@@ -1474,7 +1429,7 @@
++@@ -1474,7 +1430,7 @@
}
slen = sizeof(**ma) * mentrycount;
@@@ -345,7 -344,7 +351,7 @@@
file_oomem(ms, slen);
return -1;
}
- @@ -1496,8 +1446,8 @@
-@@ -1496,8 +1451,8 @@
++@@ -1496,8 +1452,8 @@
if (me == NULL)
return;
for (i = 0; i < nme; i++)
@@@ -355,8 -354,8 +361,8 @@@
+ efree(me);
}
- private struct magic_map *
-@@ -1506,18 +1461,19 @@
+ file_private struct magic_map *
- @@ -1506,18 +1456,19 @@
++@@ -1506,18 +1462,19 @@
int errs = 0;
uint32_t i, j;
size_t files = 0, maxfiles = 0;
@@@ -381,7 -380,7 +387,7 @@@
{
file_oomem(ms, sizeof(*map));
return NULL;
- @@ -1529,52 +1480,50 @@
-@@ -1529,52 +1485,50 @@
++@@ -1529,52 +1486,50 @@
(void)fprintf(stderr, "%s\n", usg_hdr);
/* load directory or file */
@@@ -453,7 -452,7 +459,7 @@@
}
} else
load_1(ms, action, fn, &errs, mset);
- @@ -1612,7 +1561,6 @@
-@@ -1612,7 +1566,6 @@
++@@ -1612,7 +1567,6 @@
}
out:
@@@ -461,7 -460,7 +467,7 @@@
for (j = 0; j < MAGIC_SETS; j++)
magic_entry_free(mset[j].me, mset[j].count);
- @@ -2060,7 +2008,7 @@
-@@ -2060,7 +2013,7 @@
++@@ -2060,7 +2014,7 @@
if (me->cont_count == me->max_count) {
struct magic *nm;
size_t cnt = me->max_count + ALLOC_CHUNK;
@@@ -470,7 -469,7 +476,7 @@@
sizeof(*nm) * cnt))) == NULL) {
file_oomem(ms, sizeof(*nm) * cnt);
return -1;
- @@ -2075,7 +2023,7 @@
-@@ -2075,7 +2028,7 @@
++@@ -2075,7 +2029,7 @@
static const size_t len = sizeof(*m) * ALLOC_CHUNK;
if (me->mp != NULL)
return 1;
@@@ -479,7 -478,7 +485,7 @@@
file_oomem(ms, len);
return -1;
}
- @@ -2301,7 +2249,7 @@
-@@ -2301,7 +2254,7 @@
++@@ -2301,7 +2255,7 @@
m->mask_op = 0;
if (*l == '~') {
@@@ -488,7 -487,7 +494,7 @@@
m->mask_op |= FILE_OPINVERSE;
else if (ms->flags & MAGIC_CHECK)
file_magwarn(ms, "'~' invalid for string types");
- @@ -2310,7 +2258,7 @@
-@@ -2310,7 +2263,7 @@
++@@ -2310,7 +2264,7 @@
m->str_range = 0;
m->str_flags = m->type == FILE_PSTRING ? PSTRING_1_LE : 0;
if ((op = get_op(*l)) != -1) {
@@@ -497,16 -496,17 +503,16 @@@
int r;
if (op != FILE_OPDIVIDE) {
- @@ -2497,7 +2445,7 @@
-@@ -2493,8 +2446,7 @@
++@@ -2497,7 +2451,7 @@
- private int
+ file_private int
parse_extra(struct magic_set *ms, struct magic_entry *me, const char *line,
- size_t llen, off_t off, size_t len, const char *name, const char *extra,
-- int nt)
-+ size_t llen, zend_off_t off, size_t len, const char *name, const char *extra, int nt)
++ size_t llen, zend_off_t off, size_t len, const char *name, const char *extra,
+ int nt)
{
size_t i;
- @@ -2860,13 +2808,19 @@
- const char *l = line;
-@@ -2852,13 +2804,19 @@
++@@ -2860,13 +2814,19 @@
return -1;
}
if (m->type == FILE_REGEX) {
@@@ -532,7 -532,7 +538,7 @@@
}
return 0;
default:
- @@ -3245,120 +3199,83 @@
-@@ -3230,120 +3188,83 @@
++@@ -3245,120 +3205,83 @@
}
/*
@@@ -576,10 -576,10 +582,10 @@@
+ php_stream *stream = NULL;
+ php_stream_statbuf st;
+
--+
- fd = -1;
- if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
+++
+ if ((map = CAST(struct magic_map *, ecalloc(1, sizeof(*map)))) == NULL) {
file_oomem(ms, sizeof(*map));
- goto error;
@@@ -674,8 -676,8 +682,8 @@@
- free(dbname);
- return rv;
-}
-
+ -
--private int
+-file_private int
-check_buffer(struct magic_set *ms, struct magic_map *map, const char *dbname)
-{
- uint32_t *ptr;
@@@ -696,7 -696,7 +702,7 @@@
}
needsbyteswap = 1;
} else
- @@ -3368,17 +3285,29 @@
-@@ -3353,17 +3274,29 @@
++@@ -3368,17 +3291,29 @@
else
version = ptr[1];
if (version != VERSIONNO) {
@@@ -735,7 -735,7 +741,7 @@@
}
map->magic[0] = CAST(struct magic *, map->p) + 1;
nentries = 0;
- @@ -3391,15 +3320,29 @@
-@@ -3376,15 +3309,29 @@
++@@ -3391,15 +3326,29 @@
map->magic[i + 1] = map->magic[i] + map->nmagic[i];
nentries += map->nmagic[i];
}
@@@ -768,7 -768,7 +774,7 @@@
}
/*
- @@ -3410,7 +3353,7 @@
-@@ -3395,7 +3342,7 @@
++@@ -3410,7 +3359,7 @@
{
static const size_t nm = sizeof(*map->nmagic) * MAGIC_SETS;
static const size_t m = sizeof(**map->magic);
@@@ -777,7 -777,7 +783,7 @@@
size_t len;
char *dbname;
int rv = -1;
- @@ -3425,8 +3368,10 @@
-@@ -3410,8 +3357,10 @@
++@@ -3425,8 +3374,10 @@
if (dbname == NULL)
goto out;
@@@ -790,7 -790,7 +796,7 @@@
file_error(ms, errno, "cannot open `%s'", dbname);
goto out;
}
- @@ -3435,26 +3380,25 @@
-@@ -3420,26 +3369,25 @@
++@@ -3435,26 +3386,25 @@
hdr.h[1] = VERSIONNO;
memcpy(hdr.h + 2, map->nmagic, nm);
@@@ -825,7 -825,7 +831,7 @@@
return rv;
}
- @@ -3488,17 +3432,18 @@
-@@ -3473,17 +3421,18 @@
++@@ -3488,17 +3438,18 @@
q++;
/* Compatibility with old code that looked in .mime */
if (ms->flags & MAGIC_MIME) {
@@@ -851,7 -851,7 +857,7 @@@
/* Compatibility with old code that looked in .mime */
if (strstr(fn, ".mime") != NULL)
- @@ -3620,7 +3565,7 @@
-@@ -3605,7 +3554,7 @@
++@@ -3620,7 +3571,7 @@
m->offset = swap4(CAST(uint32_t, m->offset));
m->in_offset = swap4(CAST(uint32_t, m->in_offset));
m->lineno = swap4(CAST(uint32_t, m->lineno));
@@@ -860,7 -860,7 +866,7 @@@
m->str_range = swap4(m->str_range);
m->str_flags = swap4(m->str_flags);
}
- @@ -3717,7 +3662,6 @@
-@@ -3702,7 +3651,6 @@
++@@ -3717,7 +3668,6 @@
continue;
if (strcmp(ma[i].value.s, name) == 0) {
v->magic = &ma[i];
@@@ -970,8 -974,8 +976,8 @@@ diff -u libmagic.orig/buffer.c libmagic
goto out;
}
diff -u libmagic.orig/cdf.c libmagic/cdf.c
---- libmagic.orig/cdf.c 2022-09-13 20:34:25.000000000 +0200
-+++ libmagic/cdf.c 2026-10-02 19:34:05.613268663 +0200
+--- libmagic.orig/cdf.c 2022-09-24 22:56:49.000000000 +0200
- +++ libmagic/cdf.c 2024-06-09 00:31:40.345830732 +0200
+++++ libmagic/cdf.c 2026-10-02 19:35:01.765068091 +0200
@@ -43,7 +43,9 @@
#include <err.h>
#endif
diff --cc ext/filter/filter.c
index c65f9ac7994,605196e33ae..7e1a0311099
--- a/ext/filter/filter.c
+++ b/ext/filter/filter.c
@@@ -349,14 -361,23 +357,26 @@@ static unsigned int php_sapi_filter(in
}
if (retval) {
+ zend_string *tmp_str;
+ zend_string *str = zval_get_tmp_string(&new_var, &tmp_str);
if (new_val_len) {
- *new_val_len = Z_STRLEN(new_var);
+ *new_val_len = ZSTR_LEN(str);
}
efree(*val);
- if (Z_STRLEN(new_var)) {
- *val = femalloc(Z_STRLEN(new_var) + 1, /* fallible */ true);
+ if (ZSTR_LEN(str)) {
- *val = estrndup(ZSTR_VAL(str), ZSTR_LEN(str));
++ *val = femalloc(ZSTR_LEN(str) + 1, /* fallible */ true);
+ if (UNEXPECTED(!*val)) {
+ /* The caller frees *val either way, so leave it pointing at
+ * something. */
+ *val = estrdup("");
+ if (new_val_len) {
+ *new_val_len = 0;
+ }
++ zend_tmp_string_release(tmp_str);
+ zval_ptr_dtor(&new_var);
+ return 0;
+ }
- memcpy(*val, Z_STRVAL(new_var), Z_STRLEN(new_var) + 1);
++ memcpy(*val, ZSTR_VAL(str), ZSTR_LEN(str) + 1);
} else {
*val = estrdup("");
}
diff --cc ext/hash/tests/hash_pbkdf2_max_length.phpt
index eccd8abaa2b,00000000000..ac9815220e2
mode 100644,000000..100644
--- a/ext/hash/tests/hash_pbkdf2_max_length.phpt
+++ b/ext/hash/tests/hash_pbkdf2_max_length.phpt
@@@ -1,17 -1,0 +1,19 @@@
+--TEST--
+Hash: hash_pbkdf2() function : output length of PHP_INT_MAX
+--SKIPIF--
+<?php
+if (PHP_INT_SIZE == 4) die("skip this test is not for 32bit platforms");
+if (getenv("USE_ZEND_ALLOC") === "0") die("skip Zend MM disabled");
+?>
+--INI--
+memory_limit=128M
+--FILE--
+<?php
+
+hash_pbkdf2('md5', 'password', 'salt', 1, PHP_INT_MAX);
+
+?>
+--EXPECTF--
- Fatal error: Allowed memory size of 134217728 bytes exhausted%s(tried to allocate 4611686018427387904 bytes) in %s on line %d
++Hard memory limit of 268435456 bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
++
++Termsig=9
diff --cc ext/hash/tests/hash_pbkdf2_max_length_raw.phpt
index b0eceefa356,00000000000..2a31321f84c
mode 100644,000000..100644
--- a/ext/hash/tests/hash_pbkdf2_max_length_raw.phpt
+++ b/ext/hash/tests/hash_pbkdf2_max_length_raw.phpt
@@@ -1,17 -1,0 +1,19 @@@
+--TEST--
+Hash: hash_pbkdf2() function : raw output length of PHP_INT_MAX
+--SKIPIF--
+<?php
+if (PHP_INT_SIZE == 4) die("skip this test is not for 32bit platforms");
+if (getenv("USE_ZEND_ALLOC") === "0") die("skip Zend MM disabled");
+?>
+--INI--
+memory_limit=128M
+--FILE--
+<?php
+
+hash_pbkdf2('md5', 'password', 'salt', 1, PHP_INT_MAX, true);
+
+?>
+--EXPECTF--
- Fatal error: Allowed memory size of 134217728 bytes exhausted%s(tried to allocate 9223372036854775808 bytes) in %s on line %d
++Hard memory limit of 268435456 bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
++
++Termsig=9
diff --cc ext/iconv/iconv.c
index 29998c37d1a,cc0bba205b7..0212d1063c7
--- a/ext/iconv/iconv.c
+++ b/ext/iconv/iconv.c
@@@ -441,7 -446,7 +445,7 @@@ PHP_ICONV_API php_iconv_err_t php_iconv
char *out_p;
size_t bsz, result = 0;
php_iconv_err_t retval = PHP_ICONV_ERR_SUCCESS;
-- zend_string *out_buf;
++ zend_string *out_buf = NULL;
int ignore_ilseq = _php_check_ignore(out_charset);
*out = NULL;
@@@ -459,65 -464,75 +463,83 @@@
out_left = in_len + 32; /* Avoid realloc() most cases */
out_size = 0;
bsz = out_left;
- out_buf = zend_string_falloc(bsz, 0, /* fallible */ true);
- if (UNEXPECTED(out_buf == NULL)) {
- iconv_close(cd);
- return PHP_ICONV_ERR_ALLOC;
- }
- out_p = ZSTR_VAL(out_buf);
- while (in_left > 0) {
- result = iconv(cd, (ICONV_CONST char **) &in_p, &in_left, (char **) &out_p, &out_left);
- out_size = bsz - out_left;
- if (result == (size_t)(-1)) {
- if (ignore_ilseq && errno == EILSEQ) {
- if (in_left <= 1) {
- result = 0;
- } else {
- errno = 0;
- in_p++;
- in_left--;
- continue;
+ zend_try {
- out_buf = zend_string_alloc(bsz, 0);
++ out_buf = zend_string_falloc(bsz, 0, /* fallible */ true);
++ if (UNEXPECTED(out_buf == NULL)) {
++ result = -1;
++ errno = 0;
++ goto fail;
++ }
+ out_p = ZSTR_VAL(out_buf);
+
+ while (in_left > 0) {
+ result = iconv(cd, (ICONV_CONST char **) &in_p, &in_left, (char **) &out_p, &out_left);
+ out_size = bsz - out_left;
+ if (result == (size_t)(-1)) {
+ if (ignore_ilseq && errno == EILSEQ) {
+ if (in_left <= 1) {
+ result = 0;
+ } else {
+ errno = 0;
+ in_p++;
+ in_left--;
+ continue;
+ }
}
- }
- if (errno == E2BIG && in_left > 0) {
- /* converted string is longer than out buffer */
- bsz += in_len;
+ if (errno == E2BIG && in_left > 0) {
+ /* converted string is longer than out buffer */
+ bsz += in_len;
- out_buf = zend_string_extend(out_buf, bsz, 0);
- zend_string *grown = zend_string_fextend(out_buf, bsz, 0, /* fallible */ true);
- if (UNEXPECTED(grown == NULL)) {
- zend_string_efree(out_buf);
- iconv_close(cd);
- return PHP_ICONV_ERR_ALLOC;
++ zend_string *grown = zend_string_fextend(out_buf, bsz, 0, /* fallible */ true);
++ if (UNEXPECTED(grown == NULL)) {
++ result = -1;
++ errno = 0;
++ goto fail;
++ }
++ out_buf = grown;
+ out_p = ZSTR_VAL(out_buf);
+ out_p += out_size;
+ out_left = bsz - out_size;
+ continue;
}
- out_buf = grown;
- out_p = ZSTR_VAL(out_buf);
- out_p += out_size;
- out_left = bsz - out_size;
- continue;
}
+ break;
}
- break;
- }
- if (result != (size_t)(-1)) {
- /* flush the shift-out sequences */
- for (;;) {
- result = iconv(cd, NULL, NULL, (char **) &out_p, &out_left);
- out_size = bsz - out_left;
+ if (result != (size_t)(-1)) {
+ /* flush the shift-out sequences */
+ for (;;) {
+ result = iconv(cd, NULL, NULL, (char **) &out_p, &out_left);
+ out_size = bsz - out_left;
- if (result != (size_t)(-1)) {
- break;
- }
+ if (result != (size_t)(-1)) {
+ break;
+ }
- if (errno == E2BIG) {
- bsz += 16;
- zend_string *grown = zend_string_fextend(out_buf, bsz, 0, /* fallible */ true);
- if (UNEXPECTED(grown == NULL)) {
- zend_string_efree(out_buf);
- iconv_close(cd);
- return PHP_ICONV_ERR_ALLOC;
+ if (errno == E2BIG) {
+ bsz += 16;
- out_buf = zend_string_extend(out_buf, bsz, 0);
++ zend_string *grown = zend_string_fextend(out_buf, bsz, 0, /* fallible */ true);
++ if (UNEXPECTED(grown == NULL)) {
++ result = -1;
++ errno = 0;
++ goto fail;
++ }
++ out_buf = grown;
+ out_p = ZSTR_VAL(out_buf);
+ out_p += out_size;
+ out_left = bsz - out_size;
+ } else {
+ break;
}
- out_buf = grown;
- out_p = ZSTR_VAL(out_buf);
- out_p += out_size;
- out_left = bsz - out_size;
- } else {
- break;
}
}
- }
++fail:;
+ } zend_catch {
+ iconv_close(cd);
+ zend_bailout();
+ } zend_end_try();
iconv_close(cd);
@@@ -538,7 -553,7 +560,9 @@@
default:
/* other error */
-- zend_string_efree(out_buf);
++ if (out_buf) {
++ zend_string_efree(out_buf);
++ }
return PHP_ICONV_ERR_UNKNOWN;
}
}
@@@ -977,221 -983,219 +1001,225 @@@ static php_iconv_err_t _php_iconv_mime_
goto out;
}
- buf = safe_femalloc(1, max_line_len, 5, /* fallible */ true);
- if (UNEXPECTED(buf == NULL)) {
- err = PHP_ICONV_ERR_ALLOC;
- goto out;
- }
+ zend_try {
- buf = safe_emalloc(1, max_line_len, 5);
++ buf = safe_femalloc(1, max_line_len, 5, /* fallible */ true);
++ if (UNEXPECTED(buf == NULL)) {
++ err = PHP_ICONV_ERR_ALLOC;
++ goto out_try;
++ }
- char_cnt = max_line_len;
+ char_cnt = max_line_len;
- _php_iconv_appendl(pretval, fname, fname_nbytes, cd_pl);
- char_cnt -= fname_nbytes;
- smart_str_appendl(pretval, ": ", sizeof(": ") - 1);
- char_cnt -= 2;
+ _php_iconv_appendl(pretval, fname, fname_nbytes, cd_pl);
+ char_cnt -= fname_nbytes;
+ smart_str_appendl(pretval, ": ", sizeof(": ") - 1);
+ char_cnt -= 2;
- in_p = fval;
- in_left = fval_nbytes;
+ in_p = fval;
+ in_left = fval_nbytes;
- do {
- size_t prev_in_left;
- size_t out_size;
- size_t encoded_word_min_len = sizeof("=\?\?X\?\?=")-1 + out_charset_len + (enc_scheme == PHP_ICONV_ENC_SCHEME_BASE64 ? 4 : 3);
+ do {
+ size_t prev_in_left;
+ size_t out_size;
+ size_t encoded_word_min_len = sizeof("=\?\?X\?\?=")-1 + out_charset_len + (enc_scheme == PHP_ICONV_ENC_SCHEME_BASE64 ? 4 : 3);
- if (char_cnt < encoded_word_min_len + lfchars_len + 1) {
- /* lfchars must be encoded in ASCII here*/
- smart_str_appendl(pretval, lfchars, lfchars_len);
- smart_str_appendc(pretval, ' ');
- char_cnt = max_line_len - 1;
- }
+ if (char_cnt < encoded_word_min_len + lfchars_len + 1) {
+ /* lfchars must be encoded in ASCII here*/
+ smart_str_appendl(pretval, lfchars, lfchars_len);
+ smart_str_appendc(pretval, ' ');
+ char_cnt = max_line_len - 1;
+ }
- smart_str_appendl(pretval, "=?", sizeof("=?") - 1);
- char_cnt -= 2;
- smart_str_appendl(pretval, out_charset, out_charset_len);
- char_cnt -= out_charset_len;
- smart_str_appendc(pretval, '?');
- char_cnt --;
+ smart_str_appendl(pretval, "=?", sizeof("=?") - 1);
+ char_cnt -= 2;
+ smart_str_appendl(pretval, out_charset, out_charset_len);
+ char_cnt -= out_charset_len;
+ smart_str_appendc(pretval, '?');
+ char_cnt --;
- switch (enc_scheme) {
- case PHP_ICONV_ENC_SCHEME_BASE64: {
- size_t ini_in_left;
- const char *ini_in_p;
- size_t out_reserved = 4;
+ switch (enc_scheme) {
+ case PHP_ICONV_ENC_SCHEME_BASE64: {
+ size_t ini_in_left;
+ const char *ini_in_p;
+ size_t out_reserved = 4;
- smart_str_appendc(pretval, 'B');
- char_cnt--;
- smart_str_appendc(pretval, '?');
- char_cnt--;
+ smart_str_appendc(pretval, 'B');
+ char_cnt--;
+ smart_str_appendc(pretval, '?');
+ char_cnt--;
- prev_in_left = ini_in_left = in_left;
- ini_in_p = in_p;
+ prev_in_left = ini_in_left = in_left;
+ ini_in_p = in_p;
- out_size = (char_cnt - 2) / 4 * 3;
+ out_size = (char_cnt - 2) / 4 * 3;
- for (;;) {
- out_p = buf;
+ for (;;) {
+ out_p = buf;
- if (out_size <= out_reserved) {
- err = PHP_ICONV_ERR_TOO_BIG;
- goto out;
- }
+ if (out_size <= out_reserved) {
+ err = PHP_ICONV_ERR_TOO_BIG;
+ goto out_try;
+ }
- out_left = out_size - out_reserved;
+ out_left = out_size - out_reserved;
- if (iconv(cd, (ICONV_CONST char **)&in_p, &in_left, (char **) &out_p, &out_left) == (size_t)-1) {
- switch (errno) {
- case EINVAL:
- err = PHP_ICONV_ERR_ILLEGAL_CHAR;
- goto out;
+ if (iconv(cd, (ICONV_CONST char **)&in_p, &in_left, (char **) &out_p, &out_left) == (size_t)-1) {
+ switch (errno) {
+ case EINVAL:
+ err = PHP_ICONV_ERR_ILLEGAL_CHAR;
+ goto out_try;
- case EILSEQ:
- err = PHP_ICONV_ERR_ILLEGAL_SEQ;
- goto out;
+ case EILSEQ:
+ err = PHP_ICONV_ERR_ILLEGAL_SEQ;
+ goto out_try;
- case E2BIG:
- if (prev_in_left == in_left) {
- err = PHP_ICONV_ERR_TOO_BIG;
- goto out;
- }
- break;
+ case E2BIG:
+ if (prev_in_left == in_left) {
+ err = PHP_ICONV_ERR_TOO_BIG;
+ goto out_try;
+ }
+ break;
- default:
- err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
+ default:
+ err = PHP_ICONV_ERR_UNKNOWN;
+ goto out_try;
+ }
}
- }
- out_left += out_reserved;
+ out_left += out_reserved;
+
+ if (iconv(cd, NULL, NULL, (char **) &out_p, &out_left) == (size_t)-1) {
+ if (errno != E2BIG) {
+ err = PHP_ICONV_ERR_UNKNOWN;
+ goto out_try;
+ }
+ } else {
+ break;
+ }
- if (iconv(cd, NULL, NULL, (char **) &out_p, &out_left) == (size_t)-1) {
- if (errno != E2BIG) {
+ if (iconv(cd, NULL, NULL, NULL, NULL) == (size_t)-1) {
err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
+ goto out_try;
}
- } else {
- break;
- }
- if (iconv(cd, NULL, NULL, NULL, NULL) == (size_t)-1) {
- err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
+ out_reserved += 4;
+ in_left = ini_in_left;
+ in_p = ini_in_p;
}
- out_reserved += 4;
- in_left = ini_in_left;
- in_p = ini_in_p;
- }
+ prev_in_left = in_left;
- prev_in_left = in_left;
+ encoded = php_base64_encode((unsigned char *) buf, (out_size - out_left));
- encoded = php_base64_encode((unsigned char *) buf, (out_size - out_left));
+ if (char_cnt < ZSTR_LEN(encoded)) {
+ /* something went wrong! */
+ err = PHP_ICONV_ERR_UNKNOWN;
+ goto out_try;
+ }
- if (char_cnt < ZSTR_LEN(encoded)) {
- /* something went wrong! */
- err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
- }
+ smart_str_appendl(pretval, ZSTR_VAL(encoded), ZSTR_LEN(encoded));
+ char_cnt -= ZSTR_LEN(encoded);
+ smart_str_appendl(pretval, "?=", sizeof("?=") - 1);
+ char_cnt -= 2;
- smart_str_appendl(pretval, ZSTR_VAL(encoded), ZSTR_LEN(encoded));
- char_cnt -= ZSTR_LEN(encoded);
- smart_str_appendl(pretval, "?=", sizeof("?=") - 1);
- char_cnt -= 2;
+ zend_string_release_ex(encoded, 0);
+ encoded = NULL;
+ } break; /* case PHP_ICONV_ENC_SCHEME_BASE64: */
- zend_string_release_ex(encoded, 0);
- encoded = NULL;
- } break; /* case PHP_ICONV_ENC_SCHEME_BASE64: */
+ case PHP_ICONV_ENC_SCHEME_QPRINT: {
+ size_t ini_in_left;
+ const char *ini_in_p;
+ const unsigned char *p;
+ size_t nbytes_required;
- case PHP_ICONV_ENC_SCHEME_QPRINT: {
- size_t ini_in_left;
- const char *ini_in_p;
- const unsigned char *p;
- size_t nbytes_required;
+ smart_str_appendc(pretval, 'Q');
+ char_cnt--;
+ smart_str_appendc(pretval, '?');
+ char_cnt--;
- smart_str_appendc(pretval, 'Q');
- char_cnt--;
- smart_str_appendc(pretval, '?');
- char_cnt--;
+ prev_in_left = ini_in_left = in_left;
+ ini_in_p = in_p;
- prev_in_left = ini_in_left = in_left;
- ini_in_p = in_p;
+ for (out_size = (char_cnt - 2); out_size > 0;) {
- for (out_size = (char_cnt - 2); out_size > 0;) {
+ nbytes_required = 0;
- nbytes_required = 0;
+ out_p = buf;
+ out_left = out_size;
- out_p = buf;
- out_left = out_size;
+ if (iconv(cd, (ICONV_CONST char **)&in_p, &in_left, (char **) &out_p, &out_left) == (size_t)-1) {
+ switch (errno) {
+ case EINVAL:
+ err = PHP_ICONV_ERR_ILLEGAL_CHAR;
+ goto out_try;
- if (iconv(cd, (ICONV_CONST char **)&in_p, &in_left, (char **) &out_p, &out_left) == (size_t)-1) {
- switch (errno) {
- case EINVAL:
- err = PHP_ICONV_ERR_ILLEGAL_CHAR;
- goto out;
+ case EILSEQ:
+ err = PHP_ICONV_ERR_ILLEGAL_SEQ;
+ goto out_try;
- case EILSEQ:
- err = PHP_ICONV_ERR_ILLEGAL_SEQ;
- goto out;
+ case E2BIG:
+ if (prev_in_left == in_left) {
+ err = PHP_ICONV_ERR_UNKNOWN;
+ goto out_try;
+ }
+ break;
- case E2BIG:
- if (prev_in_left == in_left) {
+ default:
err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
- }
- break;
-
- default:
+ goto out_try;
+ }
+ }
+ if (iconv(cd, NULL, NULL, (char **) &out_p, &out_left) == (size_t)-1) {
+ if (errno != E2BIG) {
err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
+ goto out_try;
+ }
}
- }
- if (iconv(cd, NULL, NULL, (char **) &out_p, &out_left) == (size_t)-1) {
- if (errno != E2BIG) {
- err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
+
+ for (p = (unsigned char *)buf; p < (unsigned char *)out_p; p++) {
+ nbytes_required += qp_table[*p];
}
- }
- for (p = (unsigned char *)buf; p < (unsigned char *)out_p; p++) {
- nbytes_required += qp_table[*p];
+ if (nbytes_required <= char_cnt - 2) {
+ break;
+ }
+
+ out_size -= ((nbytes_required - (char_cnt - 2)) + 2) / 3;
+ in_left = ini_in_left;
+ in_p = ini_in_p;
}
- if (nbytes_required <= char_cnt - 2) {
- break;
+ for (p = (unsigned char *)buf; p < (unsigned char *)out_p; p++) {
+ if (qp_table[*p] == 1) {
+ smart_str_appendc(pretval, *(char *)p);
+ char_cnt--;
+ } else {
+ static const char qp_digits[] = "0123456789ABCDEF";
+ smart_str_appendc(pretval, '=');
+ smart_str_appendc(pretval, qp_digits[(*p >> 4) & 0x0f]);
+ smart_str_appendc(pretval, qp_digits[(*p & 0x0f)]);
+ char_cnt -= 3;
+ }
}
- out_size -= ((nbytes_required - (char_cnt - 2)) + 2) / 3;
- in_left = ini_in_left;
- in_p = ini_in_p;
- }
+ smart_str_appendl(pretval, "?=", sizeof("?=") - 1);
+ char_cnt -= 2;
- for (p = (unsigned char *)buf; p < (unsigned char *)out_p; p++) {
- if (qp_table[*p] == 1) {
- smart_str_appendc(pretval, *(char *)p);
- char_cnt--;
- } else {
- static const char qp_digits[] = "0123456789ABCDEF";
- smart_str_appendc(pretval, '=');
- smart_str_appendc(pretval, qp_digits[(*p >> 4) & 0x0f]);
- smart_str_appendc(pretval, qp_digits[(*p & 0x0f)]);
- char_cnt -= 3;
+ if (iconv(cd, NULL, NULL, NULL, NULL) == (size_t)-1) {
+ err = PHP_ICONV_ERR_UNKNOWN;
+ goto out_try;
}
- }
-
- smart_str_appendl(pretval, "?=", sizeof("?=") - 1);
- char_cnt -= 2;
- if (iconv(cd, NULL, NULL, NULL, NULL) == (size_t)-1) {
- err = PHP_ICONV_ERR_UNKNOWN;
- goto out;
- }
+ } break; /* case PHP_ICONV_ENC_SCHEME_QPRINT: */
+ }
- } while (in_left > 0);
++ } while (in_left > 0 && !ZEND_HAS_OOM());
- } break; /* case PHP_ICONV_ENC_SCHEME_QPRINT: */
- }
- } while (in_left > 0 && !ZEND_HAS_OOM());
+ smart_str_0(pretval);
- smart_str_0(pretval);
+out_try: ;
+ } zend_catch {
+ bailout = true;
+ } zend_end_try();
out:
if (cd != (iconv_t)(-1)) {
diff --cc ext/intl/grapheme/grapheme_string.c
index 50e40cffc01,698b75ce30b..59c3e3cddce
--- a/ext/intl/grapheme/grapheme_string.c
+++ b/ext/intl/grapheme/grapheme_string.c
@@@ -836,82 -811,4 +836,91 @@@ PHP_FUNCTION(grapheme_extract
RETURN_STRINGL(((char *)pstr), ret_pos);
}
+PHP_FUNCTION(grapheme_str_split)
+{
+ char *pstr, *end;
+ zend_string *str;
+ zend_long split_len = 1;
+
+ unsigned char u_break_iterator_buffer[U_BRK_SAFECLONE_BUFFERSIZE];
+ UErrorCode ustatus = U_ZERO_ERROR;
+ int32_t pos, current, i, end_len = 0;
+ UBreakIterator* bi;
+ UText *ut = NULL;
+
+ ZEND_PARSE_PARAMETERS_START(1, 2)
+ Z_PARAM_STR(str)
+ Z_PARAM_OPTIONAL
+ Z_PARAM_LONG(split_len)
+ ZEND_PARSE_PARAMETERS_END();
+
+ if (split_len <= 0 || split_len > UINT_MAX / 4) {
+ zend_argument_value_error(2, "must be greater than 0 and less than or equal to %d", UINT_MAX / 4);
+ RETURN_THROWS();
+ }
+
+ if (ZSTR_LEN(str) == 0) {
+ RETURN_EMPTY_ARRAY();
+ }
+
+ pstr = ZSTR_VAL(str);
+ ut = utext_openUTF8(ut, pstr, ZSTR_LEN(str), &ustatus);
+
+ if ( U_FAILURE( ustatus ) ) {
+ /* Set global error code. */
+ intl_error_set_code( NULL, ustatus );
+
+ /* Set error messages. */
+ intl_error_set_custom_msg( NULL, "Error opening UTF-8 text", 0 );
+
+ RETURN_FALSE;
+ }
+
+ bi = NULL;
+ ustatus = U_ZERO_ERROR;
+ bi = grapheme_get_break_iterator((void*)u_break_iterator_buffer, &ustatus );
+
+ if( U_FAILURE(ustatus) ) {
+ RETURN_FALSE;
+ }
+
+ ubrk_setUText(bi, ut, &ustatus);
+
+ pos = 0;
+ array_init(return_value);
+
+ for (end = pstr, i = 0, current = 0; pos != UBRK_DONE;) {
+ end_len = pos - current;
+ pos = ubrk_next(bi);
+
+ if (i == split_len - 1) {
+ if ( pos != UBRK_DONE ) {
+ add_next_index_stringl(return_value, pstr, pos - current);
++ if (ZEND_HAS_OOM()) {
++ goto oom;
++ }
+ end = pstr + pos - current;
+ i = 0;
+ pstr += pos - current;
+ current = pos;
+ }
+ } else {
+ i += 1;
+ }
+ }
+
+ if (i != 0 && end_len != 0) {
+ add_next_index_stringl(return_value, end, end_len);
+ }
+
+ utext_close(ut);
+ ubrk_close(bi);
++ return;
++
++oom:
++ utext_close(ut);
++ ubrk_close(bi);
++ RETURN_OOM();
+}
+
/* }}} */
diff --cc ext/intl/tests/oom_grapheme_str_split.phpt
index 00000000000,00000000000..0e6778c6615
new file mode 100644
--- /dev/null
+++ b/ext/intl/tests/oom_grapheme_str_split.phpt
@@@ -1,0 -1,0 +1,15 @@@
++--TEST--
++grapheme_str_split() doesn't reach hard limit
++--EXTENSIONS--
++intl
++--INI--
++memory_limit=8M
++--FILE--
++<?php
++/* Every grapheme becomes a separate string plus an array slot */
++grapheme_str_split(str_repeat("a", 400 * 1024));
++
++echo "not reached\n";
++?>
++--EXPECTF--
++Fatal error: Allowed memory size of 8388608 bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
diff --cc ext/libxml/libxml.c
index 5fdc44c4115,04bbd6757f2..cb8a1df0efd
--- a/ext/libxml/libxml.c
+++ b/ext/libxml/libxml.c
@@@ -685,13 -714,17 +685,17 @@@ void php_libxml_issue_error(int level,
}
}
-static void php_libxml_internal_error_handler(int error_type, void *ctx, const char **msg, va_list ap)
+static void php_libxml_internal_error_handler_ex(php_libxml_error_level error_type, void *ctx, const char *msg, va_list ap, int line, int column)
{
char *buf;
- int len, len_iter, output = 0;
-
+ bool output = false;
++
+ if (ZEND_HAS_OOM()) {
+ return;
+ }
- len = vspprintf(&buf, 0, *msg, ap);
- len_iter = len;
+ size_t len = vspprintf(&buf, 0, msg, ap);
+ size_t len_iter = len;
/* remove any trailing \n */
while (len_iter && buf[--len_iter] == '\n') {
@@@ -1200,13 -1196,30 +1204,13 @@@ PHP_FUNCTION(libxml_get_errors
ZEND_PARSE_PARAMETERS_NONE();
if (LIBXML(error_list)) {
- array_init(return_value);
-
+ array_init_size(return_value, zend_llist_count(LIBXML(error_list)));
error = zend_llist_get_first(LIBXML(error_list));
- while (error != NULL) {
+ while (error != NULL && !ZEND_HAS_OOM()) {
zval z_error;
-
- object_init_ex(&z_error, libxmlerror_class_entry);
- add_property_long_ex(&z_error, "level", sizeof("level") - 1, error->level);
- add_property_long_ex(&z_error, "code", sizeof("code") - 1, error->code);
- add_property_long_ex(&z_error, "column", sizeof("column") - 1, error->int2 );
- if (error->message) {
- add_property_string_ex(&z_error, "message", sizeof("message") - 1, error->message);
- } else {
- add_property_stringl_ex(&z_error, "message", sizeof("message") - 1, "", 0);
- }
- if (error->file) {
- add_property_string_ex(&z_error, "file", sizeof("file") - 1, error->file);
- } else {
- add_property_stringl_ex(&z_error, "file", sizeof("file") - 1, "", 0);
- }
- add_property_long_ex(&z_error, "line", sizeof("line") - 1, error->line);
+ php_libxml_create_error_object(&z_error, error);
add_next_index_zval(return_value, &z_error);
-
error = zend_llist_get_next(LIBXML(error_list));
}
} else {
diff --cc ext/mbstring/libmbfl/mbfl/mbfl_encoding.h
index 620d8e07198,3a4c8009d5b..e71737318bf
--- a/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h
+++ b/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h
@@@ -162,17 -160,35 +162,34 @@@ static inline void mb_convert_buf_init(
buf->error_mode = err_mode;
}
+ static inline zend_result mb_convert_buf_ensure(mb_convert_buf *buf, unsigned char **out, unsigned char **limit, size_t needed)
+ {
+ ZEND_ASSERT(*out <= *limit);
- if ((*limit - *out) < needed) {
++ if ((size_t)(*limit - *out) < needed) {
+ size_t oldsize = *limit - (unsigned char*)ZSTR_VAL((buf)->str);
+ size_t newsize = oldsize + MAX(oldsize >> 1, needed);
+ zend_string *newstr = safe_ferealloc((buf)->str, 1, newsize, _ZSTR_STRUCT_SIZE(0), /* fallible */ true);
+ if (!newstr) {
+ /* An earlier growth may have moved the buffer. */
+ buf->out = *out;
+ buf->limit = *limit;
+ return FAILURE;
+ }
+ *out = (unsigned char*)ZSTR_VAL(newstr) + (*out - (unsigned char*)ZSTR_VAL((buf)->str));
+ *limit = (unsigned char*)ZSTR_VAL(newstr) + newsize;
+ (buf)->str = newstr;
+ }
+ return SUCCESS;
-
+ }
+
#define MB_CONVERT_BUF_ENSURE(buf, out, limit, needed) \
- ZEND_ASSERT(out <= limit); \
- if ((size_t)(limit - out) < (needed)) { \
- size_t oldsize = limit - (unsigned char*)ZSTR_VAL((buf)->str); \
- size_t newsize = oldsize + MAX(oldsize >> 1, needed); \
- zend_string *newstr = erealloc((buf)->str, _ZSTR_STRUCT_SIZE(newsize)); \
- out = (unsigned char*)ZSTR_VAL(newstr) + (out - (unsigned char*)ZSTR_VAL((buf)->str)); \
- limit = (unsigned char*)ZSTR_VAL(newstr) + newsize; \
- (buf)->str = newstr; \
+ if (!MB_CONVERT_BUF_ENSURE_EX(buf, out, limit, needed)) { \
+ return; \
}
+ #define MB_CONVERT_BUF_ENSURE_EX(buf, out, limit, needed) \
+ EXPECTED(mb_convert_buf_ensure(buf, &(out), &(limit), (needed)) == SUCCESS)
+
#define MB_CONVERT_BUF_STORE(buf, _out, _limit) (buf)->out = _out; (buf)->limit = _limit
#define MB_CONVERT_BUF_LOAD(buf, _out, _limit) _out = (buf)->out; _limit = (buf)->limit
diff --cc ext/mbstring/mbstring.c
index 7e0b47deae9,2ee6306bfb4..98beb5f0d3f
--- a/ext/mbstring/mbstring.c
+++ b/ext/mbstring/mbstring.c
@@@ -2428,46 -2444,13 +2452,48 @@@ PHP_FUNCTION(mb_strcut
RETURN_EMPTY_STRING();
}
+ if (enc->cut) {
+ RETURN_STR(enc->cut(string.val, from, len, string.val + string.len));
+ }
+
+ unsigned int char_len = string.encoding->flag & (MBFL_ENCTYPE_SBCS | MBFL_ENCTYPE_WCS2 | MBFL_ENCTYPE_WCS4);
+ if (char_len) {
+ /* Round `from` down to a multiple of `char_len`; works because `char_len` is a power of 2 */
+ from &= -char_len;
+ if (len > string.len - from) {
+ len = string.len - from;
+ }
+ RETURN_STR(zend_string_init_fast((const char*)(string.val + from), len & -char_len));
+ }
+
+ if (enc->mblen_table) {
+ const unsigned char *mbtab = enc->mblen_table;
+ const unsigned char *p, *q, *end;
+ int m = 0;
+ /* Search for start position */
+ for (p = (const unsigned char*)string.val, q = p + from; p < q; p += (m = mbtab[*p]));
+ if (p > q) {
+ p -= m;
+ }
+ const unsigned char *start = p;
+ /* Search for end position */
+ if (len >= string.len - (start - (const unsigned char*)string.val)) {
+ end = (const unsigned char*)(string.val + string.len);
+ } else {
+ for (q = p + len; p < q; p += (m = mbtab[*p]));
+ if (p > q) {
+ p -= m;
+ }
+ end = p;
+ }
+ RETURN_STR(zend_string_init_fast((const char*)start, end - start));
+ }
+
ret = mbfl_strcut(&string, &result, from, len);
- ZEND_ASSERT(ret != NULL);
- RETVAL_STRINGL((char *)ret->val, ret->len); /* the string is already strdup()'ed */
+ if (UNEXPECTED(ret == NULL)) {
+ RETURN_OOM();
+ }
-
- // TODO: avoid reallocation ???
+ RETVAL_FSTRINGL((char *)ret->val, ret->len); /* the string is already strdup()'ed */
efree(ret->val);
}
/* }}} */
@@@ -2640,8 -2615,11 +2666,10 @@@ dont_restart_conversion
append_trim_marker:
if (ZSTR_LEN(marker) > 0) {
- MB_CONVERT_BUF_ENSURE((&buf), buf.out, buf.limit, ZSTR_LEN(marker));
- if (!MB_CONVERT_BUF_ENSURE_EX(&buf, buf.out, buf.limit, ZSTR_LEN(marker))) {
++ if (!MB_CONVERT_BUF_ENSURE_EX((&buf), buf.out, buf.limit, ZSTR_LEN(marker))) {
+ return mb_convert_buf_result(&buf, enc);
+ }
- memcpy(buf.out, ZSTR_VAL(marker), ZSTR_LEN(marker));
- buf.out += ZSTR_LEN(marker);
+ buf.out = zend_mempcpy(buf.out, ZSTR_VAL(marker), ZSTR_LEN(marker));
}
/* Even if `enc` is UTF-8, don't mark the output string as valid UTF-8, because
@@@ -3801,26 -3547,21 +3834,29 @@@ static bool mb_recursive_convert_variab
zend_string *ret = php_mb_convert_encoding_ex(Z_STRVAL_P(var), Z_STRLEN_P(var), to_encoding, from_encoding);
zval_ptr_dtor(orig_var);
ZVAL_STR(orig_var, ret);
+ if (ZEND_HAS_OOM()) {
+ return true;
+ }
} else if (Z_TYPE_P(var) == IS_ARRAY || Z_TYPE_P(var) == IS_OBJECT) {
- if (Z_TYPE_P(var) == IS_ARRAY) {
- SEPARATE_ARRAY(var);
- }
- if (Z_REFCOUNTED_P(var)) {
- if (Z_IS_RECURSIVE_P(var)) {
+ HashTable *ht = HASH_OF(var);
+ HashTable *orig_ht = ht;
+
+ if (ht) {
+ if (GC_IS_RECURSIVE(ht)) {
return true;
}
- Z_PROTECT_RECURSION_P(var);
+
+ GC_TRY_PROTECT_RECURSION(ht);
}
- HashTable *ht = HASH_OF(var);
+ if (Z_TYPE_P(var) == IS_ARRAY) {
+ SEPARATE_ARRAY(var);
+ ht = Z_ARRVAL_P(var);
+
+ if (ht && ht != orig_ht && !GC_IS_RECURSIVE(ht)) {
+ GC_TRY_PROTECT_RECURSION(ht);
+ }
+ }
if (ht != NULL) {
ZEND_HASH_FOREACH_VAL(ht, entry) {
/* Can be a typed property declaration, in which case we need to remove the reference from the source list.
diff --cc ext/opcache/ZendAccelerator.c
index 7d35ef5c147,9355604141b..da3114ad60b
--- a/ext/opcache/ZendAccelerator.c
+++ b/ext/opcache/ZendAccelerator.c
@@@ -609,31 -598,22 +609,34 @@@ static zend_always_inline zend_string *
static zend_string* ZEND_FASTCALL accel_init_interned_string_for_php(const char *str, size_t size, bool permanent)
{
+ zend_string *ret;
+
if (ZCG(counted)) {
zend_ulong h = zend_inline_hash_func(str, size);
- zend_string *ret = accel_find_interned_string_ex(h, str, size);
+ ret = accel_find_interned_string_ex(h, str, size);
- if (!ret) {
- ret = zend_string_finit(str, size, permanent, CG(interned_strings_fallible));
- if (UNEXPECTED(!ret)) {
- return NULL;
- }
- ZSTR_H(ret) = h;
+ if (ret) {
+ return ret;
}
- ret = zend_string_init(str, size, permanent);
- return ret;
++ ret = zend_string_finit(str, size, permanent, CG(interned_strings_fallible));
++ if (UNEXPECTED(!ret)) {
++ return NULL;
++ }
+ ZSTR_H(ret) = h;
+ } else {
- ret = zend_string_init(str, size, permanent);
++ ret = zend_string_finit(str, size, permanent, CG(interned_strings_fallible));
+ }
+
+#if ZEND_RC_DEBUG
+ /* PHP shouldn't create persistent strings during request, but at least
+ * dl() may do this. They belong to the temporary module and are not shared. */
+ if (!zend_rc_debug && permanent) {
+ GC_MAKE_PERSISTENT_LOCAL(ret);
}
+#endif
- return zend_string_finit(str, size, permanent, CG(interned_strings_fallible));
+ return ret;
}
static inline void accel_copy_permanent_list_types(
@@@ -2486,9 -2444,8 +2490,11 @@@ static zend_class_entry* zend_accel_inh
EG(errors) = NULL;
ZCSG(map_ptr_last) = CG(map_ptr_last);
+ ZEND_ATOMIC_FENCE_RELEASE();
+ proto->inheritance_cache = entry;
+
+ zend_accel_pending_ce_cache_commit();
+
zend_shared_alloc_destroy_xlat_table();
zend_shared_alloc_unlock();
diff --cc ext/opcache/jit/zend_jit.c
index a958a2453c4,b58b515a623..fa729dfa2c4
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@@ -2564,26 -3972,9 +2564,34 @@@ static int zend_jit(const zend_op_arra
if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
break;
}
- if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
+ if (!zend_jit_rope(&ctx, opline, op2_info)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_0:
- jit_frameless_icall0(jit, opline);
++ if (!jit_frameless_icall0(jit, opline)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_1:
+ op1_info = OP1_INFO();
- jit_frameless_icall1(jit, opline, op1_info);
++ if (!jit_frameless_icall1(jit, opline, op1_info)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_2:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
- jit_frameless_icall2(jit, opline, op1_info, op2_info);
++ if (!jit_frameless_icall2(jit, opline, op1_info, op2_info)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_3:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
- jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
++ if (!jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO())) {
+ goto jit_failure;
+ }
goto done;
default:
break;
diff --cc ext/opcache/jit/zend_jit_ir.c
index 4c574a39a15,00000000000..830dbdf3272
mode 100644,000000..100644
--- a/ext/opcache/jit/zend_jit_ir.c
+++ b/ext/opcache/jit/zend_jit_ir.c
@@@ -1,17820 -1,0 +1,17874 @@@
+/*
+ * +----------------------------------------------------------------------+
+ * | Zend JIT |
+ * +----------------------------------------------------------------------+
+ * | Copyright (c) The PHP Group |
+ * +----------------------------------------------------------------------+
+ * | This source file is subject to version 3.01 of the PHP license, |
+ * | that is bundled with this package in the file LICENSE, and is |
+ * | available through the world-wide-web at the following url: |
+ * | https://www.php.net/license/3_01.txt |
+ * | If you did not receive a copy of the PHP license and are unable to |
+ * | obtain it through the world-wide-web, please send a note to |
+ * | license@php.net so we can mail you a copy immediately. |
+ * +----------------------------------------------------------------------+
+ * | Authors: Dmitry Stogov <dmitry@php.net> |
+ * +----------------------------------------------------------------------+
+ */
+
+#include "jit/ir/ir.h"
+#include "jit/ir/ir_builder.h"
+
+#if defined(__APPLE__) && defined(__x86_64__)
+# include <mach-o/dyld.h>
+#endif
+
+#if defined(IR_TARGET_X86)
+# define IR_REG_SP 4 /* IR_REG_RSP */
+# define IR_REG_FP 5 /* IR_REG_RBP */
+# define ZREG_FP 6 /* IR_REG_RSI */
+# define ZREG_IP 7 /* IR_REG_RDI */
+# define ZREG_FIRST_FPR 8
+# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
+#elif defined(IR_TARGET_X64)
+# define IR_REG_SP 4 /* IR_REG_RSP */
+# define IR_REG_FP 5 /* IR_REG_RBP */
+# define ZREG_FP 14 /* IR_REG_R14 */
+# define ZREG_IP 15 /* IR_REG_R15 */
+# define ZREG_FIRST_FPR 16
+# if defined(_WIN64)
+# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
+/*
+# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \
+ (1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \
+ (1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))
+*/
+# define IR_SHADOW_ARGS 32
+# else
+# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */
+# endif
+#elif defined(IR_TARGET_AARCH64)
+# define IR_REG_SP 31 /* IR_REG_RSP */
+# define IR_REG_FP 29 /* IR_REG_X29 */
+# define ZREG_FP 27 /* IR_REG_X27 */
+# define ZREG_IP 28 /* IR_REG_X28 */
+# define ZREG_FIRST_FPR 32
+# define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
+ (1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
+#else
+# error "Unknown IR target"
+#endif
+
+#define ZREG_RX ZREG_IP
+
+#define OPTIMIZE_FOR_SIZE 0
+
+/* IR builder defines */
+#undef _ir_CTX
+#define _ir_CTX (&jit->ctx)
+
+#undef ir_CONST_ADDR
+#define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
+#define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
+#define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
+#define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
+ ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
+
+#define ir_CONST_FUNC_PROTO(_addr, _proto) \
+ jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
+
+#undef ir_ADD_OFFSET
+#define ir_ADD_OFFSET(_addr, _offset) \
+ jit_ADD_OFFSET(jit, _addr, _offset)
+
+#ifdef ZEND_ENABLE_ZVAL_LONG64
+# define IR_LONG IR_I64
+# define ir_CONST_LONG ir_CONST_I64
+# define ir_UNARY_OP_L ir_UNARY_OP_I64
+# define ir_BINARY_OP_L ir_BINARY_OP_I64
+# define ir_ADD_L ir_ADD_I64
+# define ir_SUB_L ir_SUB_I64
+# define ir_MUL_L ir_MUL_I64
+# define ir_DIV_L ir_DIV_I64
+# define ir_MOD_L ir_MOD_I64
+# define ir_NEG_L ir_NEG_I64
+# define ir_ABS_L ir_ABS_I64
+# define ir_SEXT_L ir_SEXT_I64
+# define ir_ZEXT_L ir_ZEXT_I64
+# define ir_TRUNC_L ir_TRUNC_I64
+# define ir_BITCAST_L ir_BITCAST_I64
+# define ir_FP2L ir_FP2I64
+# define ir_ADD_OV_L ir_ADD_OV_I64
+# define ir_SUB_OV_L ir_SUB_OV_I64
+# define ir_MUL_OV_L ir_MUL_OV_I64
+# define ir_NOT_L ir_NOT_I64
+# define ir_OR_L ir_OR_I64
+# define ir_AND_L ir_AND_I64
+# define ir_XOR_L ir_XOR_I64
+# define ir_SHL_L ir_SHL_I64
+# define ir_SHR_L ir_SHR_I64
+# define ir_SAR_L ir_SAR_I64
+# define ir_ROL_L ir_ROL_I64
+# define ir_ROR_L ir_ROR_I64
+# define ir_MIN_L ir_MIN_I64
+# define ir_MAX_L ir_MAX_I64
+# define ir_LOAD_L ir_LOAD_I64
+#else
+# define IR_LONG IR_I32
+# define ir_CONST_LONG ir_CONST_I32
+# define ir_UNARY_OP_L ir_UNARY_OP_I32
+# define ir_BINARY_OP_L ir_BINARY_OP_I32
+# define ir_ADD_L ir_ADD_I32
+# define ir_SUB_L ir_SUB_I32
+# define ir_MUL_L ir_MUL_I32
+# define ir_DIV_L ir_DIV_I32
+# define ir_MOD_L ir_MOD_I32
+# define ir_NEG_L ir_NEG_I32
+# define ir_ABS_L ir_ABS_I32
+# define ir_SEXT_L ir_SEXT_I32
+# define ir_ZEXT_L ir_ZEXT_I32
+# define ir_TRUNC_L ir_TRUNC_I32
+# define ir_BITCAST_L ir_BITCAST_I32
+# define ir_FP2L ir_FP2I32
+# define ir_ADD_OV_L ir_ADD_OV_I32
+# define ir_SUB_OV_L ir_SUB_OV_I32
+# define ir_MUL_OV_L ir_MUL_OV_I32
+# define ir_NOT_L ir_NOT_I32
+# define ir_OR_L ir_OR_I32
+# define ir_AND_L ir_AND_I32
+# define ir_XOR_L ir_XOR_I32
+# define ir_SHL_L ir_SHL_I32
+# define ir_SHR_L ir_SHR_I32
+# define ir_SAR_L ir_SAR_I32
+# define ir_ROL_L ir_ROL_I32
+# define ir_ROR_L ir_ROR_I32
+# define ir_MIN_L ir_MIN_I32
+# define ir_MAX_L ir_MAX_I32
+# define ir_LOAD_L ir_LOAD_I32
+#endif
+
+/* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */
+typedef struct _ir_refs {
+ uint32_t count;
+ uint32_t limit;
+ ir_ref refs[] ZEND_ELEMENT_COUNT(count);
+} ir_refs;
+
+#define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))
+#define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \
+ do {_name->count = 0; _name->limit = (_n);} while (0)
+
+static void ir_refs_add(ir_refs *refs, ir_ref ref)
+{
+ ir_ref *ptr;
+
+ ZEND_ASSERT(refs->count < refs->limit);
+ ptr = refs->refs;
+ ptr[refs->count++] = ref;
+}
+
+static size_t zend_jit_trace_prologue_size = (size_t)-1;
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+static uint32_t allowed_opt_flags = 0;
+static uint32_t default_mflags = 0;
+#endif
+static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ???
+
+#ifdef ZTS
+# ifdef _WIN32
+extern uint32_t _tls_index;
+extern char *_tls_start;
+extern char *_tls_end;
+# endif
+
+static size_t tsrm_ls_cache_tcb_offset = 0;
+static size_t tsrm_tls_index = 0;
+static size_t tsrm_tls_offset = 0;
+
+# define EG_TLS_OFFSET(field) \
+ (executor_globals_offset + offsetof(zend_executor_globals, field))
+
+# define CG_TLS_OFFSET(field) \
+ (compiler_globals_offset + offsetof(zend_compiler_globals, field))
+
+# define jit_EG(_field) \
+ ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
+
+# define jit_CG(_field) \
+ ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
+
+#else
+
+# define jit_EG(_field) \
+ ir_CONST_ADDR(&EG(_field))
+
+# define jit_CG(_field) \
+ ir_CONST_ADDR(&CG(_field))
+
+#endif
+
+#define jit_CALL(_call, _field) \
+ ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))
+
+#define jit_EX(_field) \
+ jit_CALL(jit_FP(jit), _field)
+
+#define jit_RX(_field) \
+ jit_CALL(jit_IP(jit), _field)
+
+#define JIT_STUBS(_) \
+ _(exception_handler, IR_SKIP_PROLOGUE) \
+ _(exception_handler_undef, IR_SKIP_PROLOGUE) \
+ _(exception_handler_free_op2, IR_SKIP_PROLOGUE) \
+ _(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \
+ _(interrupt_handler, IR_SKIP_PROLOGUE) \
+ _(leave_function_handler, IR_SKIP_PROLOGUE) \
+ _(negative_shift, IR_SKIP_PROLOGUE) \
+ _(mod_by_zero, IR_SKIP_PROLOGUE) \
+ _(invalid_this, IR_SKIP_PROLOGUE) \
+ _(undefined_function, IR_SKIP_PROLOGUE) \
+ _(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \
+ _(icall_throw, IR_SKIP_PROLOGUE) \
+ _(leave_throw, IR_SKIP_PROLOGUE) \
+ _(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
+ _(trace_halt, IR_SKIP_PROLOGUE) \
+ _(trace_escape, IR_SKIP_PROLOGUE) \
+ _(trace_exit, IR_SKIP_PROLOGUE) \
+ _(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \
+ _(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \
+
+#define JIT_STUB_ID(name, flags) \
+ jit_stub_ ## name,
+
+#define JIT_STUB_FORWARD(name, flags) \
+ static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
+
+#define JIT_STUB(name, flags) \
+ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},
+
+typedef enum _jit_stub_id {
+ JIT_STUBS(JIT_STUB_ID)
+ jit_last_stub
+} jit_stub_id;
+
+typedef struct _zend_jit_reg_var {
+ ir_ref ref;
+ uint32_t flags;
+} zend_jit_reg_var;
+
+typedef struct _zend_jit_ctx {
+ ir_ctx ctx;
+ const zend_op *last_valid_opline;
+ bool use_last_valid_opline;
+ bool track_last_valid_opline;
+ bool reuse_ip;
+ uint32_t delayed_call_level;
+ int b; /* current basic block number or -1 */
+#ifdef ZTS
+ ir_ref tls;
+#endif
+ ir_ref fp;
+ ir_ref poly_func_ref; /* restored from parent trace snapshot */
+ ir_ref poly_this_ref; /* restored from parent trace snapshot */
+ ir_ref trace_loop_ref;
+ ir_ref return_inputs;
+ const zend_op_array *op_array;
+ const zend_op_array *current_op_array;
+ zend_ssa *ssa;
+ zend_string *name;
+ ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */
+ ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */
+ ir_ref *bb_edges;
+ zend_jit_trace_info *trace;
+ zend_jit_reg_var *ra;
+ int delay_var;
+ ir_refs *delay_refs;
+ ir_ref eg_exception_addr;
+ HashTable addr_hash;
+ ir_ref stub_addr[jit_last_stub];
+} zend_jit_ctx;
+
+typedef int8_t zend_reg;
+
+typedef struct _zend_jit_registers_buf {
+#if defined(IR_TARGET_X64)
+ uint64_t gpr[16]; /* general purpose integer register */
+ double fpr[16]; /* floating point registers */
+#elif defined(IR_TARGET_X86)
+ uint32_t gpr[8]; /* general purpose integer register */
+ double fpr[8]; /* floating point registers */
+#elif defined (IR_TARGET_AARCH64)
+ uint64_t gpr[32]; /* general purpose integer register */
+ double fpr[32]; /* floating point registers */
+#else
+# error "Unknown IR target"
+#endif
+} zend_jit_registers_buf;
+
+/* Keep 32 exit points in a single code block */
+#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
+#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
+
+static uint32_t zend_jit_exit_point_by_addr(const void *addr);
+int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
+
+static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr var_use_addr,
+ zend_jit_addr var_addr,
+ uint32_t var_info,
+ uint32_t var_def_info,
+ uint8_t val_type,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr res_addr,
+ zend_jit_addr ref_addr,
+ bool check_exception);
+
+static void zend_jit_preserve_parent_regs(zend_jit_ctx *jit,
+ zend_ssa *ssa,
+ zend_jit_trace_info *parent,
+ uint32_t exit_num);
+
+typedef struct _zend_jit_stub {
+ const char *name;
+ int (*stub)(zend_jit_ctx *jit);
+ uint32_t flags;
+} zend_jit_stub;
+
+JIT_STUBS(JIT_STUB_FORWARD)
+
+static const zend_jit_stub zend_jit_stubs[] = {
+ JIT_STUBS(JIT_STUB)
+};
+
+#if defined(_WIN32) || defined(IR_TARGET_AARCH64)
+/* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */
+static void** zend_jit_stub_handlers = NULL;
+#else
+static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];
+#endif
+
+#if defined(IR_TARGET_AARCH64)
+
+# ifdef __FreeBSD__
+/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */
+typedef struct TLSDescriptor {
+ void* thunk;
+ int index;
+ size_t offset;
+} TLSDescriptor;
+# endif
+
+#define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */
+
+static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)
+{
+ int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
+
+ for (i = 0; i < count; i++) {
+ if (zend_jit_stub_handlers[i] == addr) {
+ return zend_jit_stub_handlers[count + i];
+ }
+ }
+
+ if (((zend_jit_ctx*)ctx)->trace
+ && (void*)addr >= dasm_buf && (void*)addr < dasm_end) {
+ uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
+
+ if (exit_point != (uint32_t)-1) {
+ zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
+
+ ZEND_ASSERT(exit_point < t->exit_count);
+ return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));
+ }
+ }
+
+ return NULL;
+}
+
+static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)
+{
+ int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
+ uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
+
+ if (exit_point != (uint32_t)-1) {
+ return 1;
+ }
+ for (i = 0; i < count; i++) {
+ if (zend_jit_stub_handlers[i] == addr) {
+ const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];
+ *ptr = veneer;
+ ctx->flags2 |= IR_HAS_VENEERS;
+#ifdef HAVE_CAPSTONE
+ int64_t offset;
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
+ const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);
+
+ if (name && !offset) {
+ if (strstr(name, "@veneer") == NULL) {
+ char *new_name;
+
+ zend_spprintf(&new_name, 0, "%s@veneer", name);
+ ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
+ efree(new_name);
+ } else {
+ ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
+ }
+ }
+ }
+#endif
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static void zend_jit_commit_veneers(void)
+{
+ int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
+
+ for (i = 0; i < count; i++) {
+ if (zend_jit_stub_handlers[count + i]) {
+ zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];
+ zend_jit_stub_handlers[count + i] = NULL;
+ }
+ }
+}
+#endif
+
+static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)
+{
+#if defined(IR_TARGET_X86)
+ return 0; /* always use immediate value */
+#elif defined(IR_TARGET_X64)
+ return addr > 0xffffffff; /* prefer loading long constant from memery */
+#elif defined(IR_TARGET_AARCH64)
+ return addr > 0xffff;
+#else
+# error "Unknown IR target"
+#endif
+}
+
+static const char* zend_reg_name(int8_t reg)
+{
+ return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);
+}
+
+/* IR helpers */
+
+#ifdef ZTS
+static ir_ref jit_TLS(zend_jit_ctx *jit)
+{
+ ZEND_ASSERT(jit->ctx.control);
+ if (jit->tls) {
+ /* Emit "TLS" once for basic block */
+ ir_insn *insn;
+ ir_ref ref = jit->ctx.control;
+
+ while (1) {
+ if (ref == jit->tls) {
+ return jit->tls;
+ }
+ insn = &jit->ctx.ir_base[ref];
+ if (insn->op >= IR_START || insn->op == IR_CALL) {
+ break;
+ }
+ ref = insn->op1;
+ }
+ }
+ jit->tls = ir_TLS(
+ tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,
+ tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);
+ return jit->tls;
+}
+#endif
+
+static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
+{
+ ir_ref ref;
+ zval *zv;
+
+ if (addr == 0) {
+ return IR_NULL;
+ }
+ zv = zend_hash_index_lookup(&jit->addr_hash, addr);
+ if (Z_TYPE_P(zv) == IS_LONG) {
+ ref = Z_LVAL_P(zv);
+ ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR));
+ } else {
+ ref = ir_unique_const_addr(&jit->ctx, addr);
+ ZVAL_LONG(zv, ref);
+ }
+ return ref;
+}
+
+static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto)
+{
+ ir_ref ref;
+ ir_insn *insn;
+ zval *zv;
+
+ ZEND_ASSERT(addr != 0);
+ zv = zend_hash_index_lookup(&jit->addr_hash, addr);
+ if (Z_TYPE_P(zv) == IS_LONG) {
+ ref = Z_LVAL_P(zv);
+ ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto);
+ } else {
+ ref = ir_unique_const_addr(&jit->ctx, addr);
+ insn = &jit->ctx.ir_base[ref];
+ insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
+ insn->proto = proto;
+ ZVAL_LONG(zv, ref);
+ }
+ return ref;
+}
+
+static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
+{
+#if defined(IR_TARGET_X86)
+ /* TODO: dummy prototype (only flags matter) ??? */
+ ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
+#else
+ ir_ref proto = 0;
+#endif
+
+ return jit_CONST_FUNC_PROTO(jit, addr, proto);
+}
+
+static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
+{
+ if (offset) {
+ addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));
+ }
+ return addr;
+}
+
+static ir_ref jit_EG_exception(zend_jit_ctx *jit)
+{
+#ifdef ZTS
+ return jit_EG(exception);
+#else
+ ir_ref ref = jit->eg_exception_addr;
+
+ if (UNEXPECTED(!ref)) {
+ ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception));
+ jit->eg_exception_addr = ref;
+ }
+ return ref;
+#endif
+}
+
+static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)
+{
+ ir_ref ref = jit->stub_addr[id];
+
+ if (UNEXPECTED(!ref)) {
+ ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
+ jit->stub_addr[id] = ref;
+ }
+ return ref;
+}
+
+static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)
+{
+ ir_ref ref = jit->stub_addr[id];
+ ir_insn *insn;
+
+ if (UNEXPECTED(!ref)) {
+ ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
+ insn = &jit->ctx.ir_base[ref];
+ insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
+#if defined(IR_TARGET_X86)
+ /* TODO: dummy prototype (only flags matter) ??? */
+ insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
+#else
+ insn->proto = 0;
+#endif
+ jit->stub_addr[id] = ref;
+ }
+ return ref;
+}
+
+static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
+{
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
+ const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;
+ const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
+ uint32_t stack_size = op_array->last_var + op_array->T;
+
+ if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]
+ || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]
+ || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]
+ || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]
+ || ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]
+ || ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]
+ || ptr == zend_jit_stub_handlers[jit_stub_negative_shift]
+ || ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]
+ || ptr == zend_jit_stub_handlers[jit_stub_invalid_this]
+ || ptr == zend_jit_stub_handlers[jit_stub_undefined_function]
+ || ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]
+ || ptr == zend_jit_stub_handlers[jit_stub_icall_throw]
+ || ptr == zend_jit_stub_handlers[jit_stub_leave_throw]
+ || ptr == zend_jit_stub_handlers[jit_stub_trace_halt]
+ || ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {
+ /* This is a GUARD that trigger exit through a stub code (without deoptimization) */
+ return;
+ }
+
+ /* Check if we need snapshot entries for polymorphic method call */
+ zend_jit_trace_info *t = jit->trace;
+ uint32_t exit_point = 0, n = 0;
+
+ if (addr < 0) {
+ /* addr is not always the address of the *last* exit point,
+ * so we can not optimize this to 'exit_point = t->exit_count-1' */
+ exit_point = zend_jit_exit_point_by_addr(ptr);
+ ZEND_ASSERT(exit_point != -1);
+ if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
+ n = 2;
+ }
+ }
+
+ if (stack_size || n) {
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ uint32_t snapshot_size, i;
+
+ snapshot_size = stack_size;
+ while (snapshot_size > 0) {
+ ir_ref ref = STACK_REF(stack, snapshot_size - 1);
+
+ if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {
+ snapshot_size--;
+ } else {
+ break;
+ }
+ }
+ if (snapshot_size || n) {
+ ir_ref snapshot;
+
+ snapshot = ir_SNAPSHOT(snapshot_size + n);
+ for (i = 0; i < snapshot_size; i++) {
+ ir_ref ref = STACK_REF(stack, i);
+
+ if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {
+ ref = IR_UNUSED;
+ }
+ ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
+ }
+ if (n) {
+ ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref);
+ ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref);
+ }
+ }
+ }
+ }
+}
+
+static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)
+{
+ int32_t i;
+
+ for (i = 0; i < t->consts_count; i++) {
+ if (t->constants[i].i == val) {
+ return i;
+ }
+ }
+ ZEND_ASSERT(i < 0x7fffffff);
+ t->consts_count = i + 1;
+ t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));
+ t->constants[i].i = val;
+ return i;
+}
+
+uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)
+{
+ uint32_t stack_size, stack_offset;
+ uint32_t new_exit_point = t->exit_count;
+
+ if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {
+ ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
+ return exit_point;
+ }
+
+ t->exit_count++;
+ memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));
+ stack_size = t->exit_info[new_exit_point].stack_size;
+ if (stack_size != 0) {
+ stack_offset = t->stack_map_size;
+ t->stack_map_size += stack_size;
+ // TODO: reduce number of reallocations ???
+ t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
+ memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));
+ t->exit_info[new_exit_point].stack_offset = stack_offset;
+ }
+ t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;
+
+ return new_exit_point;
+}
+
+static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, int op)
+{
+ int8_t *reg_ops = ctx->regs[snapshot_ref];
+ ZEND_ASSERT(reg_ops[op] != ZREG_NONE);
+
+ int8_t reg = reg_ops[op];
+ int32_t offset;
+
+ if (IR_REG_SPILLED(reg)) {
+ reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD;
+ offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op));
+ } else {
+ offset = 0;
+ }
+
+ dest->reg = reg;
+ dest->offset = offset;
+}
+
+static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const zend_jit_ref_snapshot *b)
+{
+ return a->reg == b->reg
+ && (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset));
+}
+
+void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
+{
+ zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
+ uint32_t exit_point, exit_flags;
+ ir_ref n = snapshot->inputs_count;
+ ir_ref i;
+
+ exit_point = zend_jit_exit_point_by_addr(addr);
+ ZEND_ASSERT(exit_point < t->exit_count);
+ exit_flags = t->exit_info[exit_point].flags;
+
+ if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
+ zend_jit_ref_snapshot func, this;
+ zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1);
+ zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n);
+
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func)
+ || !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->exit_info[exit_point].poly_func = func;
+ t->exit_info[exit_point].poly_this = this;
+ n -= 2;
+ }
+
+ for (i = 2; i <= n; i++) {
+ ir_ref ref = ir_insn_op(snapshot, i);
+
+ if (ref) {
+ int8_t *reg_ops = ctx->regs[snapshot_ref];
+ int8_t reg = reg_ops[i];
+ ir_ref var = i - 2;
+
+ ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);
+ if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {
+ ZEND_ASSERT(reg != ZREG_NONE);
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
+ } else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {
+ ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);
+
+ if (ref > 0) {
+ if (reg != ZREG_NONE) {
+ if (reg & IR_REG_SPILL_LOAD) {
+ ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
+ /* spill slot on a CPU stack */
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref
+ || t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
+ || !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
+ } else if (reg & IR_REG_SPILL_SPECIAL) {
+ /* spill slot on a VM stack */
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
+ || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
+ } else {
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
+ }
+ } else {
+ if ((exit_flags & ZEND_JIT_EXIT_FIXED)
+ && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
+ || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
+ exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
+ addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
+ exit_flags &= ~ZEND_JIT_EXIT_FIXED;
+ }
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
+ }
+ } else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {
+ int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;
+ t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;
+ }
+ }
+ }
+ }
+ t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;
+ return addr;
+}
+
+static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)
+{
+ jit_SNAPSHOT(jit, addr);
+ ir_IJMP(addr);
+}
+
+/* PHP JIT helpers */
+
+static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)
+{
+#if ZEND_DEBUG
+ return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),
+ ir_CONST_ADDR(size),
+ op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
+ ir_CONST_U32(opline ? opline->lineno : 0),
+ IR_NULL,
+ ir_CONST_U32(0));
+#elif defined(HAVE_BUILTIN_CONSTANT_P)
+ if (size > 24 && size <= 32) {
+ return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));
+ } else {
+ return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
+ }
+#else
+ return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
+#endif
+}
+
+static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)
+{
+#if ZEND_DEBUG
+ return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),
+ ptr,
+ op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
+ ir_CONST_U32(opline ? opline->lineno : 0),
+ IR_NULL,
+ ir_CONST_U32(0));
+#elif defined(HAVE_BUILTIN_CONSTANT_P)
+ if (size > 24 && size <= 32) {
+ return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);
+ } else {
+ return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
+ }
+#else
+ return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
+#endif
+}
+
+static ir_ref jit_FP(zend_jit_ctx *jit)
+{
+ ZEND_ASSERT(jit->ctx.control);
+ if (jit->fp == IR_UNUSED) {
+ /* Emit "RLOAD FP" once for basic block */
+ jit->fp = ir_RLOAD_A(ZREG_FP);
+ } else {
+ ir_insn *insn;
+ ir_ref ref = jit->ctx.control;
+
+ while (1) {
+ if (ref == jit->fp) {
+ break;
+ }
+ insn = &jit->ctx.ir_base[ref];
+ if (insn->op >= IR_START || insn->op == IR_CALL) {
+ jit->fp = ir_RLOAD_A(ZREG_FP);
+ break;
+ }
+ ref = insn->op1;
+ }
+ }
+ return jit->fp;
+}
+
+static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_RSTORE(ZREG_FP, ref);
+ jit->fp = IR_UNUSED;
+}
+
+static ir_ref jit_IP(zend_jit_ctx *jit)
+{
+ return ir_RLOAD_A(ZREG_IP);
+}
+
+static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_RSTORE(ZREG_IP, ref);
+}
+
+static ir_ref jit_IP32(zend_jit_ctx *jit)
+{
+ return ir_RLOAD_U32(ZREG_IP);
+}
+
+static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref)
+{
+ if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, ref);
+ } else {
+ ir_STORE(jit_EX(opline), ref);
+ }
+}
+
+static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)
+{
+ jit_LOAD_IP(jit, ir_CONST_ADDR(target));
+}
+
+static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)
+{
+ jit->use_last_valid_opline = 0;
+ jit->track_last_valid_opline = 1;
+}
+
+static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)
+{
+ if (jit->track_last_valid_opline) {
+ jit->use_last_valid_opline = 1;
+ jit->track_last_valid_opline = 0;
+ }
+}
+
+static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)
+{
+ return jit->use_last_valid_opline;
+}
+
+static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)
+{
+ if (!jit->reuse_ip) {
+ jit->track_last_valid_opline = 1;
+ jit->last_valid_opline = opline;
+ }
+}
+
+static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)
+{
+ jit->track_last_valid_opline = 0;
+ jit->last_valid_opline = NULL;
+}
+
+static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)
+{
+ zend_jit_reset_last_valid_opline(jit);
+ jit->reuse_ip = 1;
+}
+
+static int zend_jit_reuse_ip(zend_jit_ctx *jit)
+{
+ if (!jit->reuse_ip) {
+ zend_jit_start_reuse_ip(jit);
+ // RX = EX(call);
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
+ }
+ return 1;
+}
+
+static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)
+{
+ jit->reuse_ip = 0;
+}
+
+static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)
+{
+ ir_ref rx, call;
+
+ if (call_level == 1) {
+ // JIT: call = NULL;
+ call = IR_NULL;
+ } else {
+ // JIT: call = EX(call);
+ call = ir_LOAD_A(jit_EX(call));
+ }
+
+ rx = jit_IP(jit);
+
+ // JIT: call->prev_execute_data = call;
+ ir_STORE(jit_CALL(rx, prev_execute_data), call);
+
+ // JIT: EX(call) = call;
+ ir_STORE(jit_EX(call), rx);
+
+ jit->delayed_call_level = 0;
+ delayed_call_chain = 0;
+
+ return 1;
+}
+
+static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)
+{
+ ir_ref ref;
+ ir_ref addr = IR_UNUSED;
+
+ if (jit->delayed_call_level) {
+ if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (jit->last_valid_opline) {
+ zend_jit_use_last_valid_opline(jit);
+ if (jit->last_valid_opline != target) {
+ if (GCC_GLOBAL_REGS) {
+ ref = jit_IP(jit);
+ } else {
+ addr = jit_EX(opline);
+ ref = ir_LOAD_A(addr);
+ }
+ if (target > jit->last_valid_opline) {
+ ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline);
+ } else {
+ ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target));
+ }
+ if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, ref);
+ } else {
+ ir_STORE(addr, ref);
+ }
+ }
+ } else {
+ if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, ir_CONST_ADDR(target));
+ } else {
+ ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
+ }
+ }
+ jit->reuse_ip = 0;
+ zend_jit_set_last_valid_opline(jit, target);
+ return 1;
+}
+
+static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg)
+{
+ if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) {
+ /* Optimization to avoid duplicate constant load */
+ ir_STORE(jit_EX(opline), ir_HARD_COPY_A(ir_CONST_ADDR(target)));
+ return 1;
+ }
+ return zend_jit_set_ip(jit, target);
+}
+
+static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target)
+{
+ if (jit->last_valid_opline == target) {
+ zend_jit_use_last_valid_opline(jit);
+ if (GCC_GLOBAL_REGS) {
+ // EX(opline) = opline
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+ }
+ } else {
+ ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
+ if (!GCC_GLOBAL_REGS) {
+ zend_jit_reset_last_valid_opline(jit);
+ }
+ }
+}
+
+static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_MEM_ZVAL) {
+ ir_ref reg;
+
+ if (Z_REG(addr) == ZREG_FP) {
+ reg = jit_FP(jit);
+ } else if (Z_REG(addr) == ZREG_RX) {
+ reg = jit_IP(jit);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return ir_ADD_OFFSET(reg, Z_OFFSET(addr));
+ } else if (Z_MODE(addr) == IS_REF_ZVAL) {
+ return Z_IR_REF(addr);
+ } else {
+ ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL);
+ return ir_CONST_ADDR(Z_ZV(addr));
+ }
+}
+
+static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type)));
+}
+
+static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr)));
+ } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+ ir_ref reg;
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+ if (Z_REG(addr) == ZREG_FP) {
+ reg = jit_FP(jit);
+ } else if (Z_REG(addr) == ZREG_RX) {
+ reg = jit_IP(jit);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type)));
+ } else {
+ return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags)));
+}
+
+static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr)));
+ } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+ ir_ref reg;
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+ if (Z_REG(addr) == ZREG_FP) {
+ reg = jit_FP(jit);
+ } else if (Z_REG(addr) == ZREG_RX) {
+ reg = jit_IP(jit);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags)));
+ } else {
+ return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)));
+}
+
+static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr)));
+ } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+ ir_ref reg;
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+ if (Z_REG(addr) == ZREG_FP) {
+ reg = jit_FP(jit);
+ } else if (Z_REG(addr) == ZREG_RX) {
+ reg = jit_IP(jit);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)));
+ } else {
+ return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info)
+{
+ ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info);
+}
+
+static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info)
+{
+ if (Z_MODE(addr) == IS_MEM_ZVAL) {
+ ir_ref reg;
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+ if (Z_REG(addr) == ZREG_FP) {
+ reg = jit_FP(jit);
+ } else if (Z_REG(addr) == ZREG_RX) {
+ reg = jit_IP(jit);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info);
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info);
+ }
+}
+
+static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info)
+{
+ if (type_info < IS_STRING
+ && Z_MODE(addr) == IS_MEM_ZVAL
+ && Z_REG(addr) == ZREG_FP
+ && JIT_G(current_frame)
+ && STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) {
+ /* type is already set */
+ return;
+ }
+ jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info));
+}
+
+static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type)
+{
+ return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type));
+}
+
+static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
+{
+ ZEND_ASSERT(type != IS_UNDEF);
+ return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)));
+}
+
+static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
+{
+ ir_ref ref = jit_Z_TYPE(jit, addr);
+
+ if (type != IS_UNDEF) {
+ ref = ir_NE(ref, ir_CONST_U8(type));
+ }
+ return ir_IF(ref);
+}
+
+static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
+{
+ ir_ref ref = jit_Z_TYPE(jit, addr);
+
+ if (type != IS_UNDEF) {
+ ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+}
+
+static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
+{
+ ir_ref ref = jit_Z_TYPE(jit, addr);
+
+ if (type != IS_UNDEF) {
+ ref = ir_NE(ref, ir_CONST_U8(type));
+ }
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+}
+
+static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ return ir_IF(jit_Z_TYPE_FLAGS(jit, addr));
+}
+
+static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref)
+{
+ return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE)));
+}
+
+static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_L(ref);
+}
+
+static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_D(ref);
+}
+
+static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
+{
+ if (jit->ctx.ir_base[val].op == IR_RLOAD) {
+ /* Deoptimization */
+ return 0;
+ }
+// if (jit->ctx.ir_base[val].op == IR_LOAD
+// && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
+// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
+// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
+// && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
+// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
+// /* LOAD from the same location (the LOAD is pinned) */
+// // TODO: should be anti-dependent with the following stores ???
+// return 0;
+// }
+ if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
+ /* IS_CV */
+ if (jit->ctx.ir_base[val].op == IR_LOAD
+ && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
+ && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
+ && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
+ && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
+ && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr != (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)
+ && EX_VAR_TO_NUM(jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr) < jit->current_op_array->last_var) {
+ /* binding between different CVs may cause spill conflict */
+ return 1;
+ } else if (jit->ssa->vars[var].definition >= 0
+ && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var
+ && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0
+ && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].no_val
+ && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi
+ && (jit->ssa->cfg.blocks[jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi->block].flags & ZEND_BB_LOOP_HEADER)) {
+ /* Avoid moving spill store out of loop */
+ return 1;
+ } else if (jit->ssa->vars[var].definition >= 0
+ && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var
+ && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0
+ && jit->ssa->ops[jit->ssa->vars[var].definition].op2_use >= 0
+ && jit->ra[jit->ssa->ops[jit->ssa->vars[var].definition].op2_use].ref == val) {
+ /* Avoid spill conflict between of ASSIGN.op1_def and ASSIGN.op1_use */
+ return 1;
+ }
+ return 0;
+ }
+ return 1;
+}
+
+static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
+{
+ int var;
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_REG);
+ var = Z_SSA_VAR(addr);
+ if (var == jit->delay_var) {
+ ir_refs_add(jit->delay_refs, val);
+ return;
+ }
+ ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
+
+ /* Negative "var" has special meaning for IR */
+ if (val > 0) {
+ if (jit->ctx.binding) {
+ ir_ref old = ir_binding_find(&jit->ctx, val);
+ if (old && old != -EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
+ val = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[val].type), val, 1);
+ }
+ }
+ if (!zend_jit_spilling_may_cause_conflict(jit, var, val)) {
+ val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
+ }
+ }
+ jit->ra[var].ref = val;
+
+ if (jit->ra[var].flags & ZREG_FORWARD) {
+ zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain;
+ zend_basic_block *bb;
+ int n, j, *p;
+ ir_ref *q;
+
+ jit->ra[var].flags &= ~ZREG_FORWARD;
+ while (phi != NULL) {
+ zend_ssa_phi *dst_phi = phi;
+ int src_var = var;
+
+ if (dst_phi->pi >= 0) {
+ jit->ra[src_var].ref = val;
+ src_var = dst_phi->ssa_var;
+ if (!(jit->ra[src_var].flags & ZREG_FORWARD)) {
+ phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
+ continue;
+ }
+ dst_phi = jit->ssa->vars[src_var].phi_use_chain;
+ ZEND_ASSERT(dst_phi != NULL && "reg forwarding");
+ ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding");
+ jit->ra[src_var].flags &= ~ZREG_FORWARD;
+ }
+
+ if (jit->ra[dst_phi->ssa_var].ref > 0) {
+ ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref];
+ if (phi_insn->op == IR_PHI) {
+// ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1);
+ bb = &jit->ssa->cfg.blocks[dst_phi->block];
+ n = bb->predecessors_count;
+ for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) {
+ if (*p == src_var) {
+ *q = val;
+ }
+ }
+ }
+ }
+
+ phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
+ }
+ }
+}
+
+static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ int var = Z_SSA_VAR(addr);
+
+ ZEND_ASSERT(Z_MODE(addr) == IS_REG);
+ ZEND_ASSERT(jit->ra && jit->ra[var].ref);
+ if (jit->ra[var].ref == IR_NULL) {
+ zend_jit_addr mem_addr;
+ ir_ref ref;
+
+ ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD);
+ mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var));
+ if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) {
+ ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
+ } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ zend_jit_def_reg(jit, addr, ref);
+ return ref;
+ }
+ return jit->ra[Z_SSA_VAR(addr)].ref;
+}
+
+static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi)
+{
+ int src_var = phi->sources[0];
+ int dst_var = phi->ssa_var;
+
+ ZEND_ASSERT(phi->pi >= 0);
+ ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
+ ZEND_ASSERT(jit->ra[src_var].ref);
+
+ if (jit->ra[src_var].ref == IR_NULL) {
+ /* Not defined yet */
+ if (jit->ssa->vars[dst_var].use_chain < 0
+ && jit->ssa->vars[dst_var].phi_use_chain) {
+ zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain;
+ if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) {
+ /* This is a Pi forwarded to Phi */
+ jit->ra[src_var].flags |= ZREG_FORWARD;
+ return;
+ }
+ }
+ ZEND_ASSERT(0 && "Not defined Pi source");
+ }
+ /* Reuse register */
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var),
+ zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)));
+}
+
+static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
+{
+ int dst_var = phi->ssa_var;
+ zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block];
+ int n = bb->predecessors_count;
+ int i;
+ ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE;
+ ir_ref merge = jit->bb_start_ref[phi->block];
+ ir_ref ref;
+ ir_ref old_insns_count = jit->ctx.insns_count;
+ ir_ref same_src_ref = IR_UNUSED;
+ bool phi_inputs_are_the_same = 1;
+
+ ZEND_ASSERT(phi->pi < 0);
+ ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
+ ZEND_ASSERT(merge);
+ ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN);
+ ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count);
+
+ ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1);
+ ir_set_op(&jit->ctx, ref, 1, merge);
+
+ for (i = 0; i < n; i++) {
+ int src_var = phi->sources[i];
+
+ ZEND_ASSERT(jit->ra[src_var].ref);
+ if (jit->ra[src_var].ref == IR_NULL) {
+ jit->ra[src_var].flags |= ZREG_FORWARD;
+ phi_inputs_are_the_same = 0;
+ } else {
+ ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var));
+ if (i == 0) {
+ same_src_ref = src_ref;
+ } else if (same_src_ref != src_ref) {
+ phi_inputs_are_the_same = 0;
+ }
+ ir_set_op(&jit->ctx, ref, i + 2, src_ref);
+ }
+ }
+ if (phi_inputs_are_the_same) {
+ ref = same_src_ref;
+ jit->ctx.insns_count = old_insns_count;
+ }
+
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
+}
+
+static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr)));
+ } else if (Z_MODE(addr) == IS_REG) {
+ return zend_jit_use_reg(jit, addr);
+ } else {
+ return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
+{
+ if (Z_MODE(addr) == IS_REG) {
+ zend_jit_def_reg(jit, addr, lval);
+ } else {
+ ir_STORE(jit_ZVAL_ADDR(jit, addr), lval);
+ }
+}
+
+#if SIZEOF_ZEND_LONG == 4
+static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_U32((Z_ZV(addr))->value.ww.w2);
+ } else {
+ return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)));
+ }
+}
+
+static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
+{
+ ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval);
+}
+#endif
+
+static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr)));
+ } else if (Z_MODE(addr) == IS_REG) {
+ return zend_jit_use_reg(jit, addr);
+ } else {
+ return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval)
+{
+ if (Z_MODE(addr) == IS_REG) {
+ zend_jit_def_reg(jit, addr, dval);
+ } else {
+ ir_STORE(jit_ZVAL_ADDR(jit, addr), dval);
+ }
+}
+
+static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_A(ref);
+}
+
+static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ if (Z_MODE(addr) == IS_CONST_ZVAL) {
+ return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr)));
+ } else {
+ return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr));
+ }
+}
+
+static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr)
+{
+ ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr);
+}
+
+static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_LOAD_U32(ref);
+}
+
+static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount)
+{
+ ir_STORE(ref, ir_CONST_U32(refcount));
+}
+
+static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1)));
+}
+
+static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_ref counter = ir_LOAD_U32(ref);
+ ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2)));
+}
+
+static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_ref counter = ir_LOAD_U32(ref);
+ counter = ir_SUB_U32(counter, ir_CONST_U32(1));
+ ir_STORE(ref, counter);
+ return counter;
+}
+
+static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_IF(
+ ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))),
+ ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))));
+}
+
+static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref)
+{
+ ir_ref ref = IR_UNUSED;
+
+ if (Z_TYPE_P(zv) > IS_TRUE) {
+ if (Z_TYPE_P(zv) == IS_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv)));
+ } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv)));
+ } else if (Z_TYPE_P(zv) == IS_LONG) {
+ jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv)));
+ } else {
+ ref = ir_CONST_ADDR(Z_PTR_P(zv));
+ jit_set_Z_PTR(jit, dst, ref);
+ if (addref && Z_REFCOUNTED_P(zv)) {
+ jit_GC_ADDREF(jit, ref);
+ }
+ }
+ }
+ if (Z_MODE(dst) != IS_REG) {
+ if (dst_def_info == MAY_BE_DOUBLE) {
+ if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
+ jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv));
+ }
+ }
+}
+
+static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref)
+{
+ return ir_IF(ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr))));
+}
+
+static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, bool addref)
+{
+ ir_ref ref = IR_UNUSED;
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+ if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src));
+ } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
+ jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src));
+ }
+#endif
+ ref = jit_Z_PTR(jit, src);
+ jit_set_Z_PTR(jit, dst, ref);
+ }
+ }
+ if (has_concrete_type(src_info & MAY_BE_ANY)
+ && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
+ && !(src_info & MAY_BE_GUARD)) {
+ if (Z_MODE(dst) != IS_REG
+ && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
+ uint8_t type = concrete_type(src_info);
+ jit_set_Z_TYPE_INFO(jit, dst, type);
+ }
+ } else {
+ ir_ref type = jit_Z_TYPE_INFO(jit, src);
+ jit_set_Z_TYPE_INFO_ex(jit, dst, type);
+ if (addref) {
+ if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ ir_ref if_refcounted = IR_UNUSED;
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
+ ir_IF_TRUE(if_refcounted);
+ }
+
+ jit_GC_ADDREF(jit, ref);
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
+ }
+ }
+ }
+ }
+}
+
+static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, int addref)
+{
+ ir_ref ref = IR_UNUSED;
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+ if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
+ ref = jit_Z_LVAL(jit, src);
+ jit_set_Z_LVAL(jit, dst, ref);
+ jit_set_Z_LVAL(jit, dst2, ref);
+ } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
+ ref = jit_Z_DVAL(jit, src);
+ jit_set_Z_DVAL(jit, dst, ref);
+ jit_set_Z_DVAL(jit, dst2, ref);
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
+ ref = jit_Z_W2(jit, src);
+ jit_set_Z_W2(jit, dst, ref);
+ jit_set_Z_W2(jit, dst2, ref);
+ }
+#endif
+ ref = jit_Z_PTR(jit, src);
+ jit_set_Z_PTR(jit, dst, ref);
+ jit_set_Z_PTR(jit, dst2, ref);
+ }
+ }
+ if (has_concrete_type(src_info & MAY_BE_ANY)
+ && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
+ && !(src_info & MAY_BE_GUARD)) {
+ uint8_t type = concrete_type(src_info);
+ ir_ref type_ref = ir_CONST_U32(type);
+
+ if (Z_MODE(dst) != IS_REG
+ && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
+ jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref);
+ }
+ if (Z_MODE(dst2) != IS_REG) {
+ jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref);
+ }
+ } else {
+ ir_ref type = jit_Z_TYPE_INFO(jit, src);
+ jit_set_Z_TYPE_INFO_ex(jit, dst, type);
+ jit_set_Z_TYPE_INFO_ex(jit, dst2, type);
+ if (addref) {
+ if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ ir_ref if_refcounted = IR_UNUSED;
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
+ ir_IF_TRUE(if_refcounted);
+ }
+
+ if (addref == 2) {
+ jit_GC_ADDREF2(jit, ref);
+ } else {
+ jit_GC_ADDREF(jit, ref);
+ }
+
+ if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
+ }
+ }
+ }
+ }
+}
+
+static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline)
+{
+ if (!((op_info) & MAY_BE_GUARD)
+ && has_concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ uint8_t type = concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
+ if (type == IS_STRING && !ZEND_DEBUG) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(_efree), ref);
+ return;
+ } else if (type == IS_ARRAY) {
+ if ((op_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
+ if (opline && ((op_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_array_destroy), ref);
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_array_free), ref);
+ }
+ return;
+ } else if (type == IS_OBJECT) {
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
+ return;
+ }
+ }
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(rc_dtor_func), ref);
+}
+
+static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit,
+ zend_jit_addr addr,
+ uint32_t op_info,
+ bool gc,
+ const zend_op *opline)
+{
+ ir_ref ref, ref2;
+ ir_ref if_refcounted = IR_UNUSED;
+ ir_ref if_not_zero = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+
+ if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
+ if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if_refcounted = jit_if_REFCOUNTED(jit, addr);
+ ir_IF_FALSE(if_refcounted);
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_refcounted);
+ }
+ ref = jit_Z_PTR(jit, addr);
+ ref2 = jit_GC_DELREF(jit, ref);
+
+ if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
+ if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
+ if_not_zero = ir_IF(ref2);
+ ir_IF_FALSE(if_not_zero);
+ }
+ // zval_dtor_func(r);
+ jit_ZVAL_DTOR(jit, ref, op_info, opline);
+ if (if_not_zero) {
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_not_zero);
+ }
+ }
+ if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
+ ir_ref if_may_not_leak;
+
+ if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
+ ir_ref if_ref, if_collectable;
+
+ if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE);
+ ir_IF_TRUE(if_ref);
+
+ ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+
+ if_collectable = jit_if_COLLECTABLE_ref(jit, ref2);
+ ir_IF_FALSE(if_collectable);
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_collectable);
+
+ ref2 = jit_Z_PTR_ref(jit, ref2);
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_ref);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ }
+
+ if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
+ ir_IF_TRUE(if_may_not_leak);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_may_not_leak);
+
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
+ }
+
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+ }
+}
+
+static void jit_FREE_OP(zend_jit_ctx *jit,
+ uint8_t op_type,
+ znode_op op,
+ uint32_t op_info,
+ const zend_op *opline)
+{
+ if (op_type & (IS_VAR|IS_TMP_VAR)) {
+ jit_ZVAL_PTR_DTOR(jit,
+ ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var),
+ op_info, 0, opline);
+ }
+}
+
+static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref if_not_zero, if_may_not_leak;
+
+ // JIT: if (GC_DELREF(obj) == 0) {
+ if_not_zero = ir_IF(jit_GC_DELREF(jit, ref));
+ ir_IF_FALSE(if_not_zero);
+
+ // JIT: zend_objects_store_del(obj)
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_not_zero);
+ if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
+
+ ir_IF_TRUE(if_may_not_leak);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE(if_may_not_leak);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
+ ir_END_list(end_inputs);
+
+ ir_MERGE_list(end_inputs);
+}
+
+static void zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr)
+{
+ ir_ref ref = ir_LOAD_U8(jit_EG(vm_interrupt));
+
+ if (exit_addr) {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ } else if (!opline || jit->last_valid_opline == opline) {
+ ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
+ } else {
+ ir_ref if_timeout = ir_IF(ref);
+
+ ir_IF_TRUE_cold(if_timeout);
+ jit_LOAD_IP_ADDR(jit, opline);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
+ ir_IF_FALSE(if_timeout);
+ }
+}
+
+/* stubs */
+
+static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
+{
+ const void *handler;
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ handler = zend_get_opcode_handler_func(EG(exception_op));
+
+ ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ handler = EG(exception_op)->handler;
+
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
+ } else {
+ ir_ref ref, if_negative;
+
+ ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit));
+ if_negative = ir_IF(ir_LT(ref, ir_CONST_U32(0)));
+ ir_IF_TRUE(if_negative);
+ ir_MERGE_WITH_EMPTY_FALSE(if_negative);
+ ref = ir_PHI_2(IR_I32, ref, ir_CONST_I32(1));
+ ir_RETURN(ref);
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit)
+{
+ ir_ref ref, result_type, if_result_used;
+
+ ref = jit_EG(opline_before_exception);
+ result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type)));
+
+ if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
+ ir_IF_TRUE(if_result_used);
+
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF));
+ ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ return 1;
+}
+
+static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit)
+{
+ ir_ref ref, if_dtor;
+ zend_jit_addr var_addr;
+
+ ref = ir_LOAD_A(jit_EG(opline_before_exception));
+ if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))),
+ ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
+ ir_IF_TRUE(if_dtor);
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ ref = ir_ADD_A(jit_FP(jit), ref);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
+ ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
+
+ return 1;
+}
+
+static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit)
+{
+ ir_ref ref, if_dtor;
+ zend_jit_addr var_addr;
+
+ ref = ir_LOAD_A(jit_EG(opline_before_exception));
+ if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))),
+ ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
+ ir_IF_TRUE(if_dtor);
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ ref = ir_ADD_A(jit_FP(jit), ref);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
+ ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
+
+ return 1;
+}
+
+static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
+{
+ ir_ref if_timeout, if_exception;
+
+ if (GCC_GLOBAL_REGS) {
+ // EX(opline) = opline
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+ }
+
+ ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0));
+ if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0)));
+ ir_IF_FALSE(if_timeout);
- ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout));
++ ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout)); // noreturn
+ ir_MERGE_WITH_EMPTY_TRUE(if_timeout);
+
++ // if (ZEND_HAS_OOM()) {
++ ir_ref if_delayed_oom = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(jit_EG(delayed_oom), offsetof(zend_oom_info, state))));
++ ir_IF_TRUE(if_delayed_oom);
++ ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_handle_delayed_oom));
++ // LOAD_OPLINE()
++ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
++ ir_ref if_delayed_oom_end = ir_END();
++ ir_IF_FALSE(if_delayed_oom);
++
+ if (zend_interrupt_function) {
+ ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit));
+ if_exception = ir_IF(ir_LOAD_A(jit_EG(exception)));
+ ir_IF_TRUE(if_exception);
+ ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper));
+ ir_MERGE_WITH_EMPTY_FALSE(if_exception);
+
+ jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
+ }
+
++ ir_MERGE_WITH(if_delayed_oom_end);
++
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_RETURN(ir_CONST_I32(1));
+ }
+ return 1;
+}
+
+static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
+{
+ ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
+ ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
+
+ ir_IF_FALSE(if_top);
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
+ jit_STORE_IP(jit,
+ ir_LOAD_A(jit_EX(opline)));
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
+ } else {
+ ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit));
+ }
+
+ ir_IF_TRUE(if_top);
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
+ } else {
+ ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit));
+ }
+
+ return 1;
+}
+
+static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
+{
+ ir_CALL_2(IR_VOID,
+ ir_CONST_FUNC_PROTO(zend_throw_error,
+ ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
+ ir_CONST_ADDR(zend_ce_arithmetic_error),
+ ir_CONST_ADDR("Bit shift by negative number"));
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
+ return 1;
+}
+
+static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit)
+{
+ ir_CALL_2(IR_VOID,
+ ir_CONST_FUNC_PROTO(zend_throw_error,
+ ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
+ ir_CONST_ADDR(zend_ce_division_by_zero_error),
+ ir_CONST_ADDR("Modulo by zero"));
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
+ return 1;
+}
+
+static int zend_jit_invalid_this_stub(zend_jit_ctx *jit)
+{
+ ir_CALL_2(IR_VOID,
+ ir_CONST_FUNC_PROTO(zend_throw_error,
+ ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
+ IR_NULL,
+ ir_CONST_ADDR("Using $this when not in object context"));
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
+ return 1;
+}
+
+static int zend_jit_undefined_function_stub(zend_jit_ctx *jit)
+{
+ // JIT: load EX(opline)
+ ir_ref ref = ir_LOAD_A(jit_FP(jit));
+ ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant)));
+
+ if (sizeof(void*) == 8) {
+ arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3)));
+ } else {
+ arg3 = ir_LOAD_A(arg3);
+ }
+ arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val));
+
+ ir_CALL_3(IR_VOID,
+ ir_CONST_FUNC_PROTO(zend_throw_error,
+ ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
+ IR_NULL,
+ ir_CONST_ADDR("Call to undefined function %s()"),
+ arg3);
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ return 1;
+}
+
+static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit)
+{
+ ir_ref opline, ref, rx, if_eq, if_tmp;
+
+ // JIT: opline = EX(opline)
+ opline = ir_LOAD_A(jit_FP(jit));
+
+ // JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var))
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ rx = jit_IP(jit);
+ jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF));
+
+ // last EX(call) frame may be delayed
+ // JIT: if (EX(call) == RX)
+ ref = ir_LOAD_A(jit_EX(call));
+ if_eq = ir_IF(ir_EQ(rx, ref));
+ ir_IF_FALSE(if_eq);
+
+ // JIT: RX->prev_execute_data == EX(call)
+ ir_STORE(jit_CALL(rx, prev_execute_data), ref);
+
+ // JIT: EX(call) = RX
+ ir_STORE(jit_EX(call), rx);
+ ir_MERGE_WITH_EMPTY_TRUE(if_eq);
+
+ // JIT: IP = opline
+ jit_STORE_IP(jit, opline);
+
+ // JIT: zend_cannot_pass_by_reference(opline->op2.num)
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference),
+ ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num))));
+
+ // JIT: if (IP->op1_type == IS_TMP_VAR)
+ ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type)));
+ if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR)));
+ ir_IF_TRUE(if_tmp);
+
+ // JIT: zval_ptr_dtor(EX_VAR(IP->op1.var))
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ ref = ir_ADD_A(jit_FP(jit), ref);
+ jit_ZVAL_PTR_DTOR(jit,
+ ZEND_ADDR_REF_ZVAL(ref),
+ MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
+ ir_MERGE_WITH_EMPTY_FALSE(if_tmp);
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ return 1;
+}
+
+static int zend_jit_icall_throw_stub(zend_jit_ctx *jit)
+{
+ ir_ref ip, if_set;
+
+ // JIT: zend_rethrow_exception(zend_execute_data *execute_data)
+ // JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
+ ip = jit_IP(jit);
+ if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
+ ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
+ ir_IF_FALSE(if_set);
+
+ // JIT: EG(opline_before_exception) = opline;
+ ir_STORE(jit_EG(opline_before_exception), ip);
+ ir_MERGE_WITH_EMPTY_TRUE(if_set);
+
+ // JIT: opline = EG(exception_op);
+ jit_STORE_IP(jit, jit_EG(exception_op));
+
+ if (GCC_GLOBAL_REGS) {
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+ }
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ return 1;
+}
+
+static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
+{
+ ir_ref ip, if_set;
+
+ // JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
+ ip = jit_IP(jit);
+ if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
+ ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
+ ir_IF_FALSE(if_set);
+
+ // JIT: EG(opline_before_exception) = opline;
+ ir_STORE(jit_EG(opline_before_exception), ip);
+ ir_MERGE_WITH_EMPTY_TRUE(if_set);
+
+ // JIT: opline = EG(exception_op);
+ jit_LOAD_IP(jit, jit_EG(exception_op));
+
+ if (GCC_GLOBAL_REGS) {
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+
+ // JIT: HANDLE_EXCEPTION()
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ } else {
+ ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
+ }
+
+ return 1;
+}
+
+static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
+ return 0;
+ }
+
+ ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
+ ir_IJMP(ir_LOAD_A(jit_IP(jit)));
+ return 1;
+}
+
+static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit)
+{
+ ir_ref addr, func, run_time_cache, jit_extension;
+
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
+ return 0;
+ }
+
+ addr = ir_CONST_ADDR(&zend_jit_profile_counter),
+ ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
+
+ func = ir_LOAD_A(jit_EX(func));
+ run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
+
+ if (zend_jit_profile_counter_rid) {
+ addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*));
+ } else {
+ addr = run_time_cache;
+ }
+ ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
+
+ addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler));
+ ir_IJMP(ir_LOAD_A(addr));
+
+ return 1;
+}
+
+static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost)
+{
+ ir_ref func, jit_extension, addr, ref, if_overflow;
+
+ func = ir_LOAD_A(jit_EX(func));
+ jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
+ addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter)));
+ ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
+ ir_STORE(addr, ref);
+ if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
+
+ ir_IF_TRUE_cold(if_overflow);
+ ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func),
+ jit_FP(jit),
+ jit_IP(jit));
+ ir_IJMP(ir_LOAD_A(jit_IP(jit)));
+
+ ir_IF_FALSE(if_overflow);
+ ref = ir_SUB_A(jit_IP(jit),
+ ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes))));
+ ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*)));
+
+ addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)),
+ ref);
+ ir_IJMP(ir_LOAD_A(addr));
+
+ return 1;
+}
+
+static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
+ return 0;
+ }
+
+ return _zend_jit_hybrid_hot_counter_stub(jit,
+ ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
+}
+
+static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
+ return 0;
+ }
+
+ return _zend_jit_hybrid_hot_counter_stub(jit,
+ ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
+}
+
+static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset)
+{
+ ir_ref addr;
+
+ if (GCC_GLOBAL_REGS) {
+ addr = ir_ADD_A(offset, jit_IP(jit));
+ } else {
+ addr = ir_ADD_A(offset, ir_LOAD_A(jit_EX(opline)));
+ }
+
+ return ir_LOAD_A(addr);
+}
+
+static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit)
+{
+ ir_ref func, jit_extension, offset;
+
+ func = ir_LOAD_A(jit_EX(func));
+ jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
+ offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
+ return _zend_jit_orig_opline_handler(jit, offset);
+}
+
+static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
+{
+ ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt;
+
+ func = ir_LOAD_A(jit_EX(func));
+ jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
+ offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
+ addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter)));
+ ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
+ ir_STORE(addr, ref);
+ if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
+
+ ir_IF_TRUE_cold(if_overflow);
+ ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
+ ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root),
+ jit_FP(jit),
+ jit_IP(jit));
+ if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0)));
+ ir_IF_FALSE(if_halt);
+
+ ref = jit_EG(current_execute_data);
+ jit_STORE_FP(jit, ir_LOAD_A(ref));
+ ref = ir_LOAD_A(jit_EX(opline));
+ jit_STORE_IP(jit, ref);
+ ir_IJMP(ir_LOAD_A(jit_IP(jit)));
+
+ ir_IF_FALSE(if_overflow);
+ ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
+
+ ir_IF_TRUE(if_halt);
+ ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
+
+ return 1;
+}
+
+static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
+ return 0;
+ }
+
+ return _zend_jit_hybrid_trace_counter_stub(jit,
+ ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
+}
+
+static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
+ return 0;
+ }
+
+ return _zend_jit_hybrid_trace_counter_stub(jit,
+ ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
+}
+
+static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
+ return 0;
+ }
+
+ return _zend_jit_hybrid_trace_counter_stub(jit,
+ ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
+}
+
+static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
+{
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
+ } else if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, IR_NULL);
+ ir_RETURN(IR_VOID);
+ } else {
+ ir_RETURN(ir_CONST_I32(-1)); // ZEND_VM_RETURN
+ }
+ return 1;
+}
+
+static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
+{
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
+ }
+
+ return 1;
+}
+
+static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
+{
+ ir_ref ref, ret, if_zero, addr;
+
+ if (GCC_GLOBAL_REGS) {
+ // EX(opline) = opline
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+ }
+
+ ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit));
+
+ if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0)));
+
+ ir_IF_TRUE(if_zero);
+
+ if (GCC_GLOBAL_REGS) {
+ ref = jit_EG(current_execute_data);
+ jit_STORE_FP(jit, ir_LOAD_A(ref));
+ ref = ir_LOAD_A(jit_EX(opline));
+ jit_STORE_IP(jit, ref);
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
+ }
+
+ ir_IF_FALSE(if_zero);
+
+ ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+
+ ref = jit_EG(current_execute_data);
+ jit_STORE_FP(jit, ir_LOAD_A(ref));
+
+ if (GCC_GLOBAL_REGS) {
+ ref = ir_LOAD_A(jit_EX(opline));
+ jit_STORE_IP(jit, ref);
+ }
+
+ // check for interrupt (try to avoid this ???)
+ zend_jit_check_timeout(jit, NULL, NULL);
+
+ addr = zend_jit_orig_opline_handler(jit);
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, addr);
+ } else {
+#if defined(IR_TARGET_X86)
+ addr = ir_CAST_FC_FUNC(addr);
+#endif
+ ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
+ ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
+ }
+
+ return 1;
+}
+
+static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
+{
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
+ } else {
+ ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
+ }
+
+ return 1;
+}
+
+static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
+{
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
+ } else {
+ ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
+ }
+
+ return 1;
+}
+
+static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit)
+{
+ ir_ref opline = ir_LOAD_A(jit_EX(opline));
+ ir_ref ref, if_result_used;
+
+ if_result_used = ir_IF(ir_AND_U8(
+ ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))),
+ ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
+ ir_IF_TRUE(if_result_used);
+
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+ jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF));
+ ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
+
+ ir_CALL_2(IR_VOID,
+ ir_CONST_FUNC_PROTO(zend_throw_error,
+ ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
+ IR_NULL,
+ ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied"));
+ ir_RETURN(IR_VOID);
+
+ return 1;
+}
+
+static int zend_jit_assign_const_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
+
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
+ uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
+
+ if (!zend_jit_assign_to_variable(
+ jit, NULL,
+ var_addr, var_addr, -1, -1,
+ IS_CONST, val_addr, val_info,
+ 0, 0, 0)) {
+ return 0;
+ }
+ ir_RETURN(IR_VOID);
+ return 1;
+}
+
+static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
+
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
+ uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
+
+ if (!zend_jit_assign_to_variable(
+ jit, NULL,
+ var_addr, var_addr, -1, -1,
+ IS_TMP_VAR, val_addr, val_info,
+ 0, 0, 0)) {
+ return 0;
+ }
+ ir_RETURN(IR_VOID);
+ return 1;
+}
+
+static int zend_jit_assign_var_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
+
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
+ uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
+
+ if (!zend_jit_assign_to_variable(
+ jit, NULL,
+ var_addr, var_addr, -1, -1,
+ IS_VAR, val_addr, val_info,
+ 0, 0, 0)) {
+ return 0;
+ }
+ ir_RETURN(IR_VOID);
+ return 1;
+}
+
+static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
+
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
+ uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
+
+ if (!zend_jit_assign_to_variable(
+ jit, NULL,
+ var_addr, var_addr, -1, -1,
+ IS_CV, val_addr, val_info,
+ 0, 0, 0)) {
+ return 0;
+ }
+ ir_RETURN(IR_VOID);
+ return 1;
+}
+
+static int zend_jit_new_array_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0));
+
+ jit_set_Z_PTR(jit, var_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX);
+ ir_RETURN(ref);
+ return 1;
+}
+
+static int zend_jit_assign_cv_stub(zend_jit_ctx *jit)
+{
+ ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
+ ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
+
+ zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
+ uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
+
+ if (!zend_jit_assign_to_variable(
+ jit, NULL,
+ var_addr, var_addr, -1, -1,
+ IS_CV, val_addr, val_info,
+ 0, 0, 0)) {
+ return 0;
+ }
+ ir_RETURN(IR_VOID);
+ return 1;
+}
+
+static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
+{
+#if defined (__CET__) && (__CET__ & 1) != 0
+ flags |= IR_GEN_ENDBR;
+#endif
+ flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
+
+ ir_init(&jit->ctx, flags, 256, 1024);
+ jit->ctx.ret_type = -1;
+
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+ jit->ctx.mflags |= default_mflags;
+ if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) {
+ jit->ctx.mflags |= IR_X86_AVX;
+ }
+#elif defined(IR_TARGET_AARCH64)
+ jit->ctx.get_veneer = zend_jit_get_veneer;
+ jit->ctx.set_veneer = zend_jit_set_veneer;
+#endif
+
+ jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
+ if (!(flags & IR_FUNCTION)) {
+ jit->ctx.flags |= IR_NO_STACK_COMBINE;
+ if (zend_jit_vm_kind == ZEND_VM_KIND_CALL) {
+ jit->ctx.flags |= IR_FUNCTION;
+ /* Stack must be 16 byte aligned */
+ /* TODO: select stack size ??? */
+#if defined(IR_TARGET_AARCH64)
+ jit->ctx.flags |= IR_USE_FRAME_POINTER;
+ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
+#elif defined(_WIN64)
+ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */
+#elif defined(IR_TARGET_X86_64)
+ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */
+#else /* IR_TARGET_x86 */
+ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */
+#endif
+ if (GCC_GLOBAL_REGS) {
+ jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
+ } else {
+ jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
+//#ifdef _WIN64
+// jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ???
+//#endif
+ }
+#ifdef _WIN64
+ jit->ctx.fixed_call_stack_size = 16 + IR_SHADOW_ARGS;
+#else
+ jit->ctx.fixed_call_stack_size = 16;
+#endif
+ } else {
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+ jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE;
+ if (jit->ctx.fixed_stack_red_zone > 16) {
+ jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16;
+ jit->ctx.fixed_call_stack_size = 16;
+ }
+ jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES;
+#else
+ jit->ctx.fixed_stack_red_zone = 0;
+ jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */
+ jit->ctx.fixed_call_stack_size = 16;
+#endif
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+ jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
+#endif
+ }
+ }
+
+ jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT;
+
+ jit->op_array = NULL;
+ jit->current_op_array = NULL;
+ jit->ssa = NULL;
+ jit->name = NULL;
+ jit->last_valid_opline = NULL;
+ jit->use_last_valid_opline = 0;
+ jit->track_last_valid_opline = 0;
+ jit->reuse_ip = 0;
+ jit->delayed_call_level = 0;
+ delayed_call_chain = 0;
+ jit->b = -1;
+#ifdef ZTS
+ jit->tls = IR_UNUSED;
+#endif
+ jit->fp = IR_UNUSED;
+ jit->poly_func_ref = IR_UNUSED;
+ jit->poly_this_ref = IR_UNUSED;
+ jit->trace_loop_ref = IR_UNUSED;
+ jit->return_inputs = IR_UNUSED;
+ jit->bb_start_ref = NULL;
+ jit->bb_predecessors = NULL;
+ jit->bb_edges = NULL;
+ jit->trace = NULL;
+ jit->ra = NULL;
+ jit->delay_var = -1;
+ jit->delay_refs = NULL;
+ jit->eg_exception_addr = 0;
+ zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0);
+ memset(jit->stub_addr, 0, sizeof(jit->stub_addr));
+
+ ir_START();
+}
+
+static int zend_jit_free_ctx(zend_jit_ctx *jit)
+{
+ if (jit->name) {
+ zend_string_release(jit->name);
+ }
+ zend_hash_destroy(&jit->addr_hash);
+ ir_free(&jit->ctx);
+ return 1;
+}
+
+static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name)
+{
+ void *entry;
+ ir_code_buffer code_buffer;
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) {
+ if (name) fprintf(stderr, "%s: ; after folding\n", name);
+ ir_save(ctx, 0, stderr);
+ }
+
+#if ZEND_DEBUG
+ ir_check(ctx);
+#endif
+
+ ir_build_def_use_lists(ctx);
+
+#if ZEND_DEBUG
+ ir_check(ctx);
+#endif
+
+#if 1
+ ir_sccp(ctx);
+#endif
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) {
+ if (name) fprintf(stderr, "%s: ; after SCCP\n", name);
+ ir_save(ctx, 0, stderr);
+ }
+
+ ir_build_cfg(ctx);
+ ir_build_dominators_tree(ctx);
+ ir_find_loops(ctx);
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) {
+ if (name) fprintf(stderr, "%s: ; after CFG\n", name);
+ ir_save(ctx, IR_SAVE_CFG, stderr);
+ }
+
+ ir_gcm(ctx);
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) {
+ if (name) fprintf(stderr, "%s: ; after GCM\n", name);
+ ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr);
+ }
+
+ ir_schedule(ctx);
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) {
+ if (name) fprintf(stderr, "%s: ; after schedule\n", name);
+ ir_save(ctx, IR_SAVE_CFG, stderr);
+ }
+
+ ir_match(ctx);
+#if !defined(IR_TARGET_AARCH64)
+ ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */
+#endif
+ ir_assign_virtual_registers(ctx);
+ ir_compute_live_ranges(ctx);
+ ir_coalesce(ctx);
+ ir_reg_alloc(ctx);
+
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) {
+ if (name) fprintf(stderr, "%s: ; after register allocation\n", name);
+ ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
+ ir_dump_live_ranges(ctx, stderr);
+ }
+
+ ir_schedule_blocks(ctx);
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) {
+ if (name) fprintf(stderr, "%s: ; codegen\n", name);
+ ir_dump_codegen(ctx, stderr);
+ } else {
+ if (name) fprintf(stderr, "%s: ; final\n", name);
+ ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
+ }
+ }
+
+#if ZEND_DEBUG
+ ir_check(ctx);
+#endif
+
+ code_buffer.start = dasm_buf;
+ code_buffer.end = dasm_end;
+ code_buffer.pos = *dasm_ptr;
+ ctx->code_buffer = &code_buffer;
+
+ entry = ir_emit_code(ctx, size);
+
+ *dasm_ptr = code_buffer.pos;
+
+#if defined(IR_TARGET_AARCH64)
+ if (ctx->flags2 & IR_HAS_VENEERS) {
+ zend_jit_commit_veneers();
+ }
+#endif
+
+ return entry;
+}
+
+static void zend_jit_setup_stubs(void)
+{
+ zend_jit_ctx jit;
+ void *entry;
+ size_t size;
+ uint32_t i;
+
+ for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
+ zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags);
+
+ if (!zend_jit_stubs[i].stub(&jit)) {
+ zend_jit_free_ctx(&jit);
+ zend_jit_stub_handlers[i] = NULL;
+ continue;
+ }
+
+ entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name);
+ if (!entry) {
+ zend_jit_free_ctx(&jit);
+ zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub");
+ }
+
+ zend_jit_stub_handlers[i] = entry;
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
+#ifdef HAVE_CAPSTONE
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
+ ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size);
+ }
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) {
+ ir_disasm(zend_jit_stubs[i].name,
+ entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr);
+ }
+#endif
+#ifndef _WIN32
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
+// ir_mem_unprotect(entry, size);
+ ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0);
+// ir_mem_protect(entry, size);
+ }
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ ir_perf_map_register(zend_jit_stubs[i].name, entry, size);
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
+ ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size);
+ }
+ }
+#endif
+ }
+ zend_jit_free_ctx(&jit);
+ }
+}
+
+#define REGISTER_HELPER(n) \
+ ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*));
+#define REGISTER_DATA(n) \
+ ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n));
+
+static void zend_jit_setup_disasm(void)
+{
+#ifdef HAVE_CAPSTONE
+ ir_disasm_init();
+
+ if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) {
+ zend_op opline;
+
+ memset(&opline, 0, sizeof(opline));
+
+ opline.opcode = ZEND_DO_UCALL;
+ opline.result_type = IS_UNUSED;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_DO_UCALL;
+ opline.result_type = IS_VAR;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_DO_FCALL_BY_NAME;
+ opline.result_type = IS_UNUSED;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_DO_FCALL_BY_NAME;
+ opline.result_type = IS_VAR;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_DO_FCALL;
+ opline.result_type = IS_UNUSED;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_DO_FCALL;
+ opline.result_type = IS_VAR;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_RETURN;
+ opline.op1_type = IS_CONST;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_RETURN;
+ opline.op1_type = IS_TMP_VAR;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_RETURN;
+ opline.op1_type = IS_VAR;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ opline.opcode = ZEND_RETURN;
+ opline.op1_type = IS_CV;
+ zend_vm_set_opcode_handler(&opline);
+ ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
+
+ ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*));
+ }
+
+ REGISTER_DATA(zend_jit_profile_counter);
+
+ REGISTER_HELPER(zend_runtime_jit);
+ REGISTER_HELPER(zend_jit_hot_func);
+ REGISTER_HELPER(zend_jit_trace_hot_root);
+ REGISTER_HELPER(zend_jit_trace_exit);
+
+ REGISTER_HELPER(zend_jit_array_free);
+ REGISTER_HELPER(zend_jit_undefined_op_helper);
+ REGISTER_HELPER(zend_jit_pre_inc_typed_ref);
+ REGISTER_HELPER(zend_jit_pre_dec_typed_ref);
+ REGISTER_HELPER(zend_jit_post_inc_typed_ref);
+ REGISTER_HELPER(zend_jit_post_dec_typed_ref);
+ REGISTER_HELPER(zend_jit_pre_inc);
+ REGISTER_HELPER(zend_jit_pre_dec);
+ REGISTER_HELPER(zend_jit_add_arrays_helper);
+ REGISTER_HELPER(zend_jit_fast_assign_concat_helper);
+ REGISTER_HELPER(zend_jit_fast_concat_helper);
+ REGISTER_HELPER(zend_jit_fast_concat_tmp_helper);
+ REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp);
+ REGISTER_HELPER(zend_jit_assign_op_to_typed_ref);
+ REGISTER_HELPER(zend_jit_assign_const_to_typed_ref);
+ REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref);
+ REGISTER_HELPER(zend_jit_assign_var_to_typed_ref);
+ REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref);
+ REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2);
+ REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2);
+ REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2);
+ REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2);
+ REGISTER_HELPER(zend_jit_check_constant);
+ REGISTER_HELPER(zend_jit_get_constant);
+ REGISTER_HELPER(zend_jit_int_extend_stack_helper);
+ REGISTER_HELPER(zend_jit_extend_stack_helper);
+ REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper);
+ REGISTER_HELPER(zend_jit_find_func_helper);
+ REGISTER_HELPER(zend_jit_find_ns_func_helper);
+ REGISTER_HELPER(zend_jit_jmp_frameless_helper);
+ REGISTER_HELPER(zend_jit_unref_helper);
+ REGISTER_HELPER(zend_jit_invalid_method_call);
+ REGISTER_HELPER(zend_jit_invalid_method_call_tmp);
+ REGISTER_HELPER(zend_jit_find_method_helper);
+ REGISTER_HELPER(zend_jit_find_method_tmp_helper);
+ REGISTER_HELPER(zend_jit_push_static_metod_call_frame);
+ REGISTER_HELPER(zend_jit_push_static_metod_call_frame_tmp);
+ REGISTER_HELPER(zend_jit_free_trampoline_helper);
+ REGISTER_HELPER(zend_jit_verify_return_slow);
+ REGISTER_HELPER(zend_jit_deprecated_helper);
+ REGISTER_HELPER(zend_jit_undefined_long_key);
+ REGISTER_HELPER(zend_jit_undefined_long_key_ex);
+ REGISTER_HELPER(zend_jit_undefined_string_key);
+ REGISTER_HELPER(zend_jit_copy_extra_args_helper);
+ REGISTER_HELPER(zend_jit_copy_extra_args_helper_no_skip_recv);
+ REGISTER_HELPER(zend_jit_vm_stack_free_args_helper);
+ REGISTER_HELPER(zend_free_extra_named_params);
+ REGISTER_HELPER(zend_jit_free_call_frame);
+ REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper);
+ REGISTER_HELPER(zend_jit_verify_arg_slow);
+ REGISTER_HELPER(zend_missing_arg_error);
+ REGISTER_HELPER(zend_jit_only_vars_by_reference);
+ REGISTER_HELPER(zend_jit_leave_func_helper);
+ REGISTER_HELPER(zend_jit_leave_nested_func_helper);
+ REGISTER_HELPER(zend_jit_leave_top_func_helper);
+ REGISTER_HELPER(zend_jit_fetch_global_helper);
+ REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed);
+ REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
+ REGISTER_HELPER(zend_jit_hash_lookup_rw);
+ REGISTER_HELPER(zend_jit_symtable_find);
+ REGISTER_HELPER(zend_jit_symtable_lookup_w);
+ REGISTER_HELPER(zend_jit_symtable_lookup_rw);
+ REGISTER_HELPER(zend_jit_fetch_dim_r_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_is_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_isset_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_rw_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_w_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper);
+ REGISTER_HELPER(zend_jit_invalid_array_access);
+ REGISTER_HELPER(zend_jit_zval_array_dup);
+ REGISTER_HELPER(zend_jit_prepare_assign_dim_ref);
+ REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper);
+ REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper);
+ REGISTER_HELPER(zend_jit_isset_dim_helper);
+ REGISTER_HELPER(zend_jit_assign_dim_helper);
+ REGISTER_HELPER(zend_jit_assign_dim_op_helper);
+ REGISTER_HELPER(zend_jit_fetch_obj_w_slow);
+ REGISTER_HELPER(zend_jit_fetch_obj_r_slow);
+ REGISTER_HELPER(zend_jit_fetch_obj_r_slow_ex);
+ REGISTER_HELPER(zend_jit_fetch_obj_is_slow);
+ REGISTER_HELPER(zend_jit_fetch_obj_is_slow_ex);
+ REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic);
+ REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic_ex);
+ REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic);
+ REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic_ex);
+ REGISTER_HELPER(zend_jit_check_array_promotion);
+ REGISTER_HELPER(zend_jit_create_typed_ref);
+ REGISTER_HELPER(zend_jit_invalid_property_write);
+ REGISTER_HELPER(zend_jit_invalid_property_read);
+ REGISTER_HELPER(zend_jit_extract_helper);
+ REGISTER_HELPER(zend_jit_invalid_property_assign);
+ REGISTER_HELPER(zend_jit_assign_to_typed_prop);
+ REGISTER_HELPER(zend_jit_assign_obj_helper);
+ REGISTER_HELPER(zend_jit_invalid_property_assign_op);
+ REGISTER_HELPER(zend_jit_assign_op_to_typed_prop);
+ REGISTER_HELPER(zend_jit_assign_obj_op_helper);
+ REGISTER_HELPER(zend_jit_invalid_property_incdec);
+ REGISTER_HELPER(zend_jit_inc_typed_prop);
+ REGISTER_HELPER(zend_jit_dec_typed_prop);
+ REGISTER_HELPER(zend_jit_pre_inc_typed_prop);
+ REGISTER_HELPER(zend_jit_post_inc_typed_prop);
+ REGISTER_HELPER(zend_jit_pre_dec_typed_prop);
+ REGISTER_HELPER(zend_jit_post_dec_typed_prop);
+ REGISTER_HELPER(zend_jit_pre_inc_obj_helper);
+ REGISTER_HELPER(zend_jit_post_inc_obj_helper);
+ REGISTER_HELPER(zend_jit_pre_dec_obj_helper);
+ REGISTER_HELPER(zend_jit_post_dec_obj_helper);
+ REGISTER_HELPER(zend_jit_rope_end);
+ REGISTER_HELPER(zend_fcall_interrupt);
+
+#ifndef ZTS
+ REGISTER_DATA(EG(current_execute_data));
+ REGISTER_DATA(EG(exception));
+ REGISTER_DATA(EG(opline_before_exception));
+ REGISTER_DATA(EG(vm_interrupt));
+ REGISTER_DATA(EG(timed_out));
++ REGISTER_DATA(EG(delayed_oom));
+ REGISTER_DATA(EG(uninitialized_zval));
+ REGISTER_DATA(EG(zend_constants));
+ REGISTER_DATA(EG(jit_trace_num));
+ REGISTER_DATA(EG(vm_stack_top));
+ REGISTER_DATA(EG(vm_stack_end));
+ REGISTER_DATA(EG(exception_op));
+ REGISTER_DATA(EG(symbol_table));
+
+ REGISTER_DATA(CG(map_ptr_base));
+#endif
+#endif
+}
+
+static void zend_jit_calc_trace_prologue_size(void)
+{
+ zend_jit_ctx jit_ctx;
+ zend_jit_ctx *jit = &jit_ctx;
+ void *entry;
+ size_t size;
+
+ zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
+
+ if (!GCC_GLOBAL_REGS) {
+ ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
+ jit_STORE_FP(jit, ref);
+ jit->ctx.flags |= IR_FASTCALL_FUNC;
+ }
+
+ ir_UNREACHABLE();
+
+ entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue");
+ zend_jit_free_ctx(jit);
+
+ if (!entry) {
+ zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue");
+ }
+
+ zend_jit_trace_prologue_size = size;
+}
+
+#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
+static uintptr_t zend_jit_hybrid_vm_sp_adj = 0;
+
+typedef struct _Unwind_Context _Unwind_Context;
+typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
+extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
+extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
+
+typedef struct _zend_jit_unwind_arg {
+ int cnt;
+ uintptr_t cfa[3];
+} zend_jit_unwind_arg;
+
+static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
+{
+ zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
+ arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
+ arg->cnt++;
+ if (arg->cnt == 3) {
+ return 5; // _URC_END_OF_STACK
+ }
+ return 0; // _URC_NO_REASON;
+}
+
+static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
+{
+ zend_jit_unwind_arg arg;
+
+ memset(&arg, 0, sizeof(arg));
+ _Unwind_Backtrace(zend_jit_unwind_cb, &arg);
+ if (arg.cnt == 3) {
+ zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1];
+ }
+}
+
+extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
+
+static zend_never_inline void zend_jit_set_sp_adj_vm(void)
+{
+ void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
+
+ orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
+ zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
+ execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
+ zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
+}
+#endif
+
+#ifdef _WIN64
+/*
+ * We use a single unwind entry for the whole JIT buffer.
+ * This works, because all the JIT-ed PHP functions have the same "fixed stack frame".
+ */
+static PRUNTIME_FUNCTION zend_jit_uw_func = NULL;
+
+#ifdef ZEND_JIT_RT_UNWINDER
+static PRUNTIME_FUNCTION zend_jit_unwind_callback(DWORD64 pc, PVOID context)
+{
+ return zend_jit_uw_func;
+}
+#endif
+
+static void zend_jit_setup_unwinder(void)
+{
+ /* Hardcoded SEH unwind data for JIT-ed PHP functions with "fixed stack frame" */
+ static const unsigned char uw_data[] = {
+ 0x01, // UBYTE: 3 Version , UBYTE: 5 Flags
+ 0x10, // UBYTE Size of prolog
+ 0x09, // UBYTE Count of unwind codes
+ 0x00, // UBYTE: 4 Frame Register, UBYTE: 4 Frame Register offset (scaled)
+ // USHORT * n Unwind codes array
+ 0x10, 0x82, // c: subq $0x48, %rsp
+ 0x0c, 0xf0, // a: pushq %r15
+ 0x0a, 0xe0, // 8: pushq %r14
+ 0x08, 0xd0, // 6: pushq %r13
+ 0x06, 0xc0, // 4: pushq %r12
+ 0x04, 0x70, // 3: pushq %rdi
+ 0x03, 0x60, // 2: pushq %rsi
+ 0x02, 0x50, // 1: pushq %rbp
+ 0x01, 0x30, // 0: pushq %rbx
+ 0x00, 0x00,
+ };
+ static const unsigned char uw_data_exitcall[] = {
+ 0x01, // UBYTE: 3 Version , UBYTE: 5 Flags
+ 0x10, // UBYTE Size of prolog
+ 0x0a, // UBYTE Count of unwind codes
+ 0x00, // UBYTE: 4 Frame Register, UBYTE: 4 Frame Register offset (scaled)
+ // USHORT * n Unwind codes array
+ 0x10, 0x01, 0x2f, 0x00, // c: subq 376, %rsp ; 0x48 + 32+16*8+16*8+8+8
+ 0x0c, 0xf0, // a: pushq %r15
+ 0x0a, 0xe0, // 8: pushq %r14
+ 0x08, 0xd0, // 6: pushq %r13
+ 0x06, 0xc0, // 4: pushq %r12
+ 0x04, 0x70, // 3: pushq %rdi
+ 0x03, 0x60, // 2: pushq %rsi
+ 0x02, 0x50, // 1: pushq %rbp
+ 0x01, 0x30, // 0: pushq %rbx
+ };
+
+ zend_jit_uw_func = (PRUNTIME_FUNCTION)*dasm_ptr;
+ *dasm_ptr = (char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(sizeof(RUNTIME_FUNCTION) * 4 +
+ sizeof(uw_data) + sizeof(uw_data_exitcall) + sizeof(uw_data), 16);
+
+ zend_jit_uw_func[0].BeginAddress = 0;
+ zend_jit_uw_func[1].BeginAddress = (uintptr_t)zend_jit_stub_handlers[jit_stub_trace_exit] - (uintptr_t)dasm_buf;
+ zend_jit_uw_func[2].BeginAddress = (uintptr_t)zend_jit_stub_handlers[jit_stub_undefined_offset] - (uintptr_t)dasm_buf;
+
+ zend_jit_uw_func[0].EndAddress = zend_jit_uw_func[1].BeginAddress;
+ zend_jit_uw_func[1].EndAddress = zend_jit_uw_func[2].BeginAddress;
+ zend_jit_uw_func[2].EndAddress = (uintptr_t)dasm_end - (uintptr_t)dasm_buf;
+
+ zend_jit_uw_func[0].UnwindData = (uintptr_t)zend_jit_uw_func - (uintptr_t)dasm_buf + sizeof(RUNTIME_FUNCTION) * 4;
+ zend_jit_uw_func[1].UnwindData = zend_jit_uw_func[0].UnwindData + sizeof(uw_data);
+ zend_jit_uw_func[2].UnwindData = zend_jit_uw_func[1].UnwindData + sizeof(uw_data_exitcall);
+
+ memcpy((char*)dasm_buf + zend_jit_uw_func[0].UnwindData, uw_data, sizeof(uw_data));
+ memcpy((char*)dasm_buf + zend_jit_uw_func[1].UnwindData, uw_data_exitcall, sizeof(uw_data_exitcall));
+ memcpy((char*)dasm_buf + zend_jit_uw_func[2].UnwindData, uw_data, sizeof(uw_data));
+
+#ifdef ZEND_JIT_RT_UNWINDER
+ RtlInstallFunctionTableCallback(
+ (uintptr_t)dasm_buf | 3,
+ (uintptr_t) dasm_buf,
+ (uintptr_t)dasm_end - (uintptr_t)dasm_buf,
+ zend_jit_unwind_callback,
+ NULL,
+ NULL);
+#else
+ RtlAddFunctionTable(zend_jit_uw_func, 3, (uintptr_t)dasm_buf);
+#endif
+}
+#endif
+
+#if defined(__APPLE__) && defined(__x86_64__)
+/* Thunk format used since dydl 1284 (approx. MacOS 15)
+ * https://github.com/apple-oss-distributions/dyld/blob/9307719dd8dc9b385daa412b03cfceb897b2b398/libdyld/ThreadLocalVariables.h#L146 */
+struct TLV_Thunkv2
+{
+ void* func;
+ uint32_t key;
+ uint32_t offset;
+};
+
+/* Thunk format used in earlier versions */
+struct TLV_Thunkv1
+{
+ void* func;
+ size_t key;
+ size_t offset;
+};
+#endif
+
+static void zend_jit_setup(bool reattached)
+{
+#if defined(IR_TARGET_X86)
+ if (!zend_cpu_supports_sse2()) {
+ zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2");
+ }
+#endif
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+ allowed_opt_flags = 0;
+ if (zend_cpu_supports_avx()) {
+ allowed_opt_flags |= ZEND_JIT_CPU_AVX;
+ }
+# if defined(PHP_HAVE_BUILTIN_CPU_SUPPORTS) && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
+ if (zend_cpu_supports_cldemote()) {
+ default_mflags |= IR_X86_CLDEMOTE;
+ }
+# endif
+#endif
+#ifdef ZTS
+#if defined(IR_TARGET_AARCH64)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+
+# ifdef __FreeBSD__
+ if (tsrm_ls_cache_tcb_offset == 0) {
+ TLSDescriptor **where;
+
+ __asm__(
+ "adrp %0, :tlsdesc:_tsrm_ls_cache\n"
+ "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
+ : "=r" (where));
+ /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst
+ * section "Relocations for thread-local storage".
+ * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds
+ * a platform-specific offset or pointer. */
+ TLSDescriptor *tlsdesc = where[1];
+
+ tsrm_tls_offset = tlsdesc->offset;
+ /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */
+ tsrm_tls_index = (tlsdesc->index + 1) * 8;
+ }
+# elif defined(__MUSL__)
+ if (tsrm_ls_cache_tcb_offset == 0) {
+ size_t **where;
+
+ __asm__(
+ "adrp %0, :tlsdesc:_tsrm_ls_cache\n"
+ "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
+ : "=r" (where));
+ /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst */
+ size_t *tlsdesc = where[1];
+
+ tsrm_tls_offset = tlsdesc[1];
+ tsrm_tls_index = tlsdesc[0] * 8;
+ }
+# else
+ ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
+# endif
+# elif defined(_WIN64)
+ tsrm_tls_index = _tls_index * sizeof(void*);
+
+ /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
+ /* Probably, it might be better solution */
+ do {
+ void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
+ void *val = _tsrm_ls_cache;
+ size_t offset = 0;
+ size_t size = (char*)&_tls_end - (char*)&_tls_start;
+
+ while (offset < size) {
+ if (*tls_mem == val) {
+ tsrm_tls_offset = offset;
+ break;
+ }
+ tls_mem++;
+ offset += sizeof(void*);
+ }
+ if (offset >= size) {
+ zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
+ }
+ } while(0);
+# elif defined(ZEND_WIN32)
+ tsrm_tls_index = _tls_index * sizeof(void*);
+
+ /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
+ /* Probably, it might be better solution */
+ do {
+ void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
+ void *val = _tsrm_ls_cache;
+ size_t offset = 0;
+ size_t size = (char*)&_tls_end - (char*)&_tls_start;
+
+ while (offset < size) {
+ if (*tls_mem == val) {
+ tsrm_tls_offset = offset;
+ break;
+ }
+ tls_mem++;
+ offset += sizeof(void*);
+ }
+ if (offset >= size) {
+ zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
+ }
+ } while(0);
+# elif defined(__APPLE__) && defined(__x86_64__)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+ if (tsrm_ls_cache_tcb_offset == 0) {
+ struct TLV_Thunkv2 *thunk;
+ __asm__(
+ "leaq __tsrm_ls_cache(%%rip),%0"
+ : "=r" (thunk));
+
+ /* Detect dyld 1284: With dyld 1284, thunk->func will be _tlv_get_addr.
+ * Unfortunately this symbol is private, but we can find it
+ * as _tlv_bootstrap+8: https://github.com/apple-oss-distributions/dyld/blob/9307719dd8dc9b385daa412b03cfceb897b2b398/libdyld/threadLocalHelpers.s#L54
+ * In earlier versions, thunk->func will be tlv_get_addr, which is not
+ * _tlv_bootstrap+8.
+ */
+ if (thunk->func == (void*)((char*)_tlv_bootstrap + 8)) {
+ tsrm_tls_offset = thunk->offset;
+ tsrm_tls_index = (size_t)thunk->key * 8;
+ } else {
+ struct TLV_Thunkv1 *thunkv1 = (struct TLV_Thunkv1*) thunk;
+ tsrm_tls_offset = thunkv1->offset;
+ tsrm_tls_index = thunkv1->key * 8;
+ }
+ }
+# elif defined(__GNUC__) && defined(__x86_64__)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+ if (tsrm_ls_cache_tcb_offset == 0) {
+#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \
+ !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
+ size_t ret;
+
+ asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
+ : "=r" (ret));
+ tsrm_ls_cache_tcb_offset = ret;
+#elif defined(__MUSL__)
+ size_t *ti;
+
+ __asm__(
+ "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
+ : "=D" (ti));
+ tsrm_tls_offset = ti[1];
+ tsrm_tls_index = ti[0] * 8;
+#elif defined(__FreeBSD__)
+ size_t *ti;
+
+ __asm__(
+ "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
+ : "=D" (ti));
+ tsrm_tls_offset = ti[1];
+ /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */
+ tsrm_tls_index = (ti[0] + 1) * 8;
+#else
+ size_t *ti;
+
+ __asm__(
+ "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
+ : "=D" (ti));
+ tsrm_tls_offset = ti[1];
+ tsrm_tls_index = ti[0] * 16;
+#endif
+ }
+# elif defined(__GNUC__) && defined(__i386__)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+ if (tsrm_ls_cache_tcb_offset == 0) {
+#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
+ size_t ret;
+
+ asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
+ : "=a" (ret));
+ tsrm_ls_cache_tcb_offset = ret;
+#else
+ size_t *ti, _ebx, _ecx, _edx;
+
+ __asm__(
+ "call 1f\n"
+ ".subsection 1\n"
+ "1:\tmovl (%%esp), %%ebx\n\t"
+ "ret\n"
+ ".previous\n\t"
+ "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
+ "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
+ "call ___tls_get_addr@plt\n\t"
+ "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
+ : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
+ tsrm_tls_offset = ti[1];
+ tsrm_tls_index = ti[0] * 8;
+#endif
+ }
+# endif
+#endif
+
+#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj
+ }
+#endif
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
+ zend_jit_setup_disasm();
+ }
+
+#ifndef _WIN32
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
+ ir_perf_jitdump_open();
+ }
+
+#endif
+ zend_long debug = JIT_G(debug);
+ if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) {
+ JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL|
+ ZEND_JIT_DEBUG_IR_CODEGEN|
+ ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM|
+ ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS);
+ }
+
+ zend_jit_calc_trace_prologue_size();
+ if (!reattached) {
+ zend_jit_setup_stubs();
+ }
+ JIT_G(debug) = debug;
+
+#ifdef _WIN64
+ zend_jit_setup_unwinder();
+#endif
+}
+
+static void zend_jit_shutdown_ir(void)
+{
+#ifndef _WIN32
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
+ ir_perf_jitdump_close();
+ }
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
+ ir_gdb_unregister_all();
+ }
+#endif
+#ifdef HAVE_CAPSTONE
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
+ ir_disasm_free();
+ }
+#endif
+}
+
+/* PHP control flow reconstruction helpers */
+static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block)
+{
+ ir_ref ref = ir_IF(condition);
+ /* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow.
+ *
+ * It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions
+ * to start target block
+ */
+ ir_set_op(&jit->ctx, ref, 3, true_block);
+ return ref;
+}
+
+static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block)
+{
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ ZEND_ASSERT(if_ref);
+ ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF);
+ ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3);
+ if (jit->ctx.ir_base[if_ref].op3 == true_block) {
+ ir_IF_TRUE(if_ref);
+ } else {
+ ir_IF_FALSE(if_ref);
+ }
+}
+
+static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref);
+
+static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref)
+{
+ int i, *p;
+ zend_basic_block *bb;
+ ir_ref *r, header;
+
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ bb = &jit->ssa->cfg.blocks[b];
+ p = &jit->ssa->cfg.predecessors[bb->predecessor_offset];
+ r = &jit->bb_edges[jit->bb_predecessors[b]];
+ for (i = 0; i < bb->predecessors_count; i++, p++, r++) {
+ if (*p == pred) {
+ ZEND_ASSERT(*r == IR_UNUSED || *r == ref);
+ header = jit->bb_start_ref[b];
+ if (header) {
+ /* this is back edge */
+ ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN);
+ if (jit->ctx.ir_base[ref].op == IR_END) {
+ jit->ctx.ir_base[ref].op = IR_LOOP_END;
+ } else if (jit->ctx.ir_base[ref].op == IR_IF) {
+ jit_IF_TRUE_FALSE_ex(jit, ref, b);
+ ref = ir_LOOP_END();
+ } else if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
+ zend_jit_case_start(jit, pred, b, ref);
+ ref = ir_LOOP_END();
+ } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) {
+ ir_BEGIN(ref);
+ ref = ir_LOOP_END();
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_MERGE_SET_OP(header, i + 1, ref);
+ }
+ *r = ref;
+ return;
+ }
+ }
+ ZEND_UNREACHABLE();
+}
+
+static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit,
+ uint32_t true_label,
+ uint32_t false_label,
+ ir_ref true_inputs,
+ ir_ref false_inputs)
+{
+ ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED;
+
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ if (true_inputs) {
+ ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END);
+ if (!jit->ctx.ir_base[true_inputs].op2) {
+ true_path = true_inputs;
+ } else {
+ ir_MERGE_list(true_inputs);
+ true_path = ir_END();
+ }
+ }
+ if (false_inputs) {
+ ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END);
+ if (!jit->ctx.ir_base[false_inputs].op2) {
+ false_path = false_inputs;
+ } else {
+ ir_MERGE_list(false_inputs);
+ false_path = ir_END();
+ }
+ }
+
+ if (true_label == false_label && true_path && false_path) {
+ ir_MERGE_2(true_path, false_path);
+ _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
+ } else if (!true_path && !false_path) {
+ /* dead code */
+ true_path = ir_END();
+ _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
+ } else {
+ if (true_path) {
+ _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
+ }
+ if (false_path) {
+ _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path);
+ }
+ }
+
+ jit->b = -1;
+}
+
+static void _zend_jit_fix_merges(zend_jit_ctx *jit)
+{
+ int i, count;
+ ir_ref j, k, n, *p, *q, *r;
+ ir_ref ref;
+ ir_insn *insn, *phi;
+
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ count = jit->ssa->cfg.blocks_count;
+ for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) {
+ ref = *p;
+ if (ref) {
+ insn = &jit->ctx.ir_base[ref];
+ if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
+ n = insn->inputs_count;
+ /* Remove IS_UNUSED inputs */
+ for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) {
+ if (*q) {
+ if (q != r) {
+ *r = *q;
+ phi = insn + 1 + (n >> 2);
+ while (phi->op == IR_PI) {
+ phi++;
+ }
+ while (phi->op == IR_PHI) {
+ ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2));
+ phi += 1 + ((n + 1) >> 2);
+ }
+ }
+ k++;
+ r++;
+ }
+ }
+ if (k != n) {
+ ir_ref n2, k2;
+
+ if (k <= 1) {
+ insn->op = IR_BEGIN;
+ insn->inputs_count = 0;
+ } else {
+ insn->inputs_count = k;
+ }
+ n2 = 1 + (n >> 2);
+ k2 = 1 + (k >> 2);
+ while (k2 != n2) {
+ (insn+k2)->optx = IR_NOP;
+ k2++;
+ }
+ phi = insn + 1 + (n >> 2);
+ while (phi->op == IR_PI) {
+ phi++;
+ }
+ while (phi->op == IR_PHI) {
+ if (k <= 1) {
+ phi->op = IR_COPY;
+ phi->op1 = phi->op2;
+ phi->op2 = 1;
+ phi->inputs_count = 0;
+ } else {
+ phi->inputs_count = k + 1;
+ }
+ n2 = 1 + ((n + 1) >> 2);
+ k2 = 1 + ((k + 1) >> 2);
+ while (k2 != n2) {
+ (phi+k2)->optx = IR_NOP;
+ k2++;
+ }
+ phi += 1 + ((n + 1) >> 2);
+ }
+ }
+ }
+ }
+ }
+}
+
+static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref)
+{
+ zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b];
+ const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
+
+ if (opline->opcode == ZEND_SWITCH_LONG
+ || opline->opcode == ZEND_SWITCH_STRING
+ || opline->opcode == ZEND_MATCH) {
+ HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
+ const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
+ int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes];
+ zval *zv;
+ ir_ref list = IR_UNUSED, idx;
+ bool first = 1;
+
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+ int b = jit->ssa->cfg.map[target - jit->op_array->opcodes];
+
+ if (b == case_b) {
+ if (!first) {
+ ir_END_list(list);
+ }
+ if (HT_IS_PACKED(jumptable)) {
+ idx = ir_CONST_LONG(zv - jumptable->arPacked);
+ } else {
+ idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
+ }
+ ir_CASE_VAL(switch_ref, idx);
+ first = 0;
+ }
+ } ZEND_HASH_FOREACH_END();
+ if (default_b == case_b) {
+ if (!first) {
+ ir_END_list(list);
+ }
+ if (jit->ctx.ir_base[switch_ref].op3) {
+ /* op3 may contain a list of additional "default" path inputs for MATCH */
+ ir_ref ref = jit->ctx.ir_base[switch_ref].op3;
+ jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF;
+ ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
+ ir_ref end = ref;
+ while (jit->ctx.ir_base[end].op2) {
+ ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END);
+ end = jit->ctx.ir_base[end].op2;
+ }
+ jit->ctx.ir_base[end].op2 = list;
+ list = ref;
+ }
+ ir_CASE_DEFAULT(switch_ref);
+ }
+ if (list) {
+ ir_END_list(list);
+ ir_MERGE_list(list);
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+}
+
+static int zend_jit_bb_start(zend_jit_ctx *jit, int b)
+{
+ zend_basic_block *bb;
+ int i, n, *p, pred;
+ ir_ref ref, bb_start;
+
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ ZEND_ASSERT(b < jit->ssa->cfg.blocks_count);
+ bb = &jit->ssa->cfg.blocks[b];
+ ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0);
+ n = bb->predecessors_count;
+
+ if (n == 0) {
+ /* pass */
+ ZEND_ASSERT(jit->ctx.control);
+#if ZEND_DEBUG
+ ref = jit->ctx.control;
+ ir_insn *insn = &jit->ctx.ir_base[ref];
+ while (insn->op >= IR_CALL && insn->op <= IR_TRAP) {
+ ref = insn->op1;
+ insn = &jit->ctx.ir_base[ref];
+ }
+ ZEND_ASSERT(insn->op == IR_START);
+ ZEND_ASSERT(ref == 1);
+#endif
+ bb_start = 1;
+ if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) {
+ /* prvent END/BEGIN merging */
+ jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END());
+ bb_start = jit->ctx.control;
+ }
+ } else if (n == 1) {
+ ZEND_ASSERT(!jit->ctx.control);
+ pred = jit->ssa->cfg.predecessors[bb->predecessor_offset];
+ ref = jit->bb_edges[jit->bb_predecessors[b]];
+ if (ref == IR_UNUSED) {
+ if (!jit->ctx.control) {
+ ir_BEGIN(IR_UNUSED); /* unreachable block */
+ }
+ } else {
+ ir_op op = jit->ctx.ir_base[ref].op;
+
+ if (op == IR_IF) {
+ if (!jit->ctx.control) {
+ jit_IF_TRUE_FALSE_ex(jit, ref, b);
+ } else {
+ ir_ref entry_path = ir_END();
+ jit_IF_TRUE_FALSE_ex(jit, ref, b);
+ ir_MERGE_WITH(entry_path);
+ }
+ } else if (op == IR_SWITCH) {
+ zend_jit_case_start(jit, pred, b, ref);
+ } else {
+ if (!jit->ctx.control) {
+ ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
+ if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY)
+ && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) {
+ /* prvent END/BEGIN merging */
+ jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref);
+ } else {
+ ir_BEGIN(ref);
+ }
+ } else {
+ ir_MERGE_WITH(ref);
+ }
+ }
+ }
+ bb_start = jit->ctx.control;
+ } else {
+ int forward_edges_count = 0;
+ int back_edges_count = 0;
+ ir_ref *pred_refs;
+ ir_ref entry_path = IR_UNUSED;
+ ALLOCA_FLAG(use_heap);
+
+ ZEND_ASSERT(!jit->ctx.control);
+ if (jit->ctx.control) {
+ entry_path = ir_END();
+ }
+ pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap);
+ for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) {
+ pred = *p;
+ if (jit->bb_start_ref[pred]) {
+ /* forward edge */
+ forward_edges_count++;
+ ref = jit->bb_edges[jit->bb_predecessors[b] + i];
+ if (ref == IR_UNUSED) {
+ /* dead edge */
+ pred_refs[i] = IR_UNUSED;
+ } else {
+ ir_op op = jit->ctx.ir_base[ref].op;
+
+ if (op == IR_IF) {
+ jit_IF_TRUE_FALSE_ex(jit, ref, b);
+ pred_refs[i] = ir_END();
+ } else if (op == IR_SWITCH) {
+ zend_jit_case_start(jit, pred, b, ref);
+ pred_refs[i] = ir_END();
+ } else {
+ ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
+ pred_refs[i] = ref;
+ }
+ }
+ } else {
+ /* backward edge */
+ back_edges_count++;
+ pred_refs[i] = IR_UNUSED;
+ }
+ }
+
+ if (bb->flags & ZEND_BB_LOOP_HEADER) {
+ ZEND_ASSERT(back_edges_count != 0);
+ ZEND_ASSERT(forward_edges_count != 0);
+ ir_MERGE_N(n, pred_refs);
+ jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN;
+ bb_start = jit->ctx.control;
+ if (entry_path) {
+ ir_MERGE_WITH(entry_path);
+ }
+ } else {
+// ZEND_ASSERT(back_edges_count != 0);
+ /* edges from exceptional blocks may be counted as back edges */
+ ir_MERGE_N(n, pred_refs);
+ bb_start = jit->ctx.control;
+ if (entry_path) {
+ ir_MERGE_WITH(entry_path);
+ }
+ }
+ free_alloca(pred_refs, use_heap);
+ }
+ jit->b = b;
+ jit->bb_start_ref[b] = bb_start;
+
+ if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) {
+ jit->ctx.fold_cse_limit = bb_start;
+ }
+
+ return 1;
+}
+
+static int zend_jit_bb_end(zend_jit_ctx *jit, int b)
+{
+ int succ;
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ if (jit->b != b) {
+ return 1;
+ }
+
+ bb = &jit->ssa->cfg.blocks[b];
+ ZEND_ASSERT(bb->successors_count != 0);
+ if (bb->successors_count == 1) {
+ succ = bb->successors[0];
+ } else {
+ const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
+
+ /* Use only the following successor of SWITCH and FE_RESET_R */
+ ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG
+ || opline->opcode == ZEND_SWITCH_STRING
+ || opline->opcode == ZEND_MATCH
+ || opline->opcode == ZEND_FE_RESET_R);
+ succ = b + 1;
+ }
+ _zend_jit_add_predecessor_ref(jit, succ, b, ir_END());
+ jit->b = -1;
+ return 1;
+}
+
+static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline)
+{
+ ir_ref ref;
+
+#if 1
+ if (GCC_GLOBAL_REGS) {
+ ref = jit_IP32(jit);
+ } else {
+ ref = ir_LOAD_U32(jit_EX(opline));
+ }
+ ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline));
+#else
+ if (GCC_GLOBAL_REGS) {
+ ref = jit_IP(jit);
+ } else {
+ ref = ir_LOAD_A(jit_EX(opline));
+ }
+ ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline));
+#endif
+ return ref;
+}
+
+static int zend_jit_jmp_frameless(
+ zend_jit_ctx *jit,
+ const zend_op *opline,
+ const void *exit_addr,
+ zend_jmp_fl_result guard
+) {
+ ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref;
+ zend_basic_block *bb;
+
+ // JIT: CACHED_PTR(opline->extended_value)
+ cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
+ cache_result = ir_LOAD_L(cache_slot_ref);
+
+ // JIT: if (UNEXPECTED(!result))
+ if_ref = ir_IF(cache_result);
+ ir_IF_FALSE_cold(if_ref);
+ zval *func_name_zv = RT_CONSTANT(opline, opline->op1);
+ function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper),
+ ir_CONST_ADDR(func_name_zv),
+ cache_slot_ref);
+ ir_MERGE_WITH_EMPTY_TRUE(if_ref);
+
+ phi_result = ir_PHI_2(IR_LONG, function_result, cache_result);
+
+ if (exit_addr) {
+ ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr));
+ } else {
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ // JIT: if (result == ZEND_JMP_FL_HIT)
+ ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
+ jit->b = -1;
+ }
+
+ return 1;
+}
+
+static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block)
+{
+ ir_ref ref;
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+
+ ZEND_ASSERT(bb->successors_count == 2);
+ if (bb->successors[0] == bb->successors[1]) {
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
+ jit->b = -1;
+ zend_jit_set_last_valid_opline(jit, next_opline);
+ return 1;
+ }
+
+ ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block);
+
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
+
+ jit->b = -1;
+ zend_jit_set_last_valid_opline(jit, next_opline);
+
+ return 1;
+}
+
+static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *opline, const zend_op *next_opline, uint32_t var)
+{
+ ir_ref ref;
+
+ ir_op op = (opline->result_type & IS_SMART_BRANCH_JMPZ) ? IR_EQ : IR_NE;
+ ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, op, next_opline)), ir_CONST_U32(IS_FALSE));
+
+ // EX_VAR(var) = ...
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref);
+
+ zend_jit_reset_last_valid_opline(jit);
+ return zend_jit_set_ip(jit, next_opline - 1);
+}
+
+/* PHP JIT handlers */
+static void zend_jit_check_exception(zend_jit_ctx *jit)
+{
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit, jit_stub_exception_handler));
+}
+
+static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline)
+{
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit,
+ (opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler));
+}
+
+static void zend_jit_type_check_undef(zend_jit_ctx *jit,
+ ir_ref type,
+ uint32_t var,
+ const zend_op *opline,
+ bool check_exception,
+ bool in_cold_path,
+ bool undef_result)
+{
+ ir_ref if_def = ir_IF(type);
+
+ if (!in_cold_path) {
+ ir_IF_FALSE_cold(if_def);
+ } else {
+ ir_IF_FALSE(if_def);
+ }
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
+ if (check_exception) {
+ if (undef_result) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ } else {
+ zend_jit_check_exception(jit);
+ }
+ }
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+}
+
+static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit,
+ ir_ref ref,
+ uint32_t var,
+ const zend_op *opline,
+ bool check_exception)
+{
+ ir_ref if_def, ref2;
+
+ if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
+ ir_IF_FALSE_cold(if_def);
+
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
+
+ if (check_exception) {
+ zend_jit_check_exception(jit);
+ }
+
+ ref2 = jit_EG(uninitialized_zval);
+
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+
+ return ir_PHI_2(IR_ADDR, ref2, ref);
+}
+
+static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
+{
+ zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
+ int pred;
+ ir_ref ref;
+
+ ZEND_ASSERT(bb->predecessors_count > 0);
+
+ pred = jit->bb_predecessors[b];
+ ref = jit->bb_edges[pred];
+
+ ZEND_ASSERT(ref);
+ ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
+
+ /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
+ ir_ENTRY(ref, bb->start);
+ if (!GCC_GLOBAL_REGS) {
+ /* 2 is hardcoded reference to IR_PARAM */
+ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
+ ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
+ jit_STORE_FP(jit, 2);
+ }
+
+ ir_MERGE_WITH(ref);
+ jit->bb_edges[pred] = ir_END();
+}
+
+static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
+{
+ zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
+ ir_ref ref = ir_END();
+
+ /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
+ ir_ENTRY(ref, bb->start);
+ if (!GCC_GLOBAL_REGS) {
+ /* 2 is hardcoded reference to IR_PARAM */
+ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
+ ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
+ jit_STORE_FP(jit, 2);
+ }
+
+ ir_MERGE_WITH(ref);
+}
+
+static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
+{
+ ir_ENTRY(src, label);
+ if (!GCC_GLOBAL_REGS) {
+ /* 2 is hardcoded reference to IR_PARAM */
+ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
+ ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
+ jit_STORE_FP(jit, 2);
+ }
+ return ir_END();
+}
+
+static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw)
+{
+ ir_ref ref;
+ const void *handler;
+
+ zend_jit_set_ip(jit, opline);
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ handler = zend_get_opcode_handler_func(opline);
+ } else {
+ handler = opline->handler;
+ }
+ if (GCC_GLOBAL_REGS) {
+ ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
+ } else {
+ ref = jit_FP(jit);
+ ref = ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(handler), ref);
+ }
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ /* Skip the following OP_DATA */
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ case ZEND_ASSIGN_STATIC_PROP:
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_OBJ_OP:
+ case ZEND_ASSIGN_STATIC_PROP_OP:
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ case ZEND_ASSIGN_OBJ_REF:
+ case ZEND_FRAMELESS_ICALL_3:
+ zend_jit_set_last_valid_opline(jit, opline + 2);
+ break;
+ default:
+ zend_jit_set_last_valid_opline(jit, opline + 1);
+ break;
+ }
+ return 1;
+}
+
+static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
+{
+ const void *handler;
+ ir_ref ref;
+ zend_basic_block *bb;
+
+ zend_jit_set_ip(jit, opline);
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ if (opline->opcode == ZEND_DO_UCALL ||
+ opline->opcode == ZEND_DO_FCALL_BY_NAME ||
+ opline->opcode == ZEND_DO_FCALL ||
+ opline->opcode == ZEND_RETURN) {
+
+ /* Use inlined HYBRID VM handler */
+ handler = opline->handler;
+ ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
+ } else {
+ handler = zend_get_opcode_handler_func(opline);
+ ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
+ ref = ir_LOAD_A(jit_IP(jit));
+ ir_TAILCALL(IR_VOID, ref);
+ }
+ } else {
+ handler = opline->handler;
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
+ } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
+ && (opline->opcode == ZEND_CATCH
+ || opline->opcode == ZEND_FAST_CALL
+ || opline->opcode == ZEND_FAST_RET
+ || opline->opcode == ZEND_MATCH_ERROR
+ || opline->opcode == ZEND_THROW
+ || opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
+ ref = jit_FP(jit);
+ ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
+ ir_RETURN(ir_CONST_I32(1));
+ } else {
+ ref = jit_FP(jit);
+ ir_TAILCALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
+ }
+ }
+ if (jit->b >= 0) {
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ if (bb->successors_count > 0
+ && (opline->opcode == ZEND_DO_FCALL
+ || opline->opcode == ZEND_DO_UCALL
+ || opline->opcode == ZEND_DO_FCALL_BY_NAME
+ || opline->opcode == ZEND_INCLUDE_OR_EVAL
+ || opline->opcode == ZEND_GENERATOR_CREATE
+ || opline->opcode == ZEND_YIELD
+ || opline->opcode == ZEND_YIELD_FROM
+ || opline->opcode == ZEND_FAST_CALL)) {
+ /* Add a fake control edge from UNREACHABLE to the following ENTRY */
+ int succ;
+
+ if (bb->successors_count == 1) {
+ succ = bb->successors[0];
+ ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
+ } else {
+ /* Use only the following successor of FAST_CALL */
+ ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL);
+ succ = jit->b + 1;
+ /* we need an entry */
+ jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY;
+ }
+ ref = jit->ctx.insns_count - 1;
+ ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN);
+ ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
+ _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
+ }
+ jit->b = -1;
+ zend_jit_reset_last_valid_opline(jit);
+ }
+ return 1;
+}
+
+static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block)
+{
+ return zend_jit_tail_handler(jit, opline);
+}
+
+static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
+{
+ ZEND_ASSERT(Z_MODE(src) == IS_REG);
+ ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
+{
+ ZEND_ASSERT(Z_MODE(src) == IS_REG);
+ ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
+ if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) {
+ /* invalidate memory type */
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
+ if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) {
+ /* invalidate memory type */
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
+{
+ ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
+ ZEND_ASSERT(Z_MODE(dst) == IS_REG);
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src));
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src));
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type)
+{
+ zend_jit_addr src = ZEND_ADDR_REG(ssa_var);
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ return zend_jit_spill_store(jit, src, dst, info, set_type);
+}
+
+static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type)
+{
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ jit_set_Z_DVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
+{
+ ir_ref ref = jit->ctx.control;
+ ir_insn *insn;
+
+ while (1) {
+ insn = &jit->ctx.ir_base[ref];
+ if (insn->op == IR_RLOAD && insn->op2 == reg) {
+ ZEND_ASSERT(insn->type == type);
+ return ref;
+ } else if (insn->op == IR_START) {
+ break;
+ }
+ ref = insn->op1;
+ }
+ return ir_RLOAD(type, reg);
+}
+
+/* Same as zend_jit_deopt_rload(), but 'reg' may be spilled on C stack */
+static ir_ref zend_jit_deopt_rload_spilled(zend_jit_ctx *jit, ir_type type, int8_t reg, int32_t offset)
+{
+ ZEND_ASSERT(reg >= 0);
+
+ if (IR_REG_SPILLED(reg)) {
+ return ir_LOAD(type, ir_ADD_OFFSET(zend_jit_deopt_rload(jit, type, IR_REG_NUM(reg)), offset));
+ } else {
+ return zend_jit_deopt_rload(jit, type, reg);
+ }
+}
+
+static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
+{
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+ ir_ref src = ir_CONST_LONG(val);
+
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ }
+ jit_set_Z_LVAL(jit, dst, src);
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ return 1;
+}
+
+static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val)
+{
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+ ir_ref src = ir_CONST_DOUBLE(val);
+
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ }
+ jit_set_Z_DVAL(jit, dst, src);
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ return 1;
+}
+
+static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type)
+{
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ ZEND_ASSERT(type <= IS_DOUBLE);
+ jit_set_Z_TYPE_INFO(jit, dst, type);
+ return 1;
+}
+
+static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type)
+{
+ zend_jit_addr src;
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+ ir_type type;
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ type = IR_LONG;
+ src = zend_jit_deopt_rload(jit, type, reg);
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ } else if (!in_mem) {
+ jit_set_Z_LVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ }
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ type = IR_DOUBLE;
+ src = zend_jit_deopt_rload(jit, type, reg);
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ } else if (!in_mem) {
+ jit_set_Z_DVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
+{
+ zend_jit_addr src;
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ } else {
+ jit_set_Z_LVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
+ }
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
+ if (jit->ra && jit->ra[var].ref == IR_NULL) {
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
+ } else {
+ jit_set_Z_DVAL(jit, dst, src);
+ if (set_type &&
+ (Z_REG(dst) != ZREG_FP ||
+ !JIT_G(current_frame) ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
+ jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
+ }
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
+{
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ jit_set_Z_TYPE_INFO(jit, dst, type);
+ return 1;
+}
+
+static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr)
+{
+ ir_ref if_refcounted, end1;
+
+ if_refcounted = jit_if_REFCOUNTED(jit, var_addr);
+ ir_IF_FALSE(if_refcounted);
+ end1 = ir_END();
+ ir_IF_TRUE(if_refcounted);
+ jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr));
+ ir_MERGE_WITH(end1);
+ return 1;
+}
+
+static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info)
+{
+ if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ return zend_jit_spill_store(jit, src, dst, info, 1);
+ }
+ return 1;
+}
+
+static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
+{
+ if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ bool set_type = 1;
+
+ if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
+ (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
+ if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
+ if (JIT_G(current_frame)) {
+ uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var));
+
+ if (mem_type != IS_UNKNOWN
+ && (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) {
+ set_type = 0;
+ }
+ } else {
+ set_type = 0;
+ }
+ }
+ }
+ return zend_jit_spill_store(jit, src, dst, info, set_type);
+ }
+ return 1;
+}
+
+static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var)
+{
+ zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+ zend_jit_addr dst = ZEND_ADDR_REG(ssa_var);
+
+ return zend_jit_load_reg(jit, src, dst, info);
+}
+
+static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op)
+{
+ if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
+ /* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
+ jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF);
+ }
+ return 1;
+}
+
+static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
+{
+ if (!zend_jit_same_addr(src, dst)) {
+ if (Z_MODE(src) == IS_REG) {
+ if (Z_MODE(dst) == IS_REG) {
+ zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src));
+ if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+
+ if (!zend_jit_spill_store(jit, dst, var_addr, info,
+ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ JIT_G(current_frame) == NULL ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
+ (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
+ )) {
+ return 0;
+ }
+ }
+ } else if (Z_MODE(dst) == IS_MEM_ZVAL) {
+ if (!Z_LOAD(src) && !Z_STORE(src)) {
+ if (!zend_jit_spill_store(jit, src, dst, info,
+ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ JIT_G(current_frame) == NULL ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
+ (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
+ )) {
+ return 0;
+ }
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else if (Z_MODE(src) == IS_MEM_ZVAL) {
+ if (Z_MODE(dst) == IS_REG) {
+ if (!zend_jit_load_reg(jit, src, dst, info)) {
+ return 0;
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
+ dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ if (!zend_jit_spill_store(jit, src, dst, info,
+ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ JIT_G(current_frame) == NULL ||
+ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
+ (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
+ )) {
+ return 0;
+ }
+ }
+ return 1;
+}
+
+struct jit_observer_fcall_is_unobserved_data {
+ ir_ref if_unobserved;
+ ir_ref ir_end_inputs;
+};
+
+static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_function *func, ir_ref *observer_handler, ir_ref rx, ir_ref func_ref) {
+ ir_ref run_time_cache;
+ struct jit_observer_fcall_is_unobserved_data data = { .ir_end_inputs = IR_UNUSED };
+ if (func) {
+ ZEND_ASSERT((func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
+ } else {
+ // JIT: if (function->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) {
+ ZEND_ASSERT(rx != IR_UNUSED);
+ ir_ref if_trampoline_or_generator = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
+ ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)));
+ ir_IF_TRUE(if_trampoline_or_generator);
+ ir_END_list(data.ir_end_inputs);
+ ir_IF_FALSE(if_trampoline_or_generator);
+ }
+ if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) {
+ // JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache)
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache)));
+#ifndef ZTS
+ } else if (func && rx == IS_UNUSED) { // happens for internal functions only
+ ZEND_ASSERT(!ZEND_USER_CODE(func->type));
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_CONST_ADDR(func), offsetof(zend_op_array, run_time_cache__ptr)));
+#endif
+ } else {
+ // Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these
+ if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function
+ run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache));
+ } else {
+ // JIT: ZEND_MAP_PTR_GET(func->common.runtime_cache)
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_op_array, run_time_cache__ptr)));
+ ir_ref if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
+ ir_IF_TRUE(if_odd);
+
+ ir_ref run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
+
+ ir_ref if_odd_end = ir_END();
+ ir_IF_FALSE(if_odd);
+
+ // JIT: if (func->common.runtime_cache != NULL) {
+ ir_ref if_rt_cache = ir_IF(ir_EQ(run_time_cache, IR_NULL));
+ ir_IF_TRUE(if_rt_cache);
+ ir_END_list(data.ir_end_inputs);
+ ir_IF_FALSE(if_rt_cache);
+
+ ir_MERGE_WITH(if_odd_end);
+ run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache, run_time_cache2);
+ }
+ }
+ // JIT: observer_handler = runtime_cache + ZEND_OBSERVER_HANDLE(function)
+ if (func) {
+ *observer_handler = ir_ADD_OFFSET(run_time_cache, ZEND_OBSERVER_HANDLE(func) * sizeof(void *));
+ } else {
+ // JIT: (func->type == ZEND_INTERNAL_FUNCTION ? zend_observer_fcall_internal_function_extension : zend_observer_fcall_op_array_extension) * sizeof(void *)
+ ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
+ ir_ref if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(ZEND_INTERNAL_FUNCTION)));
+ ir_IF_TRUE(if_internal_func);
+
+ ir_ref observer_handler_internal = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_internal_function_extension * sizeof(void *));
+
+ ir_ref if_internal_func_end = ir_END();
+ ir_IF_FALSE(if_internal_func);
+
+ ir_ref observer_handler_user = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_op_array_extension * sizeof(void *));
+
+ ir_MERGE_WITH(if_internal_func_end);
+ *observer_handler = ir_PHI_2(IR_ADDR, observer_handler_user, observer_handler_internal);
+ }
+
+ // JIT: if (*observer_handler == ZEND_OBSERVER_NONE_OBSERVED) {
+ data.if_unobserved = ir_IF(ir_EQ(ir_LOAD_A(*observer_handler), ir_CONST_ADDR(ZEND_OBSERVER_NONE_OBSERVED)));
+ ir_IF_FALSE(data.if_unobserved);
+ return data;
+}
+
+/* For frameless the true branch of if_unobserved is used and this function not called. */
+static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_unobserved_data *data) {
+ ir_END_list(data->ir_end_inputs);
+ ir_IF_TRUE(data->if_unobserved);
+ ir_END_list(data->ir_end_inputs);
+ ir_MERGE_list(data->ir_end_inputs);
+}
+
+static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin_prechecked), rx, observer_handler);
+}
+
+static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) {
+ // JIT: if (execute_data == EG(current_observed_frame)) {
+ ir_ref has_end_observer = ir_IF(ir_EQ(rx, ir_LOAD_A(jit_EG(current_observed_frame))));
+ ir_IF_TRUE(has_end_observer);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end_prechecked),
+ rx, res_ref);
+ ir_MERGE_WITH_EMPTY_FALSE(has_end_observer);
+}
+
+static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
+{
+ ir_ref if_long = IR_UNUSED;
+ ir_ref op1_lval_ref = IR_UNUSED;
+ ir_ref ref;
+ ir_op op;
+
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
+ if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_long);
+ }
+ if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
+ op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
+ jit_set_Z_LVAL(jit, res_addr, op1_lval_ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL
+ && Z_MODE(op1_addr) == IS_REG
+ && !Z_LOAD(op1_addr)
+ && !Z_STORE(op1_addr)) {
+ jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG);
+ }
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ op = may_overflow ? IR_ADD_OV : IR_ADD;
+ } else {
+ op = may_overflow ? IR_SUB_OV : IR_SUB;
+ }
+ if (!op1_lval_ref) {
+ op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
+ }
+ ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1));
+ if (op1_def_info & MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, op1_def_addr, ref);
+ }
+ if (may_overflow &&
+ (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) ||
+ ((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) {
+ int32_t exit_point;
+ const void *exit_addr;
+ zend_jit_trace_stack *stack;
+ uint32_t old_op1_info, old_res_info = 0;
+
+ stack = JIT_G(current_frame)->stack;
+ old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ } else {
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ }
+ if (opline->result_type != IS_UNUSED) {
+ old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ if (opline->opcode == ZEND_PRE_INC) {
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ } else if (opline->opcode == ZEND_PRE_DEC) {
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ } else if (opline->opcode == ZEND_POST_INC) {
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX));
+ } else if (opline->opcode == ZEND_POST_DEC) {
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN));
+ }
+ }
+
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
+
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
+ if (opline->result_type != IS_UNUSED) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ }
+ } else if (may_overflow) {
+ ir_ref if_overflow;
+ ir_ref merge_inputs = IR_UNUSED;
+
+ if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
+ || (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
+ int32_t exit_point;
+ const void *exit_addr;
+ zend_jit_trace_stack *stack;
+ uint32_t old_res_info = 0, old_op1_info = 0;
+
+ stack = JIT_G(current_frame)->stack;
+ if (opline->result_type != IS_UNUSED) {
+ old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) {
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ref);
+ } else {
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), op1_lval_ref);
+ }
+ }
+ old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_LONG, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ref);
+
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
+
+ if (opline->result_type != IS_UNUSED) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ }
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
+ } else {
+ if_overflow = ir_IF(ir_OVERFLOW(ref));
+ ir_IF_FALSE(if_overflow);
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ ir_END_list(merge_inputs);
+
+ /* overflow => cold path */
+ ir_IF_TRUE_cold(if_overflow);
+ }
+
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ if (Z_MODE(op1_def_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0));
+ jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000));
+#else
+ jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
+ }
+ } else {
+ if (Z_MODE(op1_def_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000));
+ jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000));
+#else
+ jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
+ }
+ }
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ if (opline->opcode == ZEND_PRE_INC) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ } else {
+ if (Z_MODE(res_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ }
+
+ if (merge_inputs) {
+ ir_END_list(merge_inputs);
+ ir_MERGE_list(merge_inputs);
+ }
+ } else {
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ }
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ ir_ref merge_inputs = ir_END();
+
+ /* !is_long => cold path */
+ ir_IF_FALSE_cold(if_long);
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ jit_SET_EX_OPLINE(jit, opline);
+ if (op1_info & MAY_BE_UNDEF) {
+ ir_ref if_def;
+
+ if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+
+ // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
+
+ jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL);
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+
+ op1_info |= MAY_BE_NULL;
+ }
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref if_ref, if_typed, func, ref2, arg2;
+
+ if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
+ ir_IF_TRUE(if_ref);
+ ref2 = jit_Z_PTR_ref(jit, ref);
+
+ if_typed = jit_if_TYPED_REF(jit, ref2);
+ ir_IF_TRUE(if_typed);
+
+ if (RETURN_VALUE_USED(opline)) {
+ ZEND_ASSERT(Z_MODE(res_addr) != IS_REG);
+ arg2 = jit_ZVAL_ADDR(jit, res_addr);
+ } else {
+ arg2 = IR_NULL;
+ }
+ if (opline->opcode == ZEND_PRE_INC) {
+ func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref);
+ } else if (opline->opcode == ZEND_PRE_DEC) {
+ func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref);
+ } else if (opline->opcode == ZEND_POST_INC) {
+ func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref);
+ } else if (opline->opcode == ZEND_POST_DEC) {
+ func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ ir_CALL_2(IR_VOID, func, ref2, arg2);
+ zend_jit_check_exception(jit);
+ ir_END_list(merge_inputs);
+
+ ir_IF_FALSE(if_typed);
+ ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
+ ir_MERGE_WITH_EMPTY_FALSE(if_ref);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ }
+
+ if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
+ jit_ZVAL_COPY(jit,
+ res_addr,
+ res_use_info,
+ ZEND_ADDR_REF_ZVAL(ref), op1_info, 1);
+ }
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
+ ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2);
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref);
+ }
+ } else {
+ if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
+ ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2);
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref);
+ }
+ }
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ } else {
+ ref = jit_Z_DVAL(jit, op1_addr);
+ if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
+ jit_set_Z_DVAL(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ op = IR_ADD;
+ } else {
+ op = IR_SUB;
+ }
+ ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0));
+ jit_set_Z_DVAL(jit, op1_def_addr, ref);
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ jit_set_Z_DVAL(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ ir_END_list(merge_inputs);
+ ir_MERGE_list(merge_inputs);
+ }
+ if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
+ return 0;
+ }
+ if (opline->result_type != IS_UNUSED) {
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_math_long_long(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint8_t opcode,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info,
+ int may_overflow)
+{
+ bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ ir_op op;
+ ir_ref op1, op2, ref, if_overflow = IR_UNUSED;
+
+ if (opcode == ZEND_ADD) {
+ op = may_overflow ? IR_ADD_OV : IR_ADD;
+ } else if (opcode == ZEND_SUB) {
+ op = may_overflow ? IR_SUB_OV : IR_SUB;
+ } else if (opcode == ZEND_MUL) {
+ op = may_overflow ? IR_MUL_OV : IR_MUL;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ op1 = jit_Z_LVAL(jit, op1_addr);
+ op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
+ ref = ir_BINARY_OP_L(op, op1, op2);
+
+ if (may_overflow) {
+ if (res_info & MAY_BE_GUARD) {
+ if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ uint32_t old_res_info;
+ int32_t exit_point;
+ const void *exit_addr;
+
+ if (opline->opcode == ZEND_ADD
+ && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
+ old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ } else if (opline->opcode == ZEND_SUB
+ && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
+ old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ } else {
+ exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ }
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
+ may_overflow = 0;
+ } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ if_overflow = ir_IF(ir_OVERFLOW(ref));
+ ir_IF_FALSE(if_overflow);
+ }
+ }
+
+ if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) != IS_REG) {
+ if (!zend_jit_same_addr(op1_addr, res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ }
+ }
+
+ if (may_overflow) {
+ ir_ref fast_path = IR_UNUSED;
+
+ if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
+ fast_path = ir_END();
+ ir_IF_TRUE_cold(if_overflow);
+ }
+ if (opcode == ZEND_ADD) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
+ ir_MERGE_WITH(fast_path);
+ }
+ return 1;
+ }
+ op = IR_ADD;
+ } else if (opcode == ZEND_SUB) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
+ ir_MERGE_WITH(fast_path);
+ }
+ return 1;
+ }
+ op = IR_SUB;
+ } else if (opcode == ZEND_MUL) {
+ op = IR_MUL;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+#if 1
+ /* reload */
+ op1 = jit_Z_LVAL(jit, op1_addr);
+ op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
+#endif
+#if 1
+ /* disable CSE */
+ ir_ref old_cse_limit = jit->ctx.fold_cse_limit;
+ jit->ctx.fold_cse_limit = 0x7fffffff;
+#endif
+ op1 = ir_INT2D(op1);
+ op2 = ir_INT2D(op2);
+#if 1
+ jit->ctx.fold_cse_limit = old_cse_limit;
+#endif
+ ref = ir_BINARY_OP_D(op, op1, op2);
+ jit_set_Z_DVAL(jit, res_addr, ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
+ ir_MERGE_WITH(fast_path);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math_long_double(zend_jit_ctx *jit,
+ uint8_t opcode,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ ir_op op;
+ ir_ref op1, op2, ref;
+
+ if (opcode == ZEND_ADD) {
+ op = IR_ADD;
+ } else if (opcode == ZEND_SUB) {
+ op = IR_SUB;
+ } else if (opcode == ZEND_MUL) {
+ op = IR_MUL;
+ } else if (opcode == ZEND_DIV) {
+ op = IR_DIV;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ op1 = jit_Z_LVAL(jit, op1_addr);
+ op2 = jit_Z_DVAL(jit, op2_addr);
+ ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2);
+ jit_set_Z_DVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) != IS_REG) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_math_double_long(zend_jit_ctx *jit,
+ uint8_t opcode,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ ir_op op;
+ ir_ref op1, op2, ref;
+
+ if (opcode == ZEND_ADD) {
+ op = IR_ADD;
+ } else if (opcode == ZEND_SUB) {
+ op = IR_SUB;
+ } else if (opcode == ZEND_MUL) {
+ op = IR_MUL;
+ } else if (opcode == ZEND_DIV) {
+ op = IR_DIV;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ op1 = jit_Z_DVAL(jit, op1_addr);
+ op2 = jit_Z_LVAL(jit, op2_addr);
+ ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2));
+ jit_set_Z_DVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) != IS_REG) {
+ if (!zend_jit_same_addr(op1_addr, res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_math_double_double(zend_jit_ctx *jit,
+ uint8_t opcode,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ ir_op op;
+ ir_ref op1, op2, ref;
+
+ if (opcode == ZEND_ADD) {
+ op = IR_ADD;
+ } else if (opcode == ZEND_SUB) {
+ op = IR_SUB;
+ } else if (opcode == ZEND_MUL) {
+ op = IR_MUL;
+ } else if (opcode == ZEND_DIV) {
+ op = IR_DIV;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ op1 = jit_Z_DVAL(jit, op1_addr);
+ op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr);
+ ref = ir_BINARY_OP_D(op, op1, op2);
+ jit_set_Z_DVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) != IS_REG) {
+ if (!zend_jit_same_addr(op1_addr, res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_math_helper(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint8_t opcode,
+ uint8_t op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ uint8_t op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ uint32_t res_var,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info,
+ int may_overflow,
+ int may_throw)
+{
+ ir_ref if_op1_long = IR_UNUSED;
+ ir_ref if_op1_double = IR_UNUSED;
+ ir_ref if_op2_double = IR_UNUSED;
+ ir_ref if_op1_long_op2_long = IR_UNUSED;
+ ir_ref if_op1_long_op2_double = IR_UNUSED;
+ ir_ref if_op1_double_op2_double = IR_UNUSED;
+ ir_ref if_op1_double_op2_long = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+ bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ ir_refs *end_inputs;
+ ir_refs *res_inputs;
+
+ ir_refs_init(end_inputs, 6);
+ ir_refs_init(res_inputs, 6);
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op1_addr);
+ }
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op2_addr);
+ }
+ }
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ jit->delay_var = Z_SSA_VAR(res_addr);
+ jit->delay_refs = res_inputs;
+ }
+
+ if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
+ if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_op1_long);
+ }
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
+ if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_TRUE(if_op1_long_op2_long);
+ }
+ if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ if (if_op1_long) {
+ ir_IF_FALSE_cold(if_op1_long);
+ ir_END_list(slow_inputs);
+ }
+ if (if_op1_long_op2_long) {
+ ir_IF_FALSE_cold(if_op1_long_op2_long);
+ ir_END_list(slow_inputs);
+ }
+ } else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
+ if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_op1_long);
+ }
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
+ if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_long_op2_long);
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_long_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_long_op2_double);
+ }
+ if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ } else {
+ ir_END_list(slow_inputs);
+ }
+ ir_IF_TRUE(if_op1_long_op2_long);
+ }
+ if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+
+ if (if_op1_long) {
+ ir_IF_FALSE_cold(if_op1_long);
+ }
+
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double);
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops) {
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_double_op2_long);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double_op2_long);
+ }
+ if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ } else if (if_op1_long) {
+ ir_END_list(slow_inputs);
+ }
+ } else if ((op1_info & MAY_BE_DOUBLE) &&
+ !(op1_info & MAY_BE_LONG) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (res_info & MAY_BE_DOUBLE)) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double);
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_double_op2_long);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double_op2_long);
+ }
+ if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ } else if ((op2_info & MAY_BE_DOUBLE) &&
+ !(op2_info & MAY_BE_LONG) &&
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (res_info & MAY_BE_DOUBLE)) {
+ if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op2_double);
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_long_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_long_op2_double);
+ }
+ if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ir_END());
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ }
+
+ if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ ir_ref func, arg1, arg2, arg3;
+
+ if (slow_inputs) {
+ ir_MERGE_list(slow_inputs);
+ }
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
+ if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ if (Z_MODE(res_addr) == IS_REG) {
+ arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
+ } else {
+ arg1 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+ arg2 = jit_ZVAL_ADDR(jit, op1_addr);
+ arg3 = jit_ZVAL_ADDR(jit, op2_addr);
+ jit_SET_EX_OPLINE(jit, opline);
+ if (opcode == ZEND_ADD) {
+ func = ir_CONST_FC_FUNC(add_function);
+ } else if (opcode == ZEND_SUB) {
+ func = ir_CONST_FC_FUNC(sub_function);
+ } else if (opcode == ZEND_MUL) {
+ func = ir_CONST_FC_FUNC(mul_function);
+ } else if (opcode == ZEND_DIV) {
+ func = ir_CONST_FC_FUNC(div_function);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
+
+ jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
+ jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
+
+ if (may_throw) {
+ if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
+ } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ } else {
+ zend_jit_check_exception(jit);
+ }
+ }
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
+ return 0;
+ }
+ }
+ ir_refs_add(end_inputs, ir_END());
+ }
+
+ if (end_inputs->count) {
+ ir_MERGE_N(end_inputs->count, end_inputs->refs);
+ }
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ ZEND_ASSERT(jit->delay_refs == res_inputs);
+ ZEND_ASSERT(end_inputs->count == res_inputs->count);
+ jit->delay_var = -1;
+ jit->delay_refs = NULL;
+ if (res_inputs->count == 1) {
+ zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
+ } else {
+ ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs);
+ zend_jit_def_reg(jit, res_addr, phi);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
+{
+ ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
+
+ if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
+ return 0;
+ }
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ return 1;
+}
+
+static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
+{
+ ir_ref ref;
+ ir_ref arg1 = jit_Z_PTR(jit, op1_addr);
+ ir_ref arg2 = jit_Z_PTR(jit, op2_addr);
+
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX);
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ return 1;
+}
+
+static int zend_jit_long_math_helper(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint8_t opcode,
+ uint8_t op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ zend_ssa_range *op1_range,
+ uint8_t op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ zend_ssa_range *op2_range,
+ uint32_t res_var,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info,
+ int may_throw)
+{
+ ir_ref ref = IR_UNUSED;
+ ir_ref if_long1 = IR_UNUSED;
+ ir_ref if_long2 = IR_UNUSED;
+ bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ ir_refs *res_inputs;
+
+ ir_refs_init(res_inputs, 2);
+
+ if (Z_MODE(op1_addr) == IS_REG
+ && Z_LOAD(op1_addr)
+ && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op1_addr);
+ }
+ if (Z_MODE(op2_addr) == IS_REG
+ && Z_LOAD(op2_addr)
+ && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op2_addr);
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_long1);
+ }
+ if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_TRUE(if_long2);
+ }
+
+ if (opcode == ZEND_SL) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
+ if (EXPECTED(op2_lval > 0)) {
+ ref = ir_CONST_LONG(0);
+ } else {
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
+ if (Z_MODE(res_addr) == IS_REG) {
+ ref = ir_CONST_LONG(0); // dead code
+ }
+ }
+ } else {
+ ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
+ }
+ } else {
+ ref = jit_Z_LVAL(jit, op2_addr);
+ if (!op2_range ||
+ op2_range->min < 0 ||
+ op2_range->max >= SIZEOF_ZEND_LONG * 8) {
+
+ ir_ref if_wrong, cold_path, ref2, if_ok;
+ ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
+
+ if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
+ ir_IF_TRUE_cold(if_wrong);
+ if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
+ ir_IF_FALSE(if_ok);
+ jit_SET_EX_OPLINE(jit, opline);
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
+ ir_IF_TRUE(if_ok);
+ ref2 = ir_CONST_LONG(0);
+ cold_path = ir_END();
+ ir_IF_FALSE(if_wrong);
+ ref = ir_SHL_L(op1_ref, ref);
+ ir_MERGE_WITH(cold_path);
+ ref = ir_PHI_2(IR_LONG, ref, ref2);
+ } else {
+ ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref);
+ }
+ }
+ } else if (opcode == ZEND_SR) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
+ if (EXPECTED(op2_lval > 0)) {
+ ref = ir_SAR_L(
+ jit_Z_LVAL(jit, op1_addr),
+ ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1));
+ } else {
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
+ if (Z_MODE(res_addr) == IS_REG) {
+ ref = ir_CONST_LONG(0); // dead code
+ }
+ }
+ } else {
+ ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
+ }
+ } else {
+ ref = jit_Z_LVAL(jit, op2_addr);
+ if (!op2_range ||
+ op2_range->min < 0 ||
+ op2_range->max >= SIZEOF_ZEND_LONG * 8) {
+
+ ir_ref if_wrong, cold_path, ref2, if_ok;
+
+ if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
+ ir_IF_TRUE_cold(if_wrong);
+ if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
+ ir_IF_FALSE(if_ok);
+ jit_SET_EX_OPLINE(jit, opline);
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
+ ir_IF_TRUE(if_ok);
+ ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1);
+ cold_path = ir_END();
+ ir_IF_FALSE(if_wrong);
+ ir_MERGE_WITH(cold_path);
+ ref = ir_PHI_2(IR_LONG, ref, ref2);
+ }
+ ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref);
+ }
+ } else if (opcode == ZEND_MOD) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (op2_lval == 0) {
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
+ if (Z_MODE(res_addr) == IS_REG) {
+ ref = ir_CONST_LONG(0); // dead code
+ }
+ } else if (op2_lval == -1) {
+ /* Prevent overflow error/crash if op1 == LONG_MIN */
+ ref = ir_CONST_LONG(0);
+ } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
+ ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1));
+ } else {
+ ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
+ }
+ } else {
+ ir_ref zero_path = 0;
+ ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
+
+ ref = jit_Z_LVAL(jit, op2_addr);
+ if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
+ ir_ref if_ok = ir_IF(ref);
+ ir_IF_FALSE(if_ok);
+ jit_SET_EX_OPLINE(jit, opline);
+ zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
+ zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
+ ir_IF_TRUE(if_ok);
+ }
+
+ /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
+ if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
+ ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1)));
+ ir_IF_TRUE_cold(if_minus_one);
+ zero_path = ir_END();
+ ir_IF_FALSE(if_minus_one);
+ }
+ ref = ir_MOD_L(op1_ref, ref);
+
+ if (zero_path) {
+ ir_MERGE_WITH(zero_path);
+ ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0));
+ }
+ }
+ } else {
+ ir_op op;
+ ir_ref op1, op2;
+
+ if (opcode == ZEND_BW_OR) {
+ op = IR_OR;
+ } else if (opcode == ZEND_BW_AND) {
+ op = IR_AND;
+ } else if (opcode == ZEND_BW_XOR) {
+ op = IR_XOR;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ op1 = jit_Z_LVAL(jit, op1_addr);
+ op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
+ ref = ir_BINARY_OP_L(op, op1, op2);
+ }
+
+ if (ref) {
+ if (Z_MODE(res_addr) == IS_REG
+ && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))
+ || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
+ jit->delay_var = Z_SSA_VAR(res_addr);
+ jit->delay_refs = res_inputs;
+ }
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ if (Z_MODE(res_addr) != IS_REG) {
+ if (!zend_jit_same_addr(op1_addr, res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ }
+ }
+
+ if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ ir_ref fast_path = ir_END();
+ ir_ref func, arg1, arg2, arg3;
+
+ if (if_long2 && if_long1) {
+ ir_ref ref;
+ ir_IF_FALSE_cold(if_long2);
+ ref = ir_END();
+ ir_IF_FALSE_cold(if_long1);
+ ir_MERGE_2(ref, ir_END());
+ } else if (if_long1) {
+ ir_IF_FALSE_cold(if_long1);
+ } else if (if_long2) {
+ ir_IF_FALSE_cold(if_long2);
+ }
+
+ if (op1_info & MAY_BE_UNDEF) {
+ ir_ref if_def, ref, ref2;
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+
+ // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
+
+ ref2 = jit_EG(uninitialized_zval);
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (op2_info & MAY_BE_UNDEF) {
+ ir_ref if_def, ref, ref2;
+
+ ref = jit_ZVAL_ADDR(jit, op2_addr);
+ if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+
+ // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)));
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
+
+ ref2 = jit_EG(uninitialized_zval);
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ op2_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
+ if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ if (Z_MODE(res_addr) == IS_REG) {
+ arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
+ } else {
+ arg1 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+ arg2 = jit_ZVAL_ADDR(jit, op1_addr);
+ arg3 = jit_ZVAL_ADDR(jit, op2_addr);
+ jit_SET_EX_OPLINE(jit, opline);
+ if (opcode == ZEND_BW_OR) {
+ func = ir_CONST_FC_FUNC(bitwise_or_function);
+ } else if (opcode == ZEND_BW_AND) {
+ func = ir_CONST_FC_FUNC(bitwise_and_function);
+ } else if (opcode == ZEND_BW_XOR) {
+ func = ir_CONST_FC_FUNC(bitwise_xor_function);
+ } else if (opcode == ZEND_SL) {
+ func = ir_CONST_FC_FUNC(shift_left_function);
+ } else if (opcode == ZEND_SR) {
+ func = ir_CONST_FC_FUNC(shift_right_function);
+ } else if (opcode == ZEND_MOD) {
+ func = ir_CONST_FC_FUNC(mod_function);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
+
+ if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
+ /* compound assignment may decrement "op2" refcount */
+ op2_info |= MAY_BE_RC1;
+ }
+
+ jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
+ jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
+
+ if (may_throw) {
+ if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
+ } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ } else {
+ zend_jit_check_exception(jit);
+ }
+ }
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
+ return 0;
+ }
+ }
+
+ ir_MERGE_2(fast_path, ir_END());
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ ZEND_ASSERT(jit->delay_refs == res_inputs);
+ ZEND_ASSERT(res_inputs->count == 2);
+ jit->delay_var = -1;
+ jit->delay_refs = NULL;
+ if (res_inputs->count == 1) {
+ zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
+ } else {
+ ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs);
+ zend_jit_def_reg(jit, res_addr, phi);
+ }
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
+{
+ ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
+
+ if (!zend_jit_long_math_helper(jit, opline, opline->opcode,
+ opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
+ opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
+ opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
+ return 0;
+ }
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ return 1;
+}
+
+static int zend_jit_concat_helper(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint8_t op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ uint8_t op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ zend_jit_addr res_addr,
+ int may_throw)
+{
+ ir_ref if_op1_string = IR_UNUSED;
+ ir_ref if_op2_string = IR_UNUSED;
+ ir_ref fast_path = IR_UNUSED;
+
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
+ if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_TRUE(if_op1_string);
+ }
+ if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
+ if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
+ ir_IF_TRUE(if_op2_string);
+ }
+ if (zend_jit_same_addr(op1_addr, res_addr)) {
+ ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2);
+ /* concatenation with itself may reduce refcount */
+ op2_info |= MAY_BE_RC1;
+ } else {
+ ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
+ ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
+
+ if (op1_type == IS_CV || op1_type == IS_CONST) {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3);
+ } else {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3);
+ }
+ }
+ /* concatenation with empty string may increase refcount */
+ op2_info |= MAY_BE_RCN;
+ jit_FREE_OP(jit, op2_type, op2, op2_info, opline);
+ if (if_op1_string || if_op2_string) {
+ fast_path = ir_END();
+ }
+ }
+ if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
+ (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ if (if_op1_string && if_op2_string) {
+ ir_IF_FALSE(if_op1_string);
+ ir_MERGE_WITH_EMPTY_FALSE(if_op2_string);
+ } else if (if_op1_string) {
+ ir_IF_FALSE_cold(if_op1_string);
+ } else if (if_op2_string) {
+ ir_IF_FALSE_cold(if_op2_string);
+ }
+ }
+ ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
+ ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3);
+ /* concatenation with empty string may increase refcount */
+ op1_info |= MAY_BE_RCN;
+ op2_info |= MAY_BE_RCN;
+ jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
+ jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
+ if (may_throw) {
+ if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
+ } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ } else {
+ zend_jit_check_exception(jit);
+ }
+ }
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ ir_MERGE_WITH(fast_path);
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
+{
+ zend_jit_addr op1_addr, op2_addr;
+
+ ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
+ ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
+
+ op1_addr = OP1_ADDR();
+ op2_addr = OP2_ADDR();
+
+ return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
+}
+
+static int zend_jit_assign_op(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ zend_ssa_range *op1_range,
+ uint32_t op1_def_info,
+ zend_jit_addr op1_def_addr,
+ uint32_t op1_mem_info,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ int may_overflow,
+ int may_throw)
+{
+ int result = 1;
+ ir_ref slow_path = IR_UNUSED;
+
+ ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
+ ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref, ref2, arg2, op1_noref_path;
+ ir_ref if_op1_ref = IR_UNUSED;
+ ir_ref if_op1_typed = IR_UNUSED;
+ binary_op_type binary_op = get_binary_op(opline->extended_value);
+
+ if (Z_MODE(op2_addr) == IS_REG
+ && Z_LOAD(op2_addr)
+ && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
+ /* Force load, so that the register is defined on all paths */
+ zend_jit_use_reg(jit, op2_addr);
+ }
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
+ ir_IF_FALSE(if_op1_ref);
+ op1_noref_path = ir_END();
+ ir_IF_TRUE(if_op1_ref);
+ ref2 = jit_Z_PTR_ref(jit, ref);
+
+ if_op1_typed = jit_if_TYPED_REF(jit, ref2);
+ ir_IF_TRUE_cold(if_op1_typed);
+
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ arg2 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
+ && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp),
+ ref2, arg2, ir_CONST_FC_FUNC(binary_op));
+ } else {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
+ ref2, arg2, ir_CONST_FC_FUNC(binary_op));
+ }
+ zend_jit_check_exception(jit);
+ slow_path = ir_END();
+
+ ir_IF_FALSE(if_op1_typed);
+ ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
+
+ ir_MERGE_WITH(op1_noref_path);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ ZEND_ASSERT(op1_addr == op1_def_addr);
+ op1_def_addr = op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ switch (opline->extended_value) {
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ case ZEND_DIV:
+ result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_overflow, may_throw);
+ break;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ result = zend_jit_long_math_helper(jit, opline, opline->extended_value,
+ opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
+ opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
+ opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_throw);
+ break;
+ case ZEND_CONCAT:
+ result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_def_addr, may_throw);
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
+ return 0;
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ ir_MERGE_WITH(slow_path);
+ }
+
+ return result;
+}
+
+static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_ref if_ref, ref2;
+
+ if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE)));
+ ir_IF_TRUE(if_ref);
+ ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
+ ir_MERGE_WITH_EMPTY_FALSE(if_ref);
+ return ir_PHI_2(IR_ADDR, ref2, ref);
+}
+
+static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ ir_ref ref = jit_ZVAL_ADDR(jit, addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ return ZEND_ADDR_REF_ZVAL(ref);
+}
+
+static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
+{
+ ir_ref if_ref, ref2;
+
+ if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT)));
+ ir_IF_TRUE(if_ref);
+ ref2 = jit_Z_PTR_ref(jit, ref);
+ ir_MERGE_WITH_EMPTY_FALSE(if_ref);
+ return ir_PHI_2(IR_ADDR, ref2, ref);
+}
+
+static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
+{
+ ir_ref ref = jit_ZVAL_ADDR(jit, addr);
+ ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref);
+ return ZEND_ADDR_REF_ZVAL(ref);
+}
+
+static int zend_jit_simple_assign(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr var_addr,
+ uint32_t var_info,
+ uint32_t var_def_info,
+ uint8_t val_type,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr res_addr,
+ bool check_exception)
+{
+ ir_ref end_inputs = IR_UNUSED;
+
+ if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
+ zval *zv = Z_ZV(val_addr);
+
+ if (!res_addr) {
+ jit_ZVAL_COPY_CONST(jit,
+ var_addr,
+ var_info, var_def_info,
+ zv, 1);
+ } else {
+ jit_ZVAL_COPY_CONST(jit,
+ var_addr,
+ var_info, var_def_info,
+ zv, 1);
+ jit_ZVAL_COPY_CONST(jit,
+ res_addr,
+ -1, var_def_info,
+ zv, 1);
+ }
+ } else {
+ if (val_info & MAY_BE_UNDEF) {
+ ir_ref if_def, ret;
+
+ if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+
+ jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL);
+ if (res_addr) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+
+ ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL);
+ // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
+ ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
+
+ if (check_exception) {
+ ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
+ }
+
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ if (val_info & MAY_BE_REF) {
+ if (val_type == IS_CV) {
+ ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ val_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else {
+ ir_ref ref, type, if_ref, ref2, refcount, if_not_zero;
+
+ ref = jit_ZVAL_ADDR(jit, val_addr);
+ type = jit_Z_TYPE_ref(jit, ref);
+ if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE)));
+
+ ir_IF_TRUE_cold(if_ref);
+ ref = jit_Z_PTR_ref(jit, ref);
+ ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ if (!res_addr) {
+ jit_ZVAL_COPY(jit,
+ var_addr,
+ var_info,
+ ZEND_ADDR_REF_ZVAL(ref2), val_info, 1);
+ } else {
+ jit_ZVAL_COPY_2(jit,
+ res_addr,
+ var_addr,
+ var_info,
+ ZEND_ADDR_REF_ZVAL(ref2), val_info, 2);
+ }
+
+ refcount = jit_GC_DELREF(jit, ref);
+ if_not_zero = ir_IF(refcount);
+ ir_IF_FALSE(if_not_zero);
+ // TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ???
+ // This is related to GH-10168 (keep this before GH-10168 is completely closed)
+ // jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL);
+ jit_ZVAL_DTOR(jit, ref, val_info, opline);
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_not_zero);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE(if_ref);
+ }
+ }
+
+ if (!res_addr) {
+ jit_ZVAL_COPY(jit,
+ var_addr,
+ var_info,
+ val_addr, val_info, val_type == IS_CV);
+ } else {
+ jit_ZVAL_COPY_2(jit,
+ res_addr,
+ var_addr,
+ var_info,
+ val_addr, val_info, val_type == IS_CV ? 2 : 1);
+ }
+ }
+
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr __var_use_addr,
+ zend_jit_addr var_addr,
+ uint32_t __var_info,
+ uint32_t __var_def_info,
+ uint8_t val_type,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr __res_addr,
+ bool __check_exception)
+{
+ jit_stub_id func;
+ ir_ref undef_path = IR_UNUSED;
+
+ if (val_info & MAY_BE_UNDEF) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr);
+ } else {
+ ir_ref if_def;
+
+ ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
+ if_def = ir_IF(jit_Z_TYPE(jit, val_addr));
+ ir_IF_FALSE_cold(if_def);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
+
+ ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC),
+ jit_ZVAL_ADDR(jit, var_addr),
+ jit_EG(uninitialized_zval));
+
+ undef_path = ir_END();
+ ir_IF_TRUE(if_def);
+ }
+ }
+
+ if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
+ func = jit_stub_assign_tmp;
+ } else if (val_type == IS_CONST) {
+ func = jit_stub_assign_const;
+ } else if (val_type == IS_TMP_VAR) {
+ func = jit_stub_assign_tmp;
+ } else if (val_type == IS_VAR) {
+ if (!(val_info & MAY_BE_REF)) {
+ func = jit_stub_assign_tmp;
+ } else {
+ func = jit_stub_assign_var;
+ }
+ } else if (val_type == IS_CV) {
+ if (!(val_info & MAY_BE_REF)) {
+ func = jit_stub_assign_cv_noref;
+ } else {
+ func = jit_stub_assign_cv;
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC),
+ jit_ZVAL_ADDR(jit, var_addr),
+ jit_ZVAL_ADDR(jit, val_addr));
+
+ if (undef_path) {
+ ir_MERGE_WITH(undef_path);
+ }
+
+ return 1;
+}
+
+static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr var_use_addr,
+ zend_jit_addr var_addr,
+ uint32_t var_info,
+ uint32_t var_def_info,
+ uint8_t val_type,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr res_addr,
+ zend_jit_addr ref_addr,
+ bool check_exception)
+{
+ ir_ref if_refcounted = IR_UNUSED;
+ ir_ref simple_inputs = IR_UNUSED;
+ bool done = 0;
+ zend_jit_addr real_res_addr = 0;
+ ir_refs *end_inputs;
+ ir_refs *res_inputs;
+
+ ir_refs_init(end_inputs, 6);
+ ir_refs_init(res_inputs, 6);
+
+ if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, val_addr);
+ }
+
+ if (Z_MODE(var_addr) == IS_REG) {
+ jit->delay_var = Z_SSA_VAR(var_addr);
+ jit->delay_refs = res_inputs;
+ if (Z_MODE(res_addr) == IS_REG) {
+ real_res_addr = res_addr;
+ res_addr = 0;
+ }
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ jit->delay_var = Z_SSA_VAR(res_addr);
+ jit->delay_refs = res_inputs;
+ }
+
+ if ((var_info & MAY_BE_REF) || ref_addr) {
+ ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path;
+ uintptr_t func;
+
+ if (!ref_addr) {
+ ref = jit_ZVAL_ADDR(jit, var_use_addr);
+ if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
+ ir_IF_TRUE(if_ref);
+ ref2 = jit_Z_PTR_ref(jit, ref);
+ } else {
+ ref2 = jit_ZVAL_ADDR(jit, ref_addr);
+ }
+ if_typed = jit_if_TYPED_REF(jit, ref2);
+ ir_IF_TRUE_cold(if_typed);
+ jit_SET_EX_OPLINE(jit, opline);
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr;
+
+ if (opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_OBJ) {
+ real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ } else {
+ ZEND_ASSERT(opline->opcode == ZEND_ASSIGN);
+ real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ }
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg2 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+ if (!res_addr) {
+ if (val_type == IS_CONST) {
+ func = (uintptr_t)zend_jit_assign_const_to_typed_ref;
+ } else if (val_type == IS_TMP_VAR) {
+ func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref;
+ } else if (val_type == IS_VAR) {
+ func = (uintptr_t)zend_jit_assign_var_to_typed_ref;
+ } else if (val_type == IS_CV) {
+ func = (uintptr_t)zend_jit_assign_cv_to_typed_ref;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2);
+ } else {
+ if (val_type == IS_CONST) {
+ func = (uintptr_t)zend_jit_assign_const_to_typed_ref2;
+ } else if (val_type == IS_TMP_VAR) {
+ func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2;
+ } else if (val_type == IS_VAR) {
+ func = (uintptr_t)zend_jit_assign_var_to_typed_ref2;
+ } else if (val_type == IS_CV) {
+ func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr));
+ }
+ if (check_exception) {
+ zend_jit_check_exception(jit);
+ }
+ ir_refs_add(end_inputs, ir_END());
+
+ if (!ref_addr) {
+ ir_IF_FALSE(if_ref);
+ non_ref_path = ir_END();
+ ir_IF_FALSE(if_typed);
+ ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
+ ir_MERGE_WITH(non_ref_path);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else {
+ ir_IF_FALSE(if_typed);
+ }
+ }
+
+ if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ ir_ref ref, counter, if_not_zero;
+
+ if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr);
+ ir_IF_FALSE(if_refcounted);
+ ir_END_list(simple_inputs);
+ ir_IF_TRUE_cold(if_refcounted);
+ } else if (RC_MAY_BE_1(var_info)) {
+ done = 1;
+ }
+ ref = jit_Z_PTR(jit, var_use_addr);
+ if (RC_MAY_BE_1(var_info)) {
+ if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ counter = jit_GC_DELREF(jit, ref);
+
+ if_not_zero = ir_IF(counter);
+ ir_IF_FALSE(if_not_zero);
+ jit_ZVAL_DTOR(jit, ref, var_info, opline);
+ if (check_exception) {
+ zend_jit_check_exception(jit);
+ }
+ ir_refs_add(end_inputs, ir_END());
+ ir_IF_TRUE(if_not_zero);
+ if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
+ ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
+ ir_IF_FALSE(if_may_leak);
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
+
+ if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
+ ZEND_ASSERT(jit->delay_refs == res_inputs);
+ ZEND_ASSERT(res_inputs->count > 0);
+ ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
+ }
+ if (check_exception && (val_info & MAY_BE_UNDEF)) {
+ zend_jit_check_exception(jit);
+ }
+ ir_refs_add(end_inputs, ir_END());
+ ir_IF_TRUE(if_may_leak);
+ }
+ if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
+ ZEND_ASSERT(jit->delay_refs == res_inputs);
+ ZEND_ASSERT(res_inputs->count > 0);
+ ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
+ }
+ if (check_exception && (val_info & MAY_BE_UNDEF)) {
+ zend_jit_check_exception(jit);
+ }
+ ir_refs_add(end_inputs, ir_END());
+ } else /* if (RC_MAY_BE_N(var_info)) */ {
+ jit_GC_DELREF(jit, ref);
+ if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
+ ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
+ ir_IF_FALSE(if_may_leak);
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
+ ir_END_list(simple_inputs);
+ ir_IF_TRUE(if_may_leak);
+ }
+ ir_END_list(simple_inputs);
+ }
+ }
+
+ if (simple_inputs) {
+ ir_MERGE_list(simple_inputs);
+ }
+
+ if (!done) {
+ if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) {
+ return 0;
+ }
+ if (end_inputs->count) {
+ ir_refs_add(end_inputs, ir_END());
+ }
+ }
+
+ if (end_inputs->count) {
+ ir_MERGE_N(end_inputs->count, end_inputs->refs);
+ }
+
+ if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
+ ir_ref phi;
+
+ ZEND_ASSERT(jit->delay_refs == res_inputs);
+ ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1));
+ jit->delay_var = -1;
+ jit->delay_refs = NULL;
+ if (res_inputs->count == 1) {
+ phi = res_inputs->refs[0];
+ } else {
+ phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
+ res_inputs->count, res_inputs->refs);
+ }
+ if (Z_MODE(var_addr) == IS_REG) {
+ if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
+ phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
+ }
+ zend_jit_def_reg(jit, var_addr, phi);
+ if (real_res_addr) {
+ if (var_def_info & MAY_BE_LONG) {
+ jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr));
+ } else {
+ jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr));
+ }
+ }
+ } else {
+ zend_jit_def_reg(jit, res_addr, phi);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
+{
+ if (op1_addr != op1_def_addr) {
+ if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
+ return 0;
+ }
+ if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
+ op1_addr = op1_def_addr;
+ }
+ }
+
+ if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) {
+ return 0;
+ }
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ return 1;
+}
+
+static int zend_jit_assign(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_use_addr,
+ uint32_t op1_def_info,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_jit_addr op2_def_addr,
+ uint32_t res_info,
+ zend_jit_addr res_addr,
+ zend_jit_addr ref_addr,
+ int may_throw)
+{
+ ZEND_ASSERT(opline->op1_type == IS_CV);
+
+ if (op2_addr != op2_def_addr) {
+ if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
+ return 0;
+ }
+ if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
+ op2_addr = op2_def_addr;
+ }
+ }
+
+ if (Z_MODE(op1_addr) != IS_REG
+ && Z_MODE(op1_use_addr) == IS_REG
+ && !Z_LOAD(op1_use_addr)
+ && !Z_STORE(op1_use_addr)) {
+ /* Force type update */
+ op1_info |= MAY_BE_UNDEF;
+ }
+ if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info,
+ opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) {
+ return 0;
+ }
+ if (Z_MODE(op1_addr) == IS_REG) {
+ if (Z_STORE(op1_addr)) {
+ if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
+ return 0;
+ }
+ } else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
+ && Z_MODE(op1_use_addr) == IS_MEM_ZVAL
+ && Z_REG(op1_use_addr) == ZREG_FP
+ && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) {
+ /* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */
+ if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) {
+ jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG);
+ if (JIT_G(current_frame)) {
+ SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1);
+ }
+ } else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE);
+ if (JIT_G(current_frame)) {
+ SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1);
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ }
+ }
+ if (opline->result_type != IS_UNUSED) {
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static ir_op zend_jit_cmp_op(const zend_op *opline)
+{
+ ir_op op;
+
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ case ZEND_CASE_STRICT:
+ op = IR_EQ;
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ op = IR_NE;
+ break;
+ case ZEND_IS_SMALLER:
+ op = IR_LT;
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ op = IR_LE;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ return op;
+}
+
+static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_ssa_range *op1_range,
+ zend_jit_addr op1_addr,
+ zend_ssa_range *op2_range,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint8_t smart_branch_opcode,
+ uint32_t target_label,
+ uint32_t target_label2,
+ const void *exit_addr,
+ bool skip_comparison)
+{
+ ir_ref ref;
+ bool result;
+
+ if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
+ if (!smart_branch_opcode ||
+ smart_branch_opcode == ZEND_JMPZ_EX ||
+ smart_branch_opcode == ZEND_JMPNZ_EX) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE);
+ }
+ if (smart_branch_opcode && !exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ ||
+ smart_branch_opcode == ZEND_JMPZ_EX) {
+ return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2);
+ } else if (smart_branch_opcode == ZEND_JMPNZ ||
+ smart_branch_opcode == ZEND_JMPNZ_EX) {
+ return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ }
+ if (opline->opcode != ZEND_IS_IDENTICAL
+ && opline->opcode != ZEND_IS_NOT_IDENTICAL
+ && opline->opcode != ZEND_CASE_STRICT) {
+ return ir_END();
+ } else {
+ return IR_NULL; /* success */
+ }
+ }
+
+ ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr));
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else {
+ if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ }
+ }
+ } else if (smart_branch_opcode) {
+ return jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+
+ if (opline->opcode != ZEND_IS_IDENTICAL
+ && opline->opcode != ZEND_IS_NOT_IDENTICAL
+ && opline->opcode != ZEND_CASE_STRICT) {
+ return ir_END();
+ } else {
+ return IR_NULL; /* success */
+ }
+}
+
+static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr));
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ return jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+ return ir_END();
+}
+
+static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr)));
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ return jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+ return ir_END();
+}
+
+static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr));
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else {
+ if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ }
+ }
+ } else if (smart_branch_opcode) {
+ return jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+ if (opline->opcode != ZEND_IS_IDENTICAL
+ && opline->opcode != ZEND_IS_NOT_IDENTICAL
+ && opline->opcode != ZEND_CASE_STRICT) {
+ return ir_END();
+ } else {
+ return IR_NULL; /* success */
+ }
+}
+
+static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0));
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ return jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+
+ return ir_END();
+}
+
+static int zend_jit_cmp(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_ssa_range *op1_range,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_ssa_range *op2_range,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ int may_throw,
+ uint8_t smart_branch_opcode,
+ uint32_t target_label,
+ uint32_t target_label2,
+ const void *exit_addr,
+ bool skip_comparison)
+{
+ ir_ref ref = IR_UNUSED;
+ ir_ref if_op1_long = IR_UNUSED;
+ ir_ref if_op1_double = IR_UNUSED;
+ ir_ref if_op2_double = IR_UNUSED;
+ ir_ref if_op1_long_op2_long = IR_UNUSED;
+ ir_ref if_op1_long_op2_double = IR_UNUSED;
+ ir_ref if_op1_double_op2_double = IR_UNUSED;
+ ir_ref if_op1_double_op2_long = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+ bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ bool has_slow =
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
+ ir_refs *end_inputs;
+
+ ir_refs_init(end_inputs, 8);
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op1_addr);
+ }
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op2_addr);
+ }
+ }
+
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_op1_long);
+ }
+ if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_long_op2_long);
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_long_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_long_op2_double);
+ }
+ ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ } else {
+ ir_END_list(slow_inputs);
+ }
+ ir_IF_TRUE(if_op1_long_op2_long);
+ }
+ ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+
+ if (if_op1_long) {
+ ir_IF_FALSE_cold(if_op1_long);
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double);
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops) {
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_double_op2_long);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double_op2_long);
+ }
+ ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ } else if (if_op1_long) {
+ ir_END_list(slow_inputs);
+ }
+ } else if ((op1_info & MAY_BE_DOUBLE) &&
+ !(op1_info & MAY_BE_LONG) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
+ if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op1_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double);
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_double_op2_long);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_double_op2_long);
+ }
+ ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ } else if ((op2_info & MAY_BE_DOUBLE) &&
+ !(op2_info & MAY_BE_LONG) &&
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
+ if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
+ ir_IF_FALSE_cold(if_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op2_double);
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
+ if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
+ ir_IF_TRUE(if_op1_double_op2_double);
+ }
+ ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ if (if_op1_double_op2_double) {
+ ir_IF_FALSE_cold(if_op1_double_op2_double);
+ }
+ }
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_op1_long_op2_double);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_op1_long_op2_double);
+ }
+ ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ } else if (if_op1_double_op2_double) {
+ ir_END_list(slow_inputs);
+ }
+ }
+
+ if (has_slow ||
+ (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ ir_ref op1, op2, ref;
+
+ if (slow_inputs) {
+ ir_MERGE_list(slow_inputs);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+
+ op1 = jit_ZVAL_ADDR(jit, op1_addr);
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0);
+ }
+ op2 = jit_ZVAL_ADDR(jit, op2_addr);
+ if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
+ op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0);
+ }
+ ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2);
+ if (opline->opcode != ZEND_CASE) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
+ if (may_throw) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ }
+
+ ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ ir_refs_add(end_inputs, ref);
+ }
+
+ if (end_inputs->count) {
+ uint32_t n = end_inputs->count;
+
+ if (smart_branch_opcode && !exit_addr) {
+ zend_basic_block *bb;
+ ir_ref ref;
+ uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label2 : target_label;
+ uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label : target_label2;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ ZEND_ASSERT(bb->successors_count == 2);
+
+ if (UNEXPECTED(bb->successors[0] == bb->successors[1])) {
+ ir_ref merge_inputs = IR_UNUSED;
+
+ while (n) {
+ n--;
+ ir_IF_TRUE(end_inputs->refs[n]);
+ ir_END_list(merge_inputs);
+ ir_IF_FALSE(end_inputs->refs[n]);
+ ir_END_list(merge_inputs);
+ }
+ ir_MERGE_list(merge_inputs);
+ _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
+ } else if (n == 1) {
+ ref = end_inputs->refs[0];
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
+ } else {
+ ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
+
+ while (n) {
+ n--;
+ jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label);
+ ir_END_list(true_inputs);
+ jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label2);
+ ir_END_list(false_inputs);
+ }
+ ir_MERGE_list(true_inputs);
+ _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
+ ir_MERGE_list(false_inputs);
+ _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END());
+ }
+ jit->b = -1;
+ } else {
+ ir_MERGE_N(n, end_inputs->refs);
+ }
+ } else if (smart_branch_opcode && !exit_addr) {
+ /* dead code */
+ _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
+ jit->b = -1;
+ }
+
+ return 1;
+}
+
+static int zend_jit_identical(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_ssa_range *op1_range,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_ssa_range *op2_range,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ int may_throw,
+ uint8_t smart_branch_opcode,
+ uint32_t target_label,
+ uint32_t target_label2,
+ const void *exit_addr,
+ bool skip_comparison)
+{
+ bool always_false = 0, always_true = 0;
+ ir_ref ref = IR_UNUSED;
+
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr);
+ op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0);
+ op1_info |= MAY_BE_NULL;
+ op1_addr = ZEND_ADDR_REF_ZVAL(op1);
+ }
+ if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
+ ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr);
+ op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0);
+ op2_info |= MAY_BE_NULL;
+ op2_addr = ZEND_ADDR_REF_ZVAL(op2);
+ }
+
+ if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
+ always_false = 1;
+ } else if (has_concrete_type(op1_info)
+ && has_concrete_type(op2_info)
+ && concrete_type(op1_info) == concrete_type(op2_info)
+ && concrete_type(op1_info) <= IS_TRUE) {
+ always_true = 1;
+ } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
+ always_true = 1;
+ } else {
+ always_false = 1;
+ }
+ }
+
+ if (always_true) {
+ if (opline->opcode != ZEND_CASE_STRICT) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ if (!smart_branch_opcode
+ || smart_branch_opcode == ZEND_JMPZ_EX
+ || smart_branch_opcode == ZEND_JMPNZ_EX) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE);
+ }
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ uint32_t label;
+
+ if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label : target_label2;
+ } else {
+ label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label2 : target_label;
+ }
+ _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
+ jit->b = -1;
+ }
+ return 1;
+ } else if (always_false) {
+ if (opline->opcode != ZEND_CASE_STRICT) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ if (!smart_branch_opcode
+ || smart_branch_opcode == ZEND_JMPZ_EX
+ || smart_branch_opcode == ZEND_JMPNZ_EX) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE);
+ }
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ uint32_t label;
+
+ if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label2 : target_label;
+ } else {
+ label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
+ target_label : target_label2;
+ }
+ _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
+ jit->b = -1;
+ }
+ return 1;
+ }
+
+ if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) {
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+ if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) {
+ ref = jit_ZVAL_ADDR(jit, op2_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op2_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
+ (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
+ ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
+ if (!ref) {
+ return 0;
+ }
+ } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
+ (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
+ ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
+ if (!ref) {
+ return 0;
+ }
+ } else {
+ if (opline->op1_type != IS_CONST) {
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ }
+ if (opline->op2_type != IS_CONST) {
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ }
+ }
+
+ if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
+ zval *val = Z_ZV(op1_addr);
+
+ ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val)));
+ } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
+ zval *val = Z_ZV(op2_addr);
+
+ ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val)));
+ } else {
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ if (may_throw) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ jit_ZVAL_ADDR(jit, op2_addr));
+ }
+
+ if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
+ if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ }
+ if (opline->opcode != ZEND_CASE_STRICT) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
+ if (may_throw) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ }
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ /* swap labels */
+ uint32_t tmp = target_label;
+ target_label = target_label2;
+ target_label2 = tmp;
+ }
+ ref = jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
+ }
+ }
+
+ if (smart_branch_opcode && !exit_addr) {
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ ZEND_ASSERT(bb->successors_count == 2);
+
+ if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) {
+ ir_IF_TRUE(ref);
+ ir_MERGE_WITH_EMPTY_FALSE(ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
+ } else {
+ ZEND_ASSERT(bb->successors_count == 2);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
+ }
+ jit->b = -1;
+ }
+
+ return 1;
+}
+
+static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
+{
+ uint32_t true_label = -1;
+ uint32_t false_label = -1;
+ bool set_bool = 0;
+ bool set_bool_not = 0;
+ bool always_true = 0, always_false = 0;
+ ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
+ ir_type type = IR_UNUSED;
+
+ if (branch_opcode == ZEND_BOOL) {
+ set_bool = 1;
+ } else if (branch_opcode == ZEND_BOOL_NOT) {
+ set_bool = 1;
+ set_bool_not = 1;
+ } else if (branch_opcode == ZEND_JMPZ) {
+ true_label = target_label2;
+ false_label = target_label;
+ } else if (branch_opcode == ZEND_JMPNZ) {
+ true_label = target_label;
+ false_label = target_label2;
+ } else if (branch_opcode == ZEND_JMPZ_EX) {
+ set_bool = 1;
+ true_label = target_label2;
+ false_label = target_label;
+ } else if (branch_opcode == ZEND_JMPNZ_EX) {
+ set_bool = 1;
+ true_label = target_label;
+ false_label = target_label2;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
+ if (zend_is_true(Z_ZV(op1_addr))) {
+ always_true = 1;
+ } else {
+ always_false = 1;
+ }
+ } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
+ if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
+ always_true = 1;
+ } else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0);
+ }
+ always_false = 1;
+ }
+ }
+
+ if (always_true) {
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ if (true_label != (uint32_t)-1) {
+ ZEND_ASSERT(exit_addr == NULL);
+ _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
+ jit->b = -1;
+ }
+ return 1;
+ } else if (always_false) {
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ if (false_label != (uint32_t)-1) {
+ ZEND_ASSERT(exit_addr == NULL);
+ _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END());
+ jit->b = -1;
+ }
+ return 1;
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
+ type = jit_Z_TYPE(jit, op1_addr);
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
+ ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE)));
+
+ ir_IF_TRUE_cold(if_type);
+
+ if (op1_info & MAY_BE_UNDEF) {
+ zend_jit_type_check_undef(jit,
+ type,
+ opline->op1.var,
+ opline, 1, 0, 1);
+ }
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
+ }
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ ir_END_list(end_inputs);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (false_label != (uint32_t)-1) {
+ ir_END_list(false_inputs);
+ } else {
+ ir_END_list(end_inputs);
+ }
+ ir_IF_FALSE(if_type);
+ }
+
+ if (op1_info & MAY_BE_TRUE) {
+ ir_ref if_type = IR_UNUSED;
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+ if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE)));
+
+ ir_IF_TRUE(if_type);
+ }
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
+ }
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ ir_END_list(end_inputs);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (true_label != (uint32_t)-1) {
+ ir_END_list(true_inputs);
+ } else {
+ ir_END_list(end_inputs);
+ }
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ }
+ }
+ }
+
+ if (op1_info & MAY_BE_LONG) {
+ ir_ref if_long = IR_UNUSED;
+ ir_ref ref;
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
+ if (!type) {
+ type = jit_Z_TYPE(jit, op1_addr);
+ }
+ if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG)));
+ ir_IF_TRUE(if_long);
+ }
+ ref = jit_Z_LVAL(jit, op1_addr);
+ if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
+ ref = ir_NE(ref, ir_CONST_LONG(0));
+ if (set_bool_not) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ ir_END_list(end_inputs);
+ } else if (exit_addr) {
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE)));
+ }
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ ir_END_list(end_inputs);
+ } else {
+ ir_ref if_val = ir_IF(ref);
+ ir_IF_TRUE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
+ }
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
+ }
+ ir_END_list(false_inputs);
+ }
+ if (if_long) {
+ ir_IF_FALSE(if_long);
+ }
+ }
+
+ if (op1_info & MAY_BE_DOUBLE) {
+ ir_ref if_double = IR_UNUSED;
+ ir_ref ref;
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (!type) {
+ type = jit_Z_TYPE(jit, op1_addr);
+ }
+ if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE)));
+ ir_IF_TRUE(if_double);
+ }
+ ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0));
+ if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
+ if (set_bool_not) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ ir_END_list(end_inputs);
+ } else if (exit_addr) {
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ ir_END_list(end_inputs);
+ } else {
+ ir_ref if_val = ir_IF(ref);
+ ir_IF_TRUE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
+ }
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
+ }
+ ir_END_list(false_inputs);
+ }
+ if (if_double) {
+ ir_IF_FALSE(if_double);
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr));
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
+ if (may_throw) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ }
+ if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
+ if (set_bool_not) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ }
+ } else if (exit_addr) {
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ }
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ir_ref if_val = ir_IF(ref);
+ ir_IF_TRUE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
+ }
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(if_val);
+ if (set_bool) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
+ }
+ ir_END_list(false_inputs);
+ }
+ }
+
+ if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) {
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+ } else {
+ _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ uint32_t defined_label = (uint32_t)-1;
+ uint32_t undefined_label = (uint32_t)-1;
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ zend_jit_addr res_addr = 0;
+ ir_ref ref, ref2, if_set, if_zero, if_set2;
+ ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
+
+ if (smart_branch_opcode && !exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ defined_label = target_label2;
+ undefined_label = target_label;
+ } else if (smart_branch_opcode == ZEND_JMPNZ) {
+ defined_label = target_label;
+ undefined_label = target_label2;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ res_addr = RES_ADDR();
+ }
+
+ // if (CACHED_PTR(opline->extended_value)) {
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
+
+ if_set = ir_IF(ref);
+
+ ir_IF_FALSE_cold(if_set);
+ if_zero = ir_END();
+
+ ir_IF_TRUE(if_set);
+ if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
+ ir_IF_FALSE(if_set2);
+
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(true_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
+ ir_END_list(end_inputs);
+ }
+
+ ir_IF_TRUE_cold(if_set2);
+
+ ref2 = jit_EG(zend_constants);
+ ref = ir_SHR_A(ref, ir_CONST_ADDR(1));
+ if (sizeof(void*) == 8) {
+ ref = ir_TRUNC_U32(ref);
+ }
+ ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements))));
+ ref2 = ir_IF(ref2);
+ ir_IF_TRUE(ref2);
+
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
+ ir_END_list(end_inputs);
+ }
+
+ ir_IF_FALSE(ref2);
+ ir_MERGE_2(if_zero, ir_END());
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL);
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ ir_GUARD(ref2, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr));
+ }
+ ir_END_list(end_inputs);
+ } else if (smart_branch_opcode) {
+ ref2 = ir_IF(ref2);
+ ir_IF_TRUE(ref2);
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(ref2);
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE)));
+ ir_END_list(end_inputs);
+ }
+
+ if (!smart_branch_opcode || exit_addr) {
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+ } else {
+ _zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, const zend_op_array *op_array, int8_t reg)
+{
+ zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr));
+
+ ir_IF_FALSE_cold(if_def);
+
+ if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
+ if (!zend_jit_save_call_chain(jit, -1)) {
+ return 0;
+ }
+ }
+
+ if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
+ && (opline-1)->opcode != ZEND_FETCH_LIST_R
+ && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
+ && !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
+ zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
+
+ zend_jit_zval_try_addref(jit, val_addr);
+ }
+
+ jit_LOAD_IP_ADDR(jit, opline - 1);
+
+ /* We can't use trace_escape() because opcode handler may be overridden by JIT */
+ zend_jit_op_array_trace_extension *jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ size_t offset = jit_extension->offset;
+ ir_ref ref = ir_CONST_FC_FUNC(ZEND_OP_TRACE_INFO((opline - 1), offset)->orig_handler);
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ref);
+ } else {
+ ir_CALL_1(IR_I32, ref, jit_FP(jit));
+ ir_RETURN(ir_CONST_I32(1));
+ }
+
+ ir_IF_TRUE(if_def);
+
+ return 1;
+}
+
+static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg)
+{
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
+
+ // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
+ jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1);
+ return 1;
+}
+
+static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr val_addr,
+ uint8_t type,
+ bool deref,
+ uint32_t flags,
+ bool op1_avoid_refcounting)
+{
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ int32_t exit_point;
+ const void *res_exit_addr = NULL;
+ ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED;
+ ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
+ uint32_t old_op1_info = 0;
+ uint32_t old_info;
+ ir_ref old_ref;
+
+
+ if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
+ old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
+ if (op1_avoid_refcounting
+ || ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) {
+ SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
+ }
+ }
+ old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
+ CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
+
+ if (deref) {
+ ir_ref if_type;
+
+ if (type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
+ if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)));
+ } else {
+ if_type = jit_if_Z_TYPE(jit, val_addr, type);
+ }
+ ir_IF_TRUE(if_type);
+ end1 = ir_END();
+ ref1 = ref;
+ ir_IF_FALSE_cold(if_type);
+
+ SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
+ exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
+ res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!res_exit_addr) {
+ return 0;
+ }
+
+ jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr);
+ ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
+ val_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
+ exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
+ res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!res_exit_addr) {
+ return 0;
+ }
+
+ if (!deref && type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
+ ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr));
+ } else {
+ jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr);
+ }
+
+ if (deref) {
+ ir_MERGE_WITH(end1);
+ ref = ir_PHI_2(IR_ADDR, ref, ref1);
+ }
+
+ val_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref);
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
+ if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
+ }
+
+ return val_addr;
+}
+
+static int zend_jit_fetch_constant(zend_jit_ctx *jit,
+ const zend_op *opline,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ zend_jit_addr res_addr)
+{
+ zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
+ uint32_t res_info = RES_INFO();
+ ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path;
+
+ // JIT: c = CACHED_PTR(opline->extended_value);
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
+
+ // JIT: if (c != NULL)
+ if_set = ir_IF(ref);
+
+ if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
+ // JIT: if (!IS_SPECIAL_CACHE_VAL(c))
+ ir_IF_FALSE_cold(if_set);
+ not_set_path = ir_END();
+ ir_IF_TRUE(if_set);
+ if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
+ ir_IF_TRUE_cold(if_special);
+ special_path = ir_END();
+ ir_IF_FALSE(if_special);
+ fast_path = ir_END();
+ ir_MERGE_2(not_set_path, special_path);
+ } else {
+ ir_IF_TRUE(if_set);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_set);
+ }
+
+ // JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num);
+ jit_SET_EX_OPLINE(jit, opline);
+ ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant),
+ ir_CONST_ADDR(zv),
+ ir_CONST_U32(opline->op1.num));
+ ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ ir_MERGE_WITH(fast_path);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+
+ if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
+ uint8_t type = concrete_type(res_info);
+ zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0);
+ if (!const_addr) {
+ return 0;
+ }
+
+ res_info &= ~MAY_BE_GUARD;
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+
+ // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
+ jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1);
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ } else {
+ ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
+ jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1);
+ }
+
+
+ return 1;
+}
+
+static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ uint32_t mask;
+ zend_jit_addr op1_addr = OP1_ADDR();
+ zend_jit_addr res_addr = 0;
+ uint32_t true_label = -1, false_label = -1;
+ ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
+
+ // TODO: support for is_resource() ???
+ ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
+
+ if (smart_branch_opcode && !exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ true_label = target_label2;
+ false_label = target_label;
+ } else if (smart_branch_opcode == ZEND_JMPNZ) {
+ true_label = target_label;
+ false_label = target_label2;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ res_addr = RES_ADDR();
+ }
+
+ if (op1_info & MAY_BE_UNDEF) {
+ ir_ref if_def = IR_UNUSED;
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+ }
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
+ zend_jit_check_exception_undef_result(jit, opline);
+ if (opline->extended_value & MAY_BE_NULL) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(true_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
+ if (if_def) {
+ ir_END_list(end_inputs);
+ }
+ }
+ }
+
+ if (if_def) {
+ ir_IF_TRUE(if_def);
+ op1_info |= MAY_BE_NULL;
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ mask = opline->extended_value;
+ if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else if (end_inputs) {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(true_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
+ ir_END_list(end_inputs);
+ }
+ } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else if (end_inputs) {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ir_ref ref;
+ bool invert = 0;
+ uint8_t type;
+
+ switch (mask) {
+ case MAY_BE_NULL: type = IS_NULL; break;
+ case MAY_BE_FALSE: type = IS_FALSE; break;
+ case MAY_BE_TRUE: type = IS_TRUE; break;
+ case MAY_BE_LONG: type = IS_LONG; break;
+ case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
+ case MAY_BE_STRING: type = IS_STRING; break;
+ case MAY_BE_ARRAY: type = IS_ARRAY; break;
+ case MAY_BE_OBJECT: type = IS_OBJECT; break;
+ case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
+ case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
+ default:
+ type = 0;
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+ if (type == 0) {
+ ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask));
+ if (!smart_branch_opcode) {
+ ref = ir_NE(ref, ir_CONST_U32(0));
+ }
+ } else if (invert) {
+ ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
+ } else {
+ ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
+ }
+
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ ir_ref if_val = ir_IF(ref);
+ ir_IF_TRUE(if_val);
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(if_val);
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ ir_END_list(end_inputs);
+ }
+ }
+ }
+
+ if (!smart_branch_opcode || exit_addr) {
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ } else if (exit_addr && !jit->ctx.control) {
+ ir_BEGIN(IR_UNUSED); /* unreachable block */
+ }
+ } else {
+ _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ zend_jit_addr res_addr = RES_ADDR();
+ uint32_t true_label = -1, false_label = -1;
+ ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
+
+ // TODO: support for empty() ???
+ ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
+
+ if (smart_branch_opcode && !exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ true_label = target_label2;
+ false_label = target_label;
+ } else if (smart_branch_opcode == ZEND_JMPNZ) {
+ true_label = target_label;
+ false_label = target_label2;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ res_addr = RES_ADDR();
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
+ if (exit_addr) {
+ ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
+ } else if (smart_branch_opcode) {
+ ir_END_list(true_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
+ ir_END_list(end_inputs);
+ }
+ } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
+ if (exit_addr) {
+ ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
+ } else if (smart_branch_opcode) {
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL));
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ ir_ref if_val = ir_IF(ref);
+ ir_IF_TRUE(if_val);
+ ir_END_list(true_inputs);
+ ir_IF_FALSE(if_val);
+ ir_END_list(false_inputs);
+ } else {
+ jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
+ ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (!smart_branch_opcode || exit_addr) {
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+ } else {
+ _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
+ }
+
+ return 1;
+}
+
+/* copy of hidden zend_closure */
+typedef struct _zend_closure {
+ zend_object std;
+ zend_function func;
+ zval this_ptr;
+ zend_class_entry *called_scope;
+ zif_handler orig_internal_handler;
+} zend_closure;
+
+static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ // JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack)
+ ir_GUARD(
+ ir_UGE(
+ ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))),
+ ir_CONST_ADDR(used_stack)),
+ ir_CONST_ADDR(exit_addr));
+
+ return 1;
+}
+
+static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func)
+{
+ // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
+ ir_ref if_trampoline = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
+ ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
+
+ ir_IF_TRUE(if_trampoline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func);
+ ir_MERGE_WITH_EMPTY_FALSE(if_trampoline);
+
+ return 1;
+}
+
+static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref)
+{
+ uint32_t used_stack;
+ ir_ref used_stack_ref = IR_UNUSED;
+ bool stack_check = 1;
+ ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED;
+
+ ZEND_ASSERT(func_ref != IR_NULL);
+ if (func) {
+ used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
+ if ((int)used_stack <= checked_stack) {
+ stack_check = 0;
+ }
+ used_stack_ref = ir_CONST_ADDR(used_stack);
+ } else {
+ ir_ref num_args_ref;
+ ir_ref if_internal_func = IR_UNUSED;
+ const size_t func_type_offset = is_closure ? offsetof(zend_closure, func.type) : offsetof(zend_function, type);
+
+ used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
+ used_stack_ref = ir_CONST_ADDR(used_stack);
+ used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */
+
+ // JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) {
+ ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, func_type_offset));
+ if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1)));
+ ir_IF_FALSE(if_internal_func);
+
+ // JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
+ num_args_ref = ir_CONST_U32(opline->extended_value);
+ if (!is_closure) {
+ ref = ir_SUB_U32(
+ ir_SUB_U32(
+ ir_MIN_U32(
+ num_args_ref,
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))),
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))),
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T))));
+ } else {
+ ref = ir_SUB_U32(
+ ir_SUB_U32(
+ ir_MIN_U32(
+ num_args_ref,
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))),
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))),
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T))));
+ }
+ ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval)));
+ if (sizeof(void*) == 8) {
+ ref = ir_SEXT_A(ref);
+ }
+ ref = ir_SUB_A(used_stack_ref, ref);
+
+ ir_MERGE_WITH_EMPTY_TRUE(if_internal_func);
+ used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref);
+ }
+
+ zend_jit_start_reuse_ip(jit);
+
+ // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
+
+ if (stack_check) {
+ // JIT: Check Stack Overflow
+ ref = ir_UGE(
+ ir_SUB_A(
+ ir_LOAD_A(jit_EG(vm_stack_end)),
+ jit_IP(jit)),
+ used_stack_ref);
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ if_enough_stack = ir_IF(ref);
+ ir_IF_FALSE_cold(if_enough_stack);
+
+#ifdef _WIN32
+ if (0) {
+#else
+ if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref);
+ } else {
+ if (!is_closure) {
+ ref = func_ref;
+ } else {
+ ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func));
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper),
+ used_stack_ref, ref);
+ }
+ jit_STORE_IP(jit, ref);
+
+ cold_path = ir_END();
+ ir_IF_TRUE(if_enough_stack);
+ }
+ }
+
+ ref = jit_EG(vm_stack_top);
+ rx = jit_IP(jit);
+#if !OPTIMIZE_FOR_SIZE
+ /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack);
+ * This vesions is longer but faster
+ * mov EG(vm_stack_top), %CALL
+ * lea size(%call), %tmp
+ * mov %tmp, EG(vm_stack_top)
+ */
+ top = rx;
+#else
+ /* JIT: EG(vm_stack_top) += used_stack;
+ * Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion
+ * mov EG(vm_stack_top), %CALL
+ * add $size, EG(vm_stack_top)
+ */
+ top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
+#endif
+ ir_STORE(ref, ir_ADD_A(top, used_stack_ref));
+
+ // JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
+ // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
+ ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION));
+ }
+#ifdef _WIN32
+ if (0) {
+#else
+ if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ if (cold_path) {
+ ir_MERGE_WITH(cold_path);
+ rx = jit_IP(jit);
+ }
+
+ // JIT: call->func = func;
+ ir_STORE(jit_CALL(rx, func), func_ref);
+ } else {
+ if (!is_closure) {
+ // JIT: call->func = func;
+ ir_STORE(jit_CALL(rx, func), func_ref);
+ } else {
+ // JIT: call->func = &closure->func;
+ ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
+ }
+ if (cold_path) {
+ ir_MERGE_WITH(cold_path);
+ rx = jit_IP(jit);
+ }
+ }
+ if (opline->opcode == ZEND_INIT_METHOD_CALL) {
+ // JIT: Z_PTR(call->This) = obj;
+ ZEND_ASSERT(this_ref != IR_NULL);
+ ir_STORE(jit_CALL(rx, This.value.ptr), this_ref);
+ if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
+ // JIT: call->call_info |= ZEND_CALL_HAS_THIS;
+ ref = jit_CALL(rx, This.u1.type_info);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS));
+ } else {
+ ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS)));
+ }
+ } else {
+ if (opline->op1_type == IS_CV) {
+ // JIT: GC_ADDREF(obj);
+ jit_GC_ADDREF(jit, this_ref);
+ }
+
+ // JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
+ ref = jit_CALL(rx, This.u1.type_info);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS));
+ } else {
+ ir_STORE(ref,
+ ir_OR_U32(ir_LOAD_U32(ref),
+ ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)));
+ }
+ }
+ } else if (!is_closure) {
+ // JIT: Z_CE(call->This) = called_scope;
+ ir_STORE(jit_CALL(rx, This), IR_NULL);
+ } else {
+ ir_ref object_or_called_scope, call_info, call_info2, object, if_cond;
+ ir_ref if_cond_user = IR_UNUSED;
+
+ if (opline->op2_type == IS_CV) {
+ // JIT: GC_ADDREF(closure);
+ jit_GC_ADDREF(jit, func_ref);
+ }
+
+ // JIT: RX(object_or_called_scope) = closure->called_scope;
+ object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope)));
+
+ // JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
+ // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
+ call_info = ir_OR_U32(
+ ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))),
+ ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)),
+ ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE));
+ // JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
+ if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type))));
+ ir_IF_TRUE(if_cond);
+
+ // JIT: call_info |= ZEND_CALL_HAS_THIS;
+ call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS));
+
+ // JIT: object_or_called_scope = Z_OBJ(closure->this_ptr);
+ object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr)));
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_cond);
+ call_info = ir_PHI_2(IR_U32, call_info2, call_info);
+ object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope);
+
+ // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
+ ref = jit_CALL(rx, This.u1.type_info);
+ ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info));
+
+ // JIT: Z_PTR(call->This) = object_or_called_scope;
+ ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope);
+
+ if (!func) {
+ // JIT: if (closure->func.common.type & ZEND_USER_FUNCTION)
+ ir_ref type = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.type)));
+ if_cond_user = ir_IF(ir_AND_U8(type, ir_CONST_U8(ZEND_USER_FUNCTION)));
+ ir_IF_TRUE(if_cond_user);
+ }
+
+ if (!func || func->common.type == ZEND_USER_FUNCTION) {
+ // JIT: zend_jit_init_func_run_time_cache_helper(closure->func);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper),
+ ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
+ }
+
+ if (!func) {
+ ir_MERGE_WITH_EMPTY_FALSE(if_cond_user);
+ }
+ }
+
+ // JIT: ZEND_CALL_NUM_ARGS(call) = num_args;
+ ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value));
+
+ return 1;
+}
+
+static int zend_jit_func_guard(zend_jit_ctx *jit, ir_ref func_ref, const zend_function *func, const void *exit_addr)
+{
+ if (func->type == ZEND_USER_FUNCTION &&
+ (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
+ (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
+ !func->common.function_name)) {
+ const zend_op *opcodes = func->op_array.opcodes;
+
+ // JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr;
+ ir_GUARD(
+ ir_EQ(
+ ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
+ ir_CONST_ADDR(opcodes)),
+ ir_CONST_ADDR(exit_addr));
+#ifdef ZEND_WIN32
+ } else if (func->type == ZEND_INTERNAL_FUNCTION) {
+ // ASLR may cause different addresses in different workers. Check for the internal function handler.
+ // JIT: if (call->func.internal_function.handler != handler) goto exit_addr;
+ ir_GUARD(
+ ir_EQ(
+ ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler))),
+ ir_CONST_FC_FUNC(func->internal_function.handler)),
+ ir_CONST_ADDR(exit_addr));
+#endif
+ } else {
+ // JIT: if (call->func != func) goto exit_addr;
+ ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
+ }
+
+ return 1;
+}
+
+static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline)
+{
+ int32_t exit_point;
+ const void *exit_addr;
+ ir_ref call;
+
+ if (func->type == ZEND_USER_FUNCTION
+ && !zend_accel_in_shm(func->op_array.opcodes)) {
+ /* op_array and op_array->opcodes are not persistent. We can't link. */
+ return 0;
+ }
+
+ exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ // call = EX(call);
+ call = ir_LOAD_A(jit_EX(call));
+ while (level > 0) {
+ // call = call->prev_execute_data
+ call = ir_LOAD_A(jit_CALL(call, prev_execute_data));
+ level--;
+ }
+
+ return zend_jit_func_guard(jit, ir_LOAD_A(jit_CALL(call, func)), func, exit_addr);
+}
+
+static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
+{
+ zend_func_info *info = ZEND_FUNC_INFO(op_array);
+ zend_call_info *call_info = NULL;
+ zend_function *func = NULL;
+ ir_ref func_ref = IR_UNUSED;
+
+ if (jit->delayed_call_level) {
+ if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (info) {
+ call_info = info->callee_info;
+ while (call_info && call_info->caller_init_opline != opline) {
+ call_info = call_info->next_callee;
+ }
+ if (call_info && call_info->callee_func && !call_info->is_prototype) {
+ func = call_info->callee_func;
+ }
+ }
+
+ if (!func
+ && trace
+ && trace->op == ZEND_JIT_TRACE_INIT_CALL) {
+#ifdef _WIN32
+ /* ASLR */
+ if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
+ func = (zend_function*)trace->func;
+ }
+#else
+ func = (zend_function*)trace->func;
+#endif
+ }
+
+#ifdef _WIN32
+ if (0) {
+#else
+ if (opline->opcode == ZEND_INIT_FCALL
+ && func
+ && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ /* load constant address later */
+ func_ref = ir_CONST_ADDR(func);
+ } else if (func && op_array == &func->op_array) {
+ /* recursive call */
+ if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
+ || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
+ func_ref = ir_LOAD_A(jit_EX(func));
+ } else {
+ func_ref = ir_CONST_ADDR(func);
+ }
+ } else {
+ ir_ref if_func, cache_slot_ref, ref;
+
+ // JIT: if (CACHED_PTR(opline->result.num))
+ cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num);
+ func_ref = ir_LOAD_A(cache_slot_ref);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && func
+ && (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
+ && opline->opcode != ZEND_INIT_FCALL) {
+ /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
+ if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func)));
+ } else {
+ if_func = ir_IF(func_ref);
+ }
+ ir_IF_FALSE_cold(if_func);
+ if (opline->opcode == ZEND_INIT_FCALL
+ && func
+ && func->type == ZEND_USER_FUNCTION
+ && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
+ ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */
+ ir_STORE(cache_slot_ref, ref);
+ ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref);
+ } else {
+ zval *zv = RT_CONSTANT(opline, opline->op2);
+
+ if (opline->opcode == ZEND_INIT_FCALL) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
+ ir_CONST_ADDR(Z_STR_P(zv)),
+ cache_slot_ref);
+ } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
+ ir_CONST_ADDR(Z_STR_P(zv + 1)),
+ cache_slot_ref);
+ } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper),
+ ir_CONST_ADDR(zv),
+ cache_slot_ref);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline,
+ func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ if (!func || opline->opcode == ZEND_INIT_FCALL) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else if (!zend_jit_func_guard(jit, ref, func, exit_addr)) {
+ return 0;
+ }
+ } else {
+jit_SET_EX_OPLINE(jit, opline);
+ ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
+ }
+ }
+ ir_MERGE_WITH_EMPTY_TRUE(if_func);
+ func_ref = ir_PHI_2(IR_ADDR, ref, func_ref);
+ }
+
+ if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) {
+ return 0;
+ }
+
+ if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
+ if (!zend_jit_save_call_chain(jit, call_level)) {
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(call_level > 0);
+ jit->delayed_call_level = call_level;
+ delayed_call_chain = 1;
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
+ if (!zend_jit_set_ip(jit, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_init_method_call(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t b,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ int call_level,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ zend_class_entry *ce,
+ bool ce_is_instanceof,
+ bool on_this,
+ bool delayed_fetch_this,
+ zend_class_entry *trace_ce,
+ zend_jit_trace_rec *trace,
+ int checked_stack,
+ ir_ref func_ref,
+ ir_ref this_ref,
+ bool polymorphic_side_trace)
+{
+ zend_func_info *info = ZEND_FUNC_INFO(op_array);
+ zend_call_info *call_info = NULL;
+ zend_function *func = NULL;
+ zval *function_name;
+ ir_ref if_static = IR_UNUSED, cold_path;
+
+ ZEND_ASSERT(opline->op2_type == IS_CONST);
+ ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
+
+ function_name = RT_CONSTANT(opline, opline->op2);
+
+ if (info) {
+ call_info = info->callee_info;
+ while (call_info && call_info->caller_init_opline != opline) {
+ call_info = call_info->next_callee;
+ }
+ if (call_info && call_info->callee_func && !call_info->is_prototype) {
+ func = call_info->callee_func;
+ }
+ }
+
+ if (polymorphic_side_trace) {
+ /* function is passed from parent snapshot */
+ ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED);
+ } else {
+ ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
+
+ if (on_this) {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ this_ref = jit_Z_PTR(jit, this_addr);
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ if (opline->op1_type == IS_CV) {
+ // JIT: ZVAL_DEREF(op1)
+ ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else {
+ ir_ref if_ref;
+
+ /* Hack: Convert reference to regular value to simplify JIT code */
+ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
+
+ if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_TRUE(if_ref);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_ref);
+ }
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
+ ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+
+ ir_IF_FALSE_cold(if_object);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp),
+ jit_ZVAL_ADDR(jit, op1_addr));
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call),
+ jit_ZVAL_ADDR(jit, op1_addr));
+ }
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_IF_TRUE(if_object);
+ }
+ }
+
+ this_ref = jit_Z_PTR(jit, op1_addr);
+ }
+
+ if (jit->delayed_call_level) {
+ if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (func) {
+ // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
+
+ if_found = ir_IF(ref);
+ ir_IF_TRUE(if_found);
+ fast_path = ir_END();
+ } else {
+ // JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) {
+ run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ref = ir_EQ(
+ ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)),
+ ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce))));
+ if_found = ir_IF(ref);
+ ir_IF_TRUE(if_found);
+
+ // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*)));
+ fast_path = ir_END();
+
+ }
+
+ ir_IF_FALSE_cold(if_found);
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (!jit->ctx.fixed_call_stack_size) {
+ // JIT: alloca(sizeof(void*));
+ this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10));
+ } else {
+#ifdef _WIN64
+ this_ref2 = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
+#else
+ this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
+#endif
+ }
+ ir_STORE(this_ref2, this_ref);
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
+ ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper),
+ this_ref,
+ ir_CONST_ADDR(function_name),
+ this_ref2);
+ } else {
+ ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper),
+ this_ref,
+ ir_CONST_ADDR(function_name),
+ this_ref2);
+ }
+
+
+ if (!jit->ctx.fixed_call_stack_size) {
+ this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
+ // JIT: revert alloca
+ ir_AFREE(ir_CONST_ADDR(0x10));
+ } else {
+#ifdef _WIN64
+ this_ref2 = ir_LOAD_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
+#else
+ this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
+#endif
+ }
+
+ ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ ir_MERGE_WITH(fast_path);
+ func_ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref);
+ }
+
+ if ((!func || zend_jit_may_be_modified(func, op_array))
+ && trace
+ && trace->op == ZEND_JIT_TRACE_INIT_CALL
+ && trace->func) {
+ int32_t exit_point;
+ const void *exit_addr;
+
+ exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
+ jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
+
+ func = (zend_function*)trace->func;
+
+ if (!zend_jit_func_guard(jit, func_ref, func, exit_addr)) {
+ return 0;
+ }
+ }
+
+ if (!func) {
+ // JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
+ if_static = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
+ ir_CONST_U32(ZEND_ACC_STATIC)));
+ ir_IF_TRUE_cold(if_static);
+ }
+
+ if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
+ ir_ref ret;
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
+ ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame_tmp),
+ this_ref,
+ func_ref,
+ ir_CONST_U32(opline->extended_value));
+ } else {
+ ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame),
+ this_ref,
+ func_ref,
+ ir_CONST_U32(opline->extended_value));
+ }
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
+ ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ }
+ jit_STORE_IP(jit, ret);
+ }
+
+ if (!func) {
+ cold_path = ir_END();
+ ir_IF_FALSE(if_static);
+ }
+
+ if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
+ if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) {
+ return 0;
+ }
+ }
+
+ if (!func) {
+ ir_MERGE_WITH(cold_path);
+ }
+ zend_jit_start_reuse_ip(jit);
+
+ if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
+ if (!zend_jit_save_call_chain(jit, call_level)) {
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(call_level > 0);
+ delayed_call_chain = 1;
+ jit->delayed_call_level = call_level;
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
+ if (!zend_jit_set_ip(jit, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_init_closure_call(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t b,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ int call_level,
+ zend_jit_trace_rec *trace,
+ int checked_stack)
+{
+ zend_function *func = NULL;
+ zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ ir_ref ref;
+
+ ref = jit_Z_PTR(jit, op2_addr);
+
+ if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
+ && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ir_GUARD(
+ ir_EQ(
+ ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))),
+ ir_CONST_ADDR(zend_ce_closure)),
+ ir_CONST_ADDR(exit_addr));
+
+ if (ssa->var_info && ssa_op->op2_use >= 0) {
+ ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
+ ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
+ }
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_INIT_CALL
+ && trace->func
+ && trace->func->type == ZEND_USER_FUNCTION) {
+ const zend_op *opcodes;
+ int32_t exit_point;
+ const void *exit_addr;
+
+ func = (zend_function*)trace->func;
+ opcodes = func->op_array.opcodes;
+ exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ir_GUARD(
+ ir_EQ(
+ ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))),
+ ir_CONST_ADDR(opcodes)),
+ ir_CONST_ADDR(exit_addr));
+ }
+
+ if (jit->delayed_call_level) {
+ if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
+ return 0;
+ }
+
+ if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
+ if (!zend_jit_save_call_chain(jit, call_level)) {
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(call_level > 0);
+ delayed_call_chain = 1;
+ jit->delayed_call_level = call_level;
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
+ if (!zend_jit_set_ip(jit, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
+{
+ uint32_t arg_num = opline->op2.num;
+ zend_jit_addr arg_addr;
+
+ ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
+
+ if (!zend_jit_reuse_ip(jit)) {
+ return 0;
+ }
+
+ if (opline->opcode == ZEND_SEND_VAL_EX) {
+ uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
+
+ ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ /* Don't generate code that always throws exception */
+ return 0;
+ }
+ } else {
+ ir_ref cond = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
+ ir_CONST_U32(mask));
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_pass_by_ref;
+
+ if_pass_by_ref = ir_IF(cond);
+
+ ir_IF_TRUE_cold(if_pass_by_ref);
+ if (Z_MODE(op1_addr) == IS_REG) {
+ /* set type to avoid zval_ptr_dtor() on uninitialized value */
+ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
+
+ ir_IF_FALSE(if_pass_by_ref);
+ }
+ }
+ }
+
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+
+ jit_ZVAL_COPY_CONST(jit,
+ arg_addr,
+ MAY_BE_ANY, MAY_BE_ANY,
+ zv, 1);
+ } else {
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+ }
+
+ return 1;
+}
+
+static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
+{
+ zend_jit_addr op1_addr, arg_addr, ref_addr;
+ ir_ref ref_path = IR_UNUSED;
+
+ op1_addr = OP1_ADDR();
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+
+ if (!zend_jit_reuse_ip(jit)) {
+ return 0;
+ }
+
+ if (opline->op1_type == IS_VAR) {
+ if (op1_info & MAY_BE_INDIRECT) {
+ op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
+ }
+ } else if (opline->op1_type == IS_CV) {
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
+ ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
+ ir_IF_FALSE(if_def);
+ // JIT: ZVAL_NULL(op1)
+ jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
+ ir_MERGE_WITH_EMPTY_TRUE(if_def);
+ }
+ op1_info &= ~MAY_BE_UNDEF;
+ op1_info |= MAY_BE_NULL;
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
+ ir_ref ref, ref2;
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref if_ref;
+
+ // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
+ if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_TRUE(if_ref);
+ // JIT: ref = Z_PTR_P(op1)
+ ref = jit_Z_PTR(jit, op1_addr);
+ // JIT: GC_ADDREF(ref)
+ jit_GC_ADDREF(jit, ref);
+ // JIT: ZVAL_REFERENCE(arg, ref)
+ jit_set_Z_PTR(jit, arg_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
+ ref_path = ir_END();
+ ir_IF_FALSE(if_ref);
+ }
+
+ // JIT: ZVAL_NEW_REF(arg, varptr);
+ // JIT: ref = emalloc(sizeof(zend_reference));
+ ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
+ // JIT: GC_REFCOUNT(ref) = 2
+ jit_set_GC_REFCOUNT(jit, ref, 2);
+ // JIT: GC_TYPE(ref) = GC_REFERENCE
+ ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE));
+ ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL);
+ ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
+
+ // JIT: ZVAL_COPY_VALUE(&ref->val, op1)
+ jit_ZVAL_COPY(jit,
+ ref_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+
+ // JIT: ZVAL_REFERENCE(arg, ref)
+ jit_set_Z_PTR(jit, op1_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
+
+ // JIT: ZVAL_REFERENCE(arg, ref)
+ jit_set_Z_PTR(jit, arg_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
+ }
+
+ if (ref_path) {
+ ir_MERGE_WITH(ref_path);
+ }
+
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+
+ return 1;
+}
+
+static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
+{
+ uint32_t arg_num = opline->op2.num;
+ zend_jit_addr arg_addr;
+ ir_ref end_inputs = IR_UNUSED;
+
+ ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
+ opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
+ arg_num <= MAX_ARG_FLAG_NUM);
+
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+
+ if (!zend_jit_reuse_ip(jit)) {
+ return 0;
+ }
+
+ if (opline->opcode == ZEND_SEND_VAR_EX) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
+ return 0;
+ }
+ return 1;
+ }
+ } else {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+
+ // JIT: if (RX->func->quick_arg_flags & mask)
+ ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
+ ir_CONST_U32(mask)));
+ ir_IF_TRUE_cold(if_send_by_ref);
+
+ if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
+ return 0;
+ }
+
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_send_by_ref);
+ }
+ } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+
+ // JIT: ZVAL_COPY_VALUE(arg, op1)
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+
+ if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!(op1_info & MAY_BE_REF)) {
+ /* Don't generate code that always throws exception */
+ return 0;
+ } else {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ // JIT: if (Z_TYPE_P(op1) != IS_REFERENCE)
+ ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
+ ir_CONST_ADDR(exit_addr));
+ }
+ }
+ return 1;
+ }
+ } else {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+ ir_ref func, if_send_by_ref, if_prefer_ref;
+
+ // JIT: if (RX->func->quick_arg_flags & mask)
+ func = ir_LOAD_A(jit_RX(func));
+ if_send_by_ref = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
+ ir_CONST_U32(mask)));
+ ir_IF_TRUE_cold(if_send_by_ref);
+
+ mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
+
+ // JIT: ZVAL_COPY_VALUE(arg, op1)
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
+ ir_IF_TRUE(if_ref);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_ref);
+ }
+
+ // JIT: if (RX->func->quick_arg_flags & mask)
+ if_prefer_ref = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
+ ir_CONST_U32(mask)));
+ ir_IF_TRUE(if_prefer_ref);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_prefer_ref);
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
+ jit_ZVAL_ADDR(jit, arg_addr));
+ zend_jit_check_exception(jit);
+ ir_END_list(end_inputs);
+ }
+
+ ir_IF_FALSE(if_send_by_ref);
+ }
+ } else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
+ return 0;
+ }
+ return 1;
+ }
+ } else {
+ // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF)
+ ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(jit_RX(This.u1.type_info)),
+ ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
+ ir_IF_TRUE_cold(if_send_by_ref);
+
+ if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
+ return 0;
+ }
+
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_send_by_ref);
+ }
+ }
+
+ if (op1_info & MAY_BE_UNDEF) {
+ ir_ref ref, if_def = IR_UNUSED;
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
+ ir_IF_FALSE_cold(if_def);
+ }
+
+ // JIT: zend_jit_undefined_op_helper(opline->op1.var)
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
+ ir_CONST_U32(opline->op1.var));
+
+ // JIT: ZVAL_NULL(arg)
+ jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
+
+ // JIT: check_exception
+ ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_def);
+ } else {
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+ return 1;
+ }
+ }
+
+ if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
+ // JIT: ZVAL_COPY_VALUE(arg, op1)
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+ if (op1_info & MAY_BE_REF) {
+ // JIT: if (Z_TYPE_P(arg) == IS_REFERENCE)
+ ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
+ ir_IF_TRUE(if_ref);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_ref);
+ }
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr));
+ } else {
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
+ jit_ZVAL_ADDR(jit, arg_addr));
+ zend_jit_check_exception(jit);
+ }
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ if (opline->op1_type == IS_CV) {
+ ir_ref ref;
+
+ // JIT: ZVAL_DEREF(op1)
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ // JIT: ZVAL_COPY(arg, op1)
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 1);
+ } else {
+ ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted;
+ zend_jit_addr ref_addr;
+
+ // JIT: if (Z_TYPE_P(op1) == IS_REFERENCE)
+ if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_TRUE_cold(if_ref);
+
+ // JIT: ref = Z_COUNTED_P(op1);
+ ref = jit_Z_PTR(jit, op1_addr);
+ ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
+
+ // JIT: ZVAL_COPY_VALUE(arg, op1);
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ ref_addr, op1_info, 0);
+
+ // JIT: if (GC_DELREF(ref) != 0)
+ refcount = jit_GC_DELREF(jit, ref);
+ if_not_zero = ir_IF(refcount);
+ ir_IF_TRUE(if_not_zero);
+
+ // JIT: if (Z_REFCOUNTED_P(arg)
+ if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
+ ir_IF_TRUE(if_refcounted);
+ // JIT: Z_ADDREF_P(arg)
+ jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_refcounted);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE(if_not_zero);
+
+ // JIT: efree(ref)
+ jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE(if_ref);
+
+ // JIT: ZVAL_COPY_VALUE(arg, op1);
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, 0);
+ }
+ } else {
+ if (op1_addr != op1_def_addr) {
+ if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
+ return 0;
+ }
+ if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
+ op1_addr = op1_def_addr;
+ }
+ }
+
+ // JIT: ZVAL_COPY_VALUE(arg, op1)
+ jit_ZVAL_COPY(jit,
+ arg_addr,
+ MAY_BE_ANY,
+ op1_addr, op1_info, opline->op1_type == IS_CV);
+ }
+ }
+
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline)
+{
+ uint32_t arg_num = opline->op2.num;
+ ir_ref ref;
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
+ TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
+ // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
+ if (jit->reuse_ip) {
+ ref = jit_IP(jit);
+ } else {
+ ref = ir_LOAD_A(jit_EX(call));
+ }
+ ref = jit_CALL(ref, This.u1.type_info);
+ ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
+ }
+ } else {
+ if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
+ TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
+ // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
+ if (jit->reuse_ip) {
+ ref = jit_IP(jit);
+ } else {
+ ref = ir_LOAD_A(jit_EX(call));
+ }
+ ref = jit_CALL(ref, This.u1.type_info);
+ ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
+ }
+ }
+ } else {
+ // JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+ ir_ref rx, if_ref, cold_path;
+
+ if (!zend_jit_reuse_ip(jit)) {
+ return 0;
+ }
+
+ rx = jit_IP(jit);
+
+ ref = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))),
+ ir_CONST_U32(mask));
+ if_ref = ir_IF(ref);
+ ir_IF_TRUE_cold(if_ref);
+
+ // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
+ ref = jit_CALL(rx, This.u1.type_info);
+ ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
+
+ cold_path = ir_END();
+ ir_IF_FALSE(if_ref);
+
+ // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
+ ref = jit_CALL(rx, This.u1.type_info);
+ ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
+
+ ir_MERGE_WITH(cold_path);
+ }
+
+ return 1;
+}
+
+static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline)
+{
+ ir_ref call, if_may_have_undef, ret;
+
+ if (jit->reuse_ip) {
+ call = jit_IP(jit);
+ } else {
+ call = ir_LOAD_A(jit_EX(call));
+ }
+
+ if_may_have_undef = ir_IF(ir_AND_U8(
+ ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)),
+ ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24)));
+
+ ir_IF_TRUE_cold(if_may_have_undef);
+ jit_SET_EX_OPLINE(jit, opline);
+ ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call);
+ ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef);
+
+ return 1;
+}
+
+static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
+{
+ zend_func_info *info = ZEND_FUNC_INFO(op_array);
+ zend_call_info *call_info = NULL;
+ const zend_function *func = NULL;
+ uint32_t i;
+ uint32_t call_num_args = 0;
+ bool unknown_num_args = 0;
+ const void *exit_addr = NULL;
+ const zend_op *prev_opline;
+ ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED;
+
+ prev_opline = opline - 1;
+ while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
+ prev_opline--;
+ }
+ if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
+ prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
+ unknown_num_args = 1;
+ }
+
+ if (info) {
+ call_info = info->callee_info;
+ while (call_info && call_info->caller_call_opline != opline) {
+ call_info = call_info->next_callee;
+ }
+ if (call_info && call_info->callee_func && !call_info->is_prototype) {
+ func = call_info->callee_func;
+ }
+ if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
+ && (!JIT_G(current_frame)
+ || !JIT_G(current_frame)->call
+ || !JIT_G(current_frame)->call->func)) {
+ call_info = NULL; func = NULL; /* megamorphic call from trait */
+ }
+ }
+ if (!func) {
+ /* resolve function at run time */
+ } else if (func->type == ZEND_USER_FUNCTION) {
+ ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
+ call_num_args = call_info->num_args;
+ } else if (func->type == ZEND_INTERNAL_FUNCTION) {
+ ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
+ call_num_args = call_info->num_args;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ if (trace && !func) {
+ if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
+ ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
+#ifndef ZEND_WIN32
+ // TODO: ASLR may cause different addresses in different workers ???
+ func = trace->func;
+ if (JIT_G(current_frame) &&
+ JIT_G(current_frame)->call &&
+ TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
+ call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
+ } else {
+ unknown_num_args = 1;
+ }
+#endif
+ } else if (trace->op == ZEND_JIT_TRACE_ENTER) {
+ ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
+ if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
+ func = trace->func;
+ if (JIT_G(current_frame) &&
+ JIT_G(current_frame)->call &&
+ TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
+ call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
+ } else {
+ unknown_num_args = 1;
+ }
+ }
+ }
+ }
+
+ bool may_have_extra_named_params =
+ opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
+ (!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
+
+ if (!jit->reuse_ip) {
+ zend_jit_start_reuse_ip(jit);
+ // JIT: call = EX(call);
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
+ }
+ rx = jit_IP(jit);
+ zend_jit_stop_reuse_ip(jit);
+
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
+ if (!func) {
+ if (trace) {
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ func_ref = ir_LOAD_A(jit_CALL(rx, func));
+ ir_GUARD_NOT(
+ ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
+ ir_CONST_U32(ZEND_ACC_DEPRECATED)),
+ ir_CONST_ADDR(exit_addr));
+ }
+ }
+ }
+
+ if (!jit->delayed_call_level) {
+ // JIT: EX(call) = call->prev_execute_data;
+ ir_STORE(jit_EX(call),
+ (call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data)));
+ }
+ delayed_call_chain = 0;
+ jit->delayed_call_level = 0;
+
+ // JIT: call->prev_execute_data = execute_data;
+ ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
+
+ if (!func) {
+ if (!func_ref) {
+ func_ref = ir_LOAD_A(jit_CALL(rx, func));
+ }
+ }
+
+ if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
+ if (!func) {
+ if (!trace) {
+ ir_ref if_deprecated, ret;
+
+ if_deprecated = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
+ ir_CONST_U32(ZEND_ACC_DEPRECATED)));
+ ir_IF_TRUE_cold(if_deprecated);
+
+ if (GCC_GLOBAL_REGS) {
+ ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
+ } else {
+ ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
+ }
+ ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
+ }
+ } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
+ ir_ref ret;
+
+ if (GCC_GLOBAL_REGS) {
+ ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
+ } else {
+ ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
+ }
+ ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ }
+ }
+
+ if (!func
+ && opline->opcode != ZEND_DO_UCALL
+ && opline->opcode != ZEND_DO_ICALL) {
+ ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
+ if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION)));
+ ir_IF_TRUE(if_user);
+ }
+
+ if ((!func || func->type == ZEND_USER_FUNCTION)
+ && opline->opcode != ZEND_DO_ICALL) {
+ bool recursive_call_through_jmp = 0;
+ uint32_t num_args = 0;
+
+ // JIT: EX(call) = NULL;
+ ir_STORE(jit_CALL(rx, call), IR_NULL);
+
+ // JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0;
+ ir_STORE(jit_CALL(rx, return_value),
+ RETURN_VALUE_USED(opline) ?
+ jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
+ IR_NULL);
+
+ // JIT: EX_LOAD_RUN_TIME_CACHE(op_array);
+ if (!func || func->op_array.cache_size) {
+ ir_ref run_time_cache;
+
+ if (func && op_array == &func->op_array) {
+ /* recursive call */
+ run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ } else if (func
+ && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
+ && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)),
+ (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)));
+ } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
+ (JIT_G(current_frame) &&
+ JIT_G(current_frame)->call &&
+ TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
+ /* Closures always use direct pointers */
+ ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
+
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
+ } else {
+ ir_ref if_odd, run_time_cache2;
+ ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
+
+ run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
+ if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
+ ir_IF_TRUE(if_odd);
+
+ run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_odd);
+ run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache);
+ }
+
+ ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache);
+ }
+
+ // JIT: EG(current_execute_data) = execute_data = call;
+ ir_STORE(jit_EG(current_execute_data), rx);
+ jit_STORE_FP(jit, rx);
+
+ // JIT: opline = op_array->opcodes;
+ if (func && !unknown_num_args) {
+
+ for (i = call_num_args; i < func->op_array.last_var; i++) {
+ uint32_t n = EX_NUM_TO_VAR(i);
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n);
+
+ jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
+ }
+
+ if (call_num_args <= func->op_array.num_args) {
+ if (!trace || (trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
+ if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
+ if (trace) {
+ num_args = 0;
+ } else if (call_info) {
+ num_args = skip_valid_arguments(op_array, ssa, call_info);
+ } else {
+ num_args = call_num_args;
+ }
+ } else {
+ num_args = call_num_args;
+ }
+ if (zend_accel_in_shm(func->op_array.opcodes)) {
+ jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
+ } else {
+ if (!func_ref) {
+ func_ref = ir_LOAD_A(jit_CALL(rx, func));
+ }
+ ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
+ if (num_args) {
+ ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op));
+ }
+ jit_LOAD_IP(jit, ip);
+ }
+
+ if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) {
+ /* recursive call */
+ recursive_call_through_jmp = 1;
+ }
+ }
+ } else {
+ ir_ref helper;
+ if (!trace || (trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
+ ir_ref ip;
+
+ if (zend_accel_in_shm(func->op_array.opcodes)) {
+ ip = ir_CONST_ADDR(func->op_array.opcodes);
+ } else {
+ if (!func_ref) {
+ func_ref = ir_LOAD_A(jit_CALL(rx, func));
+ }
+ ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
+ }
+ jit_LOAD_IP(jit, ip);
+ helper = ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper);
+ } else {
+ helper = ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper_no_skip_recv);
+ }
+ if (GCC_GLOBAL_REGS) {
+ ir_CALL(IR_VOID, helper);
+ } else {
+ ir_CALL_1(IR_VOID, helper, jit_FP(jit));
+ }
+ }
+ } else {
+ ir_ref ip;
+ ir_ref merge_inputs = IR_UNUSED;
+
+ // JIT: opline = op_array->opcodes
+ if (func && zend_accel_in_shm(func->op_array.opcodes)) {
+ ip = ir_CONST_ADDR(func->op_array.opcodes);
+ } else {
+ if (!func_ref) {
+ func_ref = ir_LOAD_A(jit_CALL(rx, func));
+ }
+ ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
+ }
+ jit_LOAD_IP(jit, ip);
+
+ // JIT: num_args = EX_NUM_ARGS();
+ ir_ref num_args, first_extra_arg;
+
+ num_args = ir_LOAD_U32(jit_EX(This.u2.num_args));
+ if (func) {
+ first_extra_arg = ir_CONST_U32(func->op_array.num_args);
+ } else {
+ // JIT: first_extra_arg = op_array->num_args;
+ ZEND_ASSERT(func_ref);
+ first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args)));
+ }
+
+ // JIT: if (UNEXPECTED(num_args > first_extra_arg))
+ ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg));
+ ir_IF_TRUE_cold(if_extra_args);
+ if (GCC_GLOBAL_REGS) {
+ ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
+ }
+ ir_END_list(merge_inputs);
+ ir_IF_FALSE(if_extra_args);
+ if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
+ if (!func) {
+ // JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
+ ir_ref if_has_type_hints = ir_IF(ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
+ ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS)));
+ ir_IF_TRUE(if_has_type_hints);
+ ir_END_list(merge_inputs);
+ ir_IF_FALSE(if_has_type_hints);
+ }
+ // JIT: opline += num_args;
+
+ ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op)));
+
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(ref);
+ }
+
+ if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
+ } else {
+ ir_ref addr = jit_EX(opline);
+
+ ir_STORE(addr, ir_ADD_A(ir_LOAD_A(addr), ref));
+ }
+ }
+
+ ir_END_list(merge_inputs);
+ ir_MERGE_list(merge_inputs);
+
+ // JIT: if (EXPECTED((int)num_args < op_array->last_var)) {
+ ir_ref last_var;
+
+ if (func) {
+ last_var = ir_CONST_U32(func->op_array.last_var);
+ } else {
+ ZEND_ASSERT(func_ref);
+ last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var)));
+ }
+
+ ir_ref idx = ir_SUB_U32(last_var, num_args);
+ ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0)));
+ ir_IF_TRUE(if_need);
+
+ // JIT: zval *var = EX_VAR_NUM(num_args);
+ if (sizeof(void*) == 8) {
+ num_args = ir_ZEXT_A(num_args);
+ }
+ ir_ref var_ref = ir_ADD_OFFSET(
+ ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
+ (ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info));
+
+ ir_ref loop = ir_LOOP_BEGIN(ir_END());
+ var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED);
+ idx = ir_PHI_2(IR_U32, idx, IR_UNUSED);
+ ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF));
+ ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval)));
+ ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1));
+ ir_PHI_SET_OP(idx, 2, idx2);
+ ir_ref if_not_zero = ir_IF(idx2);
+ ir_IF_TRUE(if_not_zero);
+ ir_MERGE_SET_OP(loop, 2, ir_LOOP_END());
+ ir_IF_FALSE(if_not_zero);
+ ir_MERGE_WITH_EMPTY_FALSE(if_need);
+ }
+
+ if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) {
+ ir_ref observer_handler;
+ ir_ref rx = jit_FP(jit);
+ const zend_op *observer_opline = NULL;
+ struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
+ if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
+ ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
+ observer_opline = trace[1].opline;
+ jit_SET_EX_OPLINE(jit, observer_opline);
+ } else if (GCC_GLOBAL_REGS) {
+ // EX(opline) = opline
+ ir_STORE(jit_EX(opline), jit_IP(jit));
+ }
+ jit_observer_fcall_begin(jit, rx, observer_handler);
+
+ zend_jit_check_timeout(jit, observer_opline, NULL);
+
+ jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
+ }
+
+ if (trace) {
+ if (!func && (opline->opcode != ZEND_DO_UCALL)) {
+ user_path = ir_END();
+ }
+ } else {
+ zend_basic_block *bb;
+
+ do {
+ if (recursive_call_through_jmp) {
+ ir_ref begin, end;
+ ir_insn *insn;
+
+ /* attempt to convert direct recursive call into loop */
+ begin = jit->bb_start_ref[num_args];
+ ZEND_ASSERT(begin != IR_UNUSED);
+ insn = &jit->ctx.ir_base[begin];
+ if (insn->op == IR_BEGIN) {
+ end = ir_LOOP_END();
+ insn = &jit->ctx.ir_base[begin];
+ insn->op = IR_LOOP_BEGIN;
+ insn->inputs_count = 2;
+ insn->op2 = end;
+ break;
+ } else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN)
+ && insn->inputs_count == 2) {
+ end = ir_LOOP_END();
+ insn = &jit->ctx.ir_base[begin];
+ insn->op = IR_LOOP_BEGIN;
+ insn->inputs_count = 3;
+ insn->op3 = end;
+ break;
+ }
+ }
+ /* fallback to indirect JMP or RETURN */
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_RETURN(ir_CONST_I32(1));
+ }
+ } while (0);
+
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ if (bb->successors_count > 0) {
+ int succ;
+ ir_ref ref;
+
+ ZEND_ASSERT(bb->successors_count == 1);
+ succ = bb->successors[0];
+ /* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */
+ ref = jit->ctx.insns_count - 1;
+ ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
+ || jit->ctx.ir_base[ref].op == IR_RETURN
+ || jit->ctx.ir_base[ref].op == IR_LOOP_END);
+ ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
+ ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
+ if (func || (opline->opcode == ZEND_DO_UCALL)) {
+ _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
+ jit->b = -1;
+ } else {
+ user_path = ref;
+ }
+ }
+ }
+ }
+
+ if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
+ && (opline->opcode != ZEND_DO_UCALL)) {
+ if (!func && (opline->opcode != ZEND_DO_ICALL)) {
+ ir_IF_FALSE(if_user);
+ }
+
+ // JIT: EG(current_execute_data) = execute_data;
+ ir_STORE(jit_EG(current_execute_data), rx);
+
+ bool may_have_observer = ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
+ if (may_have_observer) {
+ ir_ref observer_handler;
+ struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func)));
+ jit_observer_fcall_begin(jit, rx, observer_handler);
+ jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
+ }
+
+ // JIT: ZVAL_NULL(EX_VAR(opline->result.var));
+ ir_ref res_addr = IR_UNUSED, func_ptr;
+
+ if (RETURN_VALUE_USED(opline)) {
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ } else {
+ /* CPU stack allocated temporary zval */
+ ir_ref ptr;
+
+ if (!jit->ctx.fixed_call_stack_size) {
+ // JIT: alloca(sizeof(void*));
+ ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval)));
+ } else {
+#ifdef _WIN64
+ ptr = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
+#else
+ ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
+#endif
+ }
+ res_addr = ZEND_ADDR_REF_ZVAL(ptr);
+ }
+
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+
+ zend_jit_reset_last_valid_opline(jit);
+
+ // JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
+ ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
+ if (zend_execute_internal) {
+ ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, res_ref);
+ } else {
+ if (func) {
+ func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler);
+ } else {
+ func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler)));
+#if defined(IR_TARGET_X86)
+ func_ptr = ir_CAST_FC_FUNC(func_ptr);
+#endif
+ }
+ ir_CALL_2(IR_VOID, func_ptr, rx, res_ref);
+ }
+
+ if (may_have_observer) {
+ jit_observer_fcall_end(jit, rx, res_ref);
+ }
+
+ /* When zend_interrupt_function is set, it gets called while
+ * the frame is still on top. This is less efficient than
+ * doing it later once it's popped off. There is code further
+ * down that handles when there isn't an interrupt function.
+ */
+ if (zend_interrupt_function) {
+ // JIT: if (EG(vm_interrupt)) zend_fcall_interrupt(execute_data);
+ ir_ref if_interrupt = ir_IF(ir_LOAD_U8(jit_EG(vm_interrupt)));
+ ir_IF_TRUE_cold(if_interrupt);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_fcall_interrupt), rx);
+ ir_MERGE_WITH_EMPTY_FALSE(if_interrupt);
+ }
+
+ // JIT: EG(current_execute_data) = execute_data;
+ ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
+
+ // JIT: zend_vm_stack_free_args(call);
+ if (func && !unknown_num_args) {
+ for (i = 0; i < call_num_args; i++ ) {
+ if (zend_jit_needs_arg_dtor(func, i, call_info)) {
+ uint32_t offset = EX_NUM_TO_VAR(i);
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
+
+ jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
+ }
+ }
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx);
+ }
+
+ if (may_have_extra_named_params) {
+ // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS))
+ ir_ref if_has_named = ir_IF(ir_AND_U8(
+ ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)),
+ ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)));
+ ir_IF_TRUE_cold(if_has_named);
+
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params),
+ ir_LOAD_A(jit_CALL(rx, extra_named_params)));
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_has_named);
+ }
+
+ if (opline->opcode == ZEND_DO_FCALL) {
+ // TODO: optimize ???
+ // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
+ ir_ref if_release_this = ir_IF(ir_AND_U8(
+ ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
+ ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16)));
+ ir_IF_TRUE_cold(if_release_this);
+
+ // JIT: OBJ_RELEASE(Z_OBJ(RX->This));
+ jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
+
+ ir_MERGE_WITH_EMPTY_FALSE(if_release_this);
+ }
+
+
+ ir_ref allocated_path = IR_UNUSED;
+
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ !JIT_G(current_frame)->call ||
+ !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
+ prev_opline->opcode == ZEND_SEND_UNPACK ||
+ prev_opline->opcode == ZEND_SEND_ARRAY ||
+ prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
+
+ // JIT: zend_vm_stack_free_call_frame(call);
+ // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED))
+ ir_ref if_allocated = ir_IF(ir_AND_U8(
+ ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
+ ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16)));
+ ir_IF_TRUE_cold(if_allocated);
+
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx);
+
+ allocated_path = ir_END();
+ ir_IF_FALSE(if_allocated);
+ }
+
+ ir_STORE(jit_EG(vm_stack_top), rx);
+
+ if (allocated_path) {
+ ir_MERGE_WITH(allocated_path);
+ }
+
+ if (!RETURN_VALUE_USED(opline)) {
+ zend_class_entry *ce;
+ bool ce_is_instanceof;
+ uint32_t func_info = call_info ?
+ zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
+ (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
+
+ /* If an exception is thrown, the return_value may stay at the
+ * original value of null. */
+ func_info |= MAY_BE_NULL;
+
+ if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ ir_ref sp;
+ if (!jit->ctx.fixed_call_stack_size) {
+ sp = ir_RLOAD_A(IR_REG_SP);
+ } else {
+#ifdef _WIN64
+ sp = jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS);
+#else
+ sp = ir_RLOAD_A(IR_REG_SP);
+#endif
+ }
+ res_addr = ZEND_ADDR_REF_ZVAL(sp);
+ jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
+ }
+ if (!jit->ctx.fixed_call_stack_size) {
+ // JIT: revert alloca
+ ir_AFREE(ir_CONST_ADDR(sizeof(zval)));
+ }
+ }
+
+ // JIT: if (UNEXPECTED(EG(exception) != NULL)) {
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
+ jit_STUB_ADDR(jit, jit_stub_icall_throw));
+
+ /* If there isn't a zend_interrupt_function, the timeout is
+ * handled here because it's more efficient.
+ */
+ if (!zend_interrupt_function) {
+ // TODO: Can we avoid checking for interrupts after each call ???
+ if (trace && jit->last_valid_opline != opline) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ } else {
+ exit_addr = NULL;
+ }
+
+ zend_jit_check_timeout(jit, opline + 1, exit_addr);
+ }
+
+ if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
+ jit_LOAD_IP_ADDR(jit, opline + 1);
+ } else if (trace
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
+ jit_LOAD_IP_ADDR(jit, opline + 1);
+ }
+ }
+
+ if (user_path) {
+ ir_MERGE_WITH(user_path);
+ }
+
+ return 1;
+}
+
+static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block)
+{
+ ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
+
+ ir_IF_FALSE(if_skip_constructor);
+
+ if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
+ if (!zend_jit_tail_handler(jit, opline)) {
+ return 0;
+ }
+ } else {
+ if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
+ return 0;
+ }
+ }
+
+ /* override predecessors of the next block */
+ ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
+ if (!jit->ctx.control) {
+ ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
+ ir_IF_TRUE(if_skip_constructor);
+ ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
+ jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
+ } else {
+ ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
+ /* merge current control path with the true branch of constructor skip condition */
+ ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor);
+ jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
+
+ jit->b = -1;
+ }
+
+ return 1;
+}
+
+static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
+{
+ zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
+ ir_ref ref, fast_path = IR_UNUSED;
+
+ ref = jit_ZVAL_ADDR(jit, res_addr);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->prev) {
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
+
+ if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
+ return 1;
+ }
+ }
+
+ if (ZEND_ARG_SEND_MODE(arg_info)) {
+ if (opline->opcode == ZEND_RECV_INIT) {
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ } else {
+ ref = jit_Z_PTR_ref(jit, ref);
+ ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ }
+ }
+
+ if (type_mask != 0) {
+ if (is_power_of_two(type_mask)) {
+ uint32_t type_code = concrete_type(type_mask);
+ ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
+ ir_IF_TRUE(if_ok);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_ok);
+ } else {
+ ir_ref if_ok = ir_IF(ir_AND_U32(
+ ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
+ ir_CONST_U32(type_mask)));
+ ir_IF_TRUE(if_ok);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_ok);
+ }
+ }
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow),
+ ref, ir_CONST_ADDR(arg_info));
+
+ if (check_exception) {
+ ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ }
+
+ if (fast_path) {
+ ir_MERGE_WITH(fast_path);
+ }
+
+ return 1;
+}
+
+static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
+{
+ uint32_t arg_num = opline->op1.num;
+ zend_arg_info *arg_info = NULL;
+
+ if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ if (EXPECTED(arg_num <= op_array->num_args)) {
+ arg_info = &op_array->arg_info[arg_num-1];
+ } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
+ arg_info = &op_array->arg_info[op_array->num_args];
+ }
+ if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
+ arg_info = NULL;
+ }
+ }
+
+ if (arg_info || (opline+1)->opcode != ZEND_RECV) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ if (!JIT_G(current_frame) ||
+ TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
+ arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)),
+ ir_CONST_ADDR(exit_addr));
+ }
+ } else {
+ ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
+ ir_IF_FALSE_cold(if_ok);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_IF_TRUE(if_ok);
+ }
+ }
+
+ if (arg_info) {
+ if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
+{
+ uint32_t arg_num = opline->op1.num;
+ zval *zv = RT_CONSTANT(opline, opline->op2);
+ zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ ir_ref ref, if_fail, skip_path = IR_UNUSED;
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
+ if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
+ jit_ZVAL_COPY_CONST(jit,
+ res_addr,
+ -1, -1,
+ zv, 1);
+ }
+ } else {
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
+ ir_IF_TRUE(if_skip);
+ skip_path = ir_END();
+ ir_IF_FALSE(if_skip);
+ }
+ jit_ZVAL_COPY_CONST(jit,
+ res_addr,
+ -1, -1,
+ zv, 1);
+ }
+
+ if (Z_CONSTANT_P(zv)) {
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex),
+ jit_ZVAL_ADDR(jit, res_addr),
+ ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope))));
+
+ if_fail = ir_IF(ref);
+ ir_IF_TRUE_cold(if_fail);
+ jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_IF_FALSE(if_fail);
+ }
+
+ if (skip_path) {
+ ir_MERGE_WITH(skip_path);
+ }
+
+ if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ do {
+ zend_arg_info *arg_info;
+
+ if (arg_num <= op_array->num_args) {
+ arg_info = &op_array->arg_info[arg_num-1];
+ } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
+ arg_info = &op_array->arg_info[op_array->num_args];
+ } else {
+ break;
+ }
+ if (!ZEND_TYPE_IS_SET(arg_info->type)) {
+ break;
+ }
+ if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
+ return 0;
+ }
+ } while (0);
+ }
+
+ return 1;
+}
+
+static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
+{
+ zend_arg_info *arg_info = &op_array->arg_info[-1];
+ ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
+ zend_jit_addr op1_addr = OP1_ADDR();
+ bool needs_slow_check = 1;
+ uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
+ ir_ref fast_path = IR_UNUSED;
+
+ if (type_mask != 0) {
+ if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
+ /* pass */
+ } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
+ needs_slow_check = 0;
+ } else if (is_power_of_two(type_mask)) {
+ uint32_t type_code = concrete_type(type_mask);
+ ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
+
+ ir_IF_TRUE(if_ok);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_ok);
+ } else {
+ ir_ref if_ok = ir_IF(ir_AND_U32(
+ ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
+ ir_CONST_U32(type_mask)));
+
+ ir_IF_TRUE(if_ok);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_ok);
+ }
+ }
+ if (needs_slow_check) {
+ ir_ref ref;
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if (op1_info & MAY_BE_UNDEF) {
+ ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
+ }
+
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow),
+ ref,
+ ir_LOAD_A(jit_EX(func)),
+ ir_CONST_ADDR(arg_info),
+ ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->op2.num));
+
+ zend_jit_check_exception(jit);
+
+ if (fast_path) {
+ ir_MERGE_WITH(fast_path);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_leave_frame(zend_jit_ctx *jit)
+{
+ // JIT: EG(current_execute_data) = EX(prev_execute_data);
+ ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
+ return 1;
+}
+
+static int zend_jit_free_cvs(zend_jit_ctx *jit)
+{
+ // JIT: EG(current_execute_data) = EX(prev_execute_data);
+ ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
+
+ // JIT: zend_free_compiled_variables(execute_data);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
+ return 1;
+}
+
+static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var)
+{
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
+
+ jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
+ }
+ return 1;
+}
+
+static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset)
+{
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
+ }
+ return 1;
+}
+
+static int zend_jit_leave_func(zend_jit_ctx *jit,
+ const zend_op_array *op_array,
+ const zend_op *opline,
+ uint32_t op1_info,
+ bool left_frame,
+ zend_jit_trace_rec *trace,
+ zend_jit_trace_info *trace_info,
+ int indirect_var_access,
+ int may_throw)
+{
+ bool may_be_top_frame =
+ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ !TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
+ bool may_need_call_helper =
+ indirect_var_access || /* may have symbol table */
+ !op_array->function_name || /* may have symbol table */
+ may_be_top_frame ||
+ (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
+ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
+ (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
+ bool may_need_release_this =
+ !(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
+ op_array->scope &&
+ !(op_array->fn_flags & ZEND_ACC_STATIC) &&
+ (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
+ ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED;
+
+ if (may_need_call_helper) {
+ if (!left_frame) {
+ left_frame = 1;
+ if (!zend_jit_leave_frame(jit)) {
+ return 0;
+ }
+ }
+ /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
+ call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
+ ref = ir_AND_U32(call_info,
+ ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE));
+ if (trace && trace->op != ZEND_JIT_TRACE_END) {
+ ir_ref if_slow = ir_IF(ref);
+
+ ir_IF_TRUE_cold(if_slow);
+ if (!GCC_GLOBAL_REGS) {
+ ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
+ } else {
+ ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper));
+ }
+
+ if (may_be_top_frame) {
+ // TODO: try to avoid this check ???
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+#if 0
+ /* this check should be handled by the following OPLINE guard */
+ | cmp IP, zend_jit_halt_op
+ | je ->trace_halt
+#endif
+ } else if (GCC_GLOBAL_REGS) {
+ ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ } else {
+ ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ }
+ }
+
+ if (!GCC_GLOBAL_REGS) {
+ // execute_data = EG(current_execute_data)
+ jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
+ }
+ cold_path = ir_END();
+ ir_IF_FALSE(if_slow);
+ } else {
+ ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
+ }
+ }
+
+ if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) {
+ if (!left_frame) {
+ left_frame = 1;
+ if (!zend_jit_leave_frame(jit)) {
+ return 0;
+ }
+ }
+ // JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
+ jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
+ } else if (may_need_release_this) {
+ ir_ref if_release, fast_path = IR_UNUSED;
+
+ if (!left_frame) {
+ left_frame = 1;
+ if (!zend_jit_leave_frame(jit)) {
+ return 0;
+ }
+ }
+ if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
+ // JIT: if (call_info & ZEND_CALL_RELEASE_THIS)
+ if (!call_info) {
+ call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
+ }
+ if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS)));
+ ir_IF_FALSE(if_release);
+ fast_path = ir_END();
+ ir_IF_TRUE(if_release);
+ }
+ // JIT: OBJ_RELEASE(execute_data->This))
+ jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
+ if (fast_path) {
+ ir_MERGE_WITH(fast_path);
+ }
+ // TODO: avoid EG(excption) check for $this->foo() calls
+ may_throw = 1;
+ }
+
+ // JIT: EG(vm_stack_top) = (zval*)execute_data
+ ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
+
+ // JITL execute_data = EX(prev_execute_data)
+ jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
+
+ if (!left_frame) {
+ // JIT: EG(current_execute_data) = execute_data
+ ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
+ }
+
+ if (trace) {
+ if (trace->op != ZEND_JIT_TRACE_END
+ && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
+ zend_jit_reset_last_valid_opline(jit);
+ } else {
+ if (GCC_GLOBAL_REGS) {
+ /* We add extra RLOAD and RSTORE to make fusion for persistent register
+ * mov (%FP), %IP
+ * add $0x1c, %IP
+ * The naive (commented) code leads to extra register allocation and move.
+ * mov (%FP), %tmp
+ * add $0x1c, %tmp
+ * mov %tmp, %FP
+ */
+#if 0
+ jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
+#else
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
+ jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
+#endif
+ } else {
+ ir_ref ref = jit_EX(opline);
+
+ ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
+ }
+ }
+
+ if (cold_path) {
+ ir_MERGE_WITH(cold_path);
+ }
+
+ if (trace->op == ZEND_JIT_TRACE_BACK
+ && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
+ const zend_op *next_opline = trace->opline;
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && (op1_info & MAY_BE_RC1)
+ && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
+ /* exception might be thrown during destruction of unused return value */
+ // JIT: if (EG(exception))
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
+ }
+ do {
+ trace++;
+ } while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
+ ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
+ next_opline = trace->opline;
+ ZEND_ASSERT(next_opline != NULL);
+
+ if (trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+ trace_info->flags |= ZEND_JIT_TRACE_LOOP;
+
+ ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
+
+ ir_IF_TRUE(if_eq);
+ ZEND_ASSERT(jit->trace_loop_ref);
+ ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
+ ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
+ ir_IF_FALSE(if_eq);
+
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+#else
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
+#endif
+ } else {
+ ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
+ }
+
+ zend_jit_set_last_valid_opline(jit, trace->opline);
+
+ return 1;
+ } else if (may_throw ||
+ (((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && (op1_info & MAY_BE_RC1)
+ && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
+ && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
+ // JIT: if (EG(exception))
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
+ }
+
+ return 1;
+ } else {
+ // JIT: if (EG(exception))
+ ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
+ // JIT: opline = EX(opline) + 1
+ if (GCC_GLOBAL_REGS) {
+ jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
+ jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
+ } else {
+ ir_ref ref = jit_EX(opline);
+
+ ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
+ }
+ }
+
+ if (GCC_GLOBAL_REGS) {
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
+ }
+
+ jit->b = -1;
+
+ return 1;
+}
+
+static void zend_jit_common_return(zend_jit_ctx *jit)
+{
+ ZEND_ASSERT(jit->return_inputs);
+ ir_MERGE_list(jit->return_inputs);
+}
+
+static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
+{
+ zend_jit_addr ret_addr;
+ int8_t return_value_used = -1;
+ ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED;
+
+ ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
+ ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ jit->return_inputs = IR_UNUSED;
+ if (JIT_G(current_frame)) {
+ if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
+ return_value_used = 1;
+ } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
+ return_value_used = 0;
+ } else {
+ return_value_used = -1;
+ }
+ }
+ }
+
+ if (ZEND_OBSERVER_ENABLED) {
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+
+ if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
+ return 0;
+ }
+ op1_addr = dst;
+ }
+ jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr));
+ }
+
+ // JIT: if (!EX(return_value))
+ return_value = ir_LOAD_A(jit_EX(return_value));
+ ret_addr = ZEND_ADDR_REF_ZVAL(return_value);
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if (return_value_used == -1) {
+ if_return_value_used = ir_IF(return_value);
+ ir_IF_FALSE_cold(if_return_value_used);
+ }
+ if (return_value_used != 1) {
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
+ ir_IF_FALSE(if_refcounted);
+ ir_END_list(jit->return_inputs);
+ ir_IF_TRUE(if_refcounted);
+ }
+ ref = jit_Z_PTR(jit, op1_addr);
+ refcount = jit_GC_DELREF(jit, ref);
+
+ if (RC_MAY_BE_1(op1_info)) {
+ if (RC_MAY_BE_N(op1_info)) {
+ ir_ref if_non_zero = ir_IF(refcount);
+ ir_IF_TRUE(if_non_zero);
+ ir_END_list(jit->return_inputs);
+ ir_IF_FALSE(if_non_zero);
+ }
+ jit_ZVAL_DTOR(jit, ref, op1_info, opline);
+ }
+ if (return_value_used == -1) {
+ ir_END_list(jit->return_inputs);
+ }
+ }
+ } else if (return_value_used == -1) {
+ if_return_value_used = ir_IF(return_value);
+ ir_IF_FALSE_cold(if_return_value_used);
+ ir_END_list(jit->return_inputs);
+ }
+
+ if (if_return_value_used) {
+ ir_IF_TRUE(if_return_value_used);
+ }
+
+ if (return_value_used == 0) {
+ if (jit->return_inputs) {
+ ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE);
+ ir_END_list(jit->return_inputs);
+ ir_MERGE_list(jit->return_inputs);
+ jit->return_inputs = IR_UNUSED;
+ }
+ return 1;
+ }
+
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+
+ jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
+ } else if (opline->op1_type == IS_TMP_VAR) {
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
+ } else if (opline->op1_type == IS_CV) {
+ if (op1_info & MAY_BE_REF) {
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
+ (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
+ !op_array->function_name) {
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
+ } else if (return_value_used != 1) {
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
+ // JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
+ } else {
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
+ }
+ } else {
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
+ }
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ ir_ref if_ref, ref2, if_non_zero;
+ zend_jit_addr ref_addr;
+
+ if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_TRUE_cold(if_ref);
+
+ // JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr)
+ ref = jit_Z_PTR(jit, op1_addr);
+
+ // JIT: ZVAL_COPY_VALUE(return_value, &ref->value)
+ ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
+ ref2 = jit_GC_DELREF(jit, ref);
+ if_non_zero = ir_IF(ref2);
+ ir_IF_TRUE(if_non_zero);
+
+ // JIT: if (IS_REFCOUNTED())
+ ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
+ ir_IF_FALSE(if_refcounted);
+ ir_END_list(jit->return_inputs);
+ ir_IF_TRUE(if_refcounted);
+
+ // JIT: ADDREF
+ ref2 = jit_Z_PTR(jit, ret_addr);
+ jit_GC_ADDREF(jit, ref2);
+ ir_END_list(jit->return_inputs);
+
+ ir_IF_FALSE(if_non_zero);
+
+ jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
+ ir_END_list(jit->return_inputs);
+
+ ir_IF_FALSE(if_ref);
+ }
+ jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
+ }
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ if (jit->return_inputs) {
+ ir_END_list(jit->return_inputs);
+ ir_MERGE_list(jit->return_inputs);
+ jit->return_inputs = IR_UNUSED;
+ }
+ } else {
+ ir_END_list(jit->return_inputs);
+ jit->b = -1;
+ }
+
+ return 1;
+}
+
+static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
+{
+ zend_jit_addr op1_addr = OP1_ADDR();
+ zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
+ ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2;
+ ir_ref if_fit, if_reference, if_same_key, fast_path;
+ ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
+
+ // JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1;
+ cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
+ idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1));
+
+ // JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
+ num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)),
+ ir_CONST_U32(sizeof(Bucket)));
+ if (sizeof(void*) == 8) {
+ num_used_ref = ir_ZEXT_A(num_used_ref);
+ }
+ if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref));
+ ir_IF_FALSE_cold(if_fit);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_fit);
+
+ // JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
+ bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref);
+ if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
+ ir_IF_FALSE_cold(if_reference);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_reference);
+
+ // JIT: (EXPECTED(p->key == varname))
+ if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname)));
+ ir_IF_FALSE_cold(if_same_key);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_same_key);
+
+ // JIT: GC_ADDREF(Z_PTR(p->val))
+ ref = jit_Z_PTR_ref(jit, bucket_ref);
+ jit_GC_ADDREF(jit, ref);
+
+ fast_path = ir_END();
+ ir_MERGE_list(slow_inputs);
+
+ ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper),
+ ir_CONST_ADDR(varname),
+ cache_slot_ref);
+
+ ir_MERGE_WITH(fast_path);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak;
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ // JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
+ if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
+ ir_IF_TRUE_cold(if_refcounted);
+ }
+
+ // JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
+ ref2 = jit_Z_PTR(jit, op1_addr);
+
+ // JIT: ZVAL_REF(variable_ptr, ref)
+ jit_set_Z_PTR(jit, op1_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
+
+ // JIT: if (GC_DELREF(garbage) == 0)
+ refcount = jit_GC_DELREF(jit, ref2);
+ if_non_zero = ir_IF(refcount);
+ if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
+ ir_IF_TRUE(if_non_zero);
+ ir_END_list(end_inputs);
+ }
+ ir_IF_FALSE(if_non_zero);
+
+ jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
+ if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_non_zero);
+
+ // JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
+ if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
+ ir_IF_TRUE(if_may_not_leak);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_may_not_leak);
+ if (opline) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2);
+ }
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_refcounted);
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ // JIT: ZVAL_REF(variable_ptr, ref)
+ jit_set_Z_PTR(jit, op1_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
+ }
+
+ if (end_inputs) {
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw)
+{
+ zend_jit_addr op1_addr = OP1_ADDR();
+
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ if (may_throw) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
+ ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED;
+
+ if (op1_info & MAY_BE_ARRAY) {
+ if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
+ ir_IF_TRUE(if_array);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_array);
+ }
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
+ if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1)));
+ ir_IF_TRUE(if_exists);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_exists);
+
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref);
+
+ ir_END_list(end_inputs);
+ ir_MERGE_list(end_inputs);
+ }
+
+ jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
+{
+ if (opline->op1_type == IS_CONST) {
+ zval *zv;
+ size_t len;
+
+ zv = RT_CONSTANT(opline, opline->op1);
+ ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
+ len = Z_STRLEN_P(zv);
+
+ if (len > 0) {
+ const char *str = Z_STRVAL_P(zv);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
+ ir_CONST_ADDR(str), ir_CONST_ADDR(len));
+
+ zend_jit_check_exception(jit);
+ }
+ } else {
+ zend_jit_addr op1_addr = OP1_ADDR();
+ ir_ref ref;
+
+ ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
+
+ jit_SET_EX_OPLINE(jit, opline);
+
+ ref = jit_Z_PTR(jit, op1_addr);
+ ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
+ ir_ADD_OFFSET(ref, offsetof(zend_string, val)),
+ ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len))));
+
+ if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
+ jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
+ }
+
+ zend_jit_check_exception(jit);
+ }
+ return 1;
+}
+
+static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
+{
+ if (opline->op1_type == IS_CONST) {
+ zval *zv;
+ size_t len;
+
+ zv = RT_CONSTANT(opline, opline->op1);
+ ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
+ len = Z_STRLEN_P(zv);
+
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
+ return 0;
+ }
+ } else {
+ ir_ref ref;
+
+ ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
+
+ ref = jit_Z_PTR(jit, op1_addr);
+ ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len)));
+ jit_set_Z_LVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
+ return 0;
+ }
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ return 1;
+}
+
+static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
+{
+ if (opline->op1_type == IS_CONST) {
+ zval *zv;
+ zend_long count;
+
+ zv = RT_CONSTANT(opline, opline->op1);
+ ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
+ count = zend_hash_num_elements(Z_ARRVAL_P(zv));
+
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
+ if (Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
+ return 0;
+ }
+ } else {
+ ir_ref ref;
+
+ ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
+ // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
+
+ ref = jit_Z_PTR(jit, op1_addr);
+ if (sizeof(void*) == 8) {
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
+ ref = ir_ZEXT_L(ref);
+ } else {
+ ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
+ }
+ jit_set_Z_LVAL(jit, res_addr, ref);
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
+ return 0;
+ }
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ return 1;
+}
+
+static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
+{
+ HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
+ zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ ir_ref ref;
+
+ ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
+ ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
+
+ // JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
+ if (opline->op1_type != IS_CONST) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find),
+ ir_CONST_ADDR(ht),
+ jit_Z_PTR(jit, op1_addr));
+ } else {
+ zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
+
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash),
+ ir_CONST_ADDR(ht), ir_CONST_ADDR(str));
+ }
+
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else if (smart_branch_opcode) {
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ ZEND_ASSERT(bb->successors_count == 2);
+ ref = jit_IF_ex(jit, ref,
+ (smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
+ jit->b = -1;
+ } else {
+ jit_set_Z_TYPE_INFO_ex(jit, res_addr,
+ ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE)));
+ }
+
+ return 1;
+}
+
+static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info)
+{
+ uint32_t offset;
+
+ offset = (opline->opcode == ZEND_ROPE_INIT) ?
+ opline->result.var :
+ opline->op1.var + opline->extended_value * sizeof(zend_string*);
+
+ if (opline->op2_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op2);
+ zend_string *str;
+
+ ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
+ str = Z_STR_P(zv);
+
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
+ } else {
+ zend_jit_addr op2_addr = OP2_ADDR();
+ ir_ref ref;
+
+ ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
+
+ ref = jit_Z_PTR(jit, op2_addr);
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
+ if (opline->op2_type == IS_CV) {
+ ir_ref if_refcounted, long_path;
+
+ if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
+ ir_IF_TRUE(if_refcounted);
+ jit_GC_ADDREF(jit, ref);
+ long_path = ir_END();
+
+ ir_IF_FALSE(if_refcounted);
+ ir_MERGE_WITH(long_path);
+ }
+ }
+
+ if (opline->opcode == ZEND_ROPE_END) {
+ zend_jit_addr res_addr = RES_ADDR();
+ ir_ref ref;
+
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end),
+ ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
+ ir_CONST_U32(opline->extended_value));
+
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
+ }
+
+ return 1;
+}
+
+static int zend_jit_zval_copy_deref_reg(zend_jit_ctx *jit, zend_jit_addr res_addr, uint32_t res_info, zend_jit_addr val_addr, ir_ref type, ir_ref *values)
+{
+ ir_ref if_type, val;
+
+ if (res_info == MAY_BE_LONG) {
+ if_type = ir_IF(ir_EQ(type, ir_CONST_U32(IS_LONG)));
+ ir_IF_TRUE(if_type);
+ val = jit_ZVAL_ADDR(jit, val_addr);
+ ir_END_PHI_list(*values, val);
+ ir_IF_FALSE(if_type);
+ val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
+ ir_END_PHI_list(*values, val);
+ } else if (res_info == MAY_BE_DOUBLE) {
+ if_type = ir_IF(ir_EQ(type, ir_CONST_U32(IS_DOUBLE)));
+ ir_IF_TRUE(if_type);
+ val = jit_ZVAL_ADDR(jit, val_addr);
+ ir_END_PHI_list(*values, val);
+ ir_IF_FALSE(if_type);
+ val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
+ ir_END_PHI_list(*values, val);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ return 1;
+}
+
+static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type)
+{
+ ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2;
+ ir_refs *merge_inputs, *types, *ptrs;
+#if SIZEOF_ZEND_LONG == 4
+ ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
+ ir_refs *values; /* we need this only for zval.w2 copy */
+#endif
+
+ ir_refs_init(merge_inputs, 4);
+ ir_refs_init(types, 4);
+ ir_refs_init(ptrs, 4);
+#if SIZEOF_ZEND_LONG == 4
+ ir_refs_init(values, 4);
+#endif
+
+ // JIT: ptr = Z_PTR_P(val);
+ ptr = jit_Z_PTR(jit, val_addr);
+
+ // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
+ if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
+ ir_IF_FALSE_cold(if_refcounted);
+ ir_refs_add(merge_inputs, ir_END());
+ ir_refs_add(types, type);
+ ir_refs_add(ptrs, ptr);
+#if SIZEOF_ZEND_LONG == 4
+ ir_refs_add(values, val);
+#endif
+
+ ir_IF_TRUE(if_refcounted);
+
+ // JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) {
+ if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX)));
+// if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ???
+ ir_IF_TRUE(if_reference);
+
+ // JIT: val = Z_REFVAL_P(val);
+ val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val));
+ type2 = jit_Z_TYPE_INFO_ref(jit, val2);
+ ptr2 = jit_Z_PTR_ref(jit, val2);
+
+ // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
+ if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
+ ir_IF_FALSE_cold(if_refcounted2);
+ ir_refs_add(merge_inputs, ir_END());
+ ir_refs_add(types, type2);
+ ir_refs_add(ptrs, ptr2);
+#if SIZEOF_ZEND_LONG == 4
+ ir_refs_add(values, val2);
+#endif
+
+ ir_IF_TRUE(if_refcounted2);
+ ir_MERGE_WITH_EMPTY_FALSE(if_reference);
+ type = ir_PHI_2(IR_U32, type2, type);
+ ptr = ir_PHI_2(IR_ADDR, ptr2, ptr);
+#if SIZEOF_ZEND_LONG == 4
+ val = ir_PHI_2(IR_ADDR, val2, val);
+#endif
+
+ // JIT: Z_ADDREF_P(val);
+ jit_GC_ADDREF(jit, ptr);
+ ir_refs_add(merge_inputs, ir_END());
+ ir_refs_add(types, type);
+ ir_refs_add(ptrs, ptr);
+#if SIZEOF_ZEND_LONG == 4
+ ir_refs_add(values, val);
+#endif
+
+ ir_MERGE_N(merge_inputs->count, merge_inputs->refs);
+ type = ir_PHI_N(IR_U32, types->count, types->refs);
+ ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs);
+#if SIZEOF_ZEND_LONG == 4
+ val = ir_PHI_N(IR_ADDR, values->count, values->refs);
+ val_addr = ZEND_ADDR_REF_ZVAL(val);
+#endif
+
+ // JIT: Z_PTR_P(res) = ptr;
+ jit_set_Z_PTR(jit, res_addr, ptr);
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
+#endif
+ jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
+
+ return 1;
+}
+
+static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t type,
+ uint32_t op1_info,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ uint8_t dim_type,
+ const void *found_exit_addr,
+ const void *not_found_exit_addr,
+ const void *exit_addr,
+ bool result_type_guard,
+ ir_ref ht_ref,
+ ir_refs *found_inputs,
+ ir_refs *found_vals,
+ ir_ref *end_inputs,
+ ir_ref *not_found_inputs)
+{
+ zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ ir_ref ref = IR_UNUSED, cond, if_found;
+ ir_ref if_type = IS_UNUSED;
+ ir_refs *test_zval_inputs, *test_zval_values;
+
+ ir_refs_init(test_zval_inputs, 4);
+ ir_refs_init(test_zval_values, 4);
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && type == BP_VAR_R
+ && !exit_addr) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ }
+
+ if (op2_info & MAY_BE_LONG) {
+ bool op2_loaded = 0;
+ bool packed_loaded = 0;
+ bool bad_packed_key = 0;
+ ir_ref if_packed = IS_UNDEF;
+ ir_ref h = IR_UNUSED;
+ ir_ref idx_not_found_inputs = IR_UNUSED;
+
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
+ // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
+ if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
+ ir_IF_TRUE(if_type);
+ }
+ if (op1_info & MAY_BE_PACKED_GUARD) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ cond = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
+ ir_CONST_U32(HASH_FLAG_PACKED));
+ if (op1_info & MAY_BE_ARRAY_PACKED) {
+ ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
+ }
+ }
+ if (type == BP_VAR_W) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ op2_loaded = 1;
+ }
+ if (op1_info & MAY_BE_ARRAY_PACKED) {
+ zend_long val = -1;
+
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ val = Z_LVAL_P(Z_ZV(op2_addr));
+ if (val >= 0 && val < HT_MAX_SIZE) {
+ packed_loaded = 1;
+ } else {
+ bad_packed_key = 1;
+ }
+ h = ir_CONST_LONG(val);
+ } else {
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ op2_loaded = 1;
+ }
+ packed_loaded = 1;
+ }
+
+ if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
+ /* don't generate "fast" code for packed array */
+ packed_loaded = 0;
+ }
+
+ if (packed_loaded) {
+ // JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
+ if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
+ if_packed = ir_IF(
+ ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
+ ir_CONST_U32(HASH_FLAG_PACKED)));
+ ir_IF_TRUE(if_packed);
+ }
+ // JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
+ ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)));
+#if SIZEOF_ZEND_LONG == 8
+ if ((Z_MODE(op2_addr) == IS_CONST_ZVAL && val >= 0 && val <= UINT32_MAX)
+ || (op2_range && op2_range->min >= 0 && op2_range->max <= UINT32_MAX)) {
+ /* comapre only the lower 32-bits to allow load fusion on x86_64 */
+ cond = ir_ULT(ir_TRUNC_U32(h), ref);
+ } else {
+ cond = ir_ULT(h, ir_ZEXT_L(ref));
+ }
+#else
+ cond = ir_ULT(h, ref);
+#endif
+ if (type == BP_JIT_IS) {
+ if (not_found_exit_addr) {
+ ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ ir_ref if_fit = ir_IF(cond);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+ } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
+ } else if (type == BP_VAR_IS && not_found_exit_addr) {
+ ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
+ } else if (type == BP_VAR_RW && not_found_exit_addr) {
+ ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
+ } else if (type == BP_VAR_IS && result_type_guard) {
+ ir_ref if_fit = ir_IF(cond);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(*not_found_inputs);
+ ir_IF_TRUE(if_fit);
+ } else {
+ ir_ref if_fit = ir_IF(cond);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(idx_not_found_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+ // JIT: _ret = &_ht->arPacked[h];
+ ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval)));
+ ref = ir_BITCAST_A(ref);
+ ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref);
+ if (type == BP_JIT_IS) {
+ ir_refs_add(test_zval_values, ref);
+ ir_refs_add(test_zval_inputs, ir_END());
+ }
+ }
+ }
+ switch (type) {
+ case BP_JIT_IS:
+ if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
+ if (if_packed) {
+ ir_IF_FALSE(if_packed);
+ if_packed = IR_UNUSED;
+ }
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ op2_loaded = 1;
+ }
+ if (packed_loaded) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
+ }
+ if (not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(test_zval_values, ref);
+ ir_refs_add(test_zval_inputs, ir_END());
+ } else if (!not_found_exit_addr && !packed_loaded) {
+ ir_END_list(*end_inputs);
+ }
+ break;
+ case BP_VAR_R:
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ if (packed_loaded) {
+ ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
+
+ if (result_type_guard) {
+ /* perform IS_UNDEF check only after result type guard (during deoptimization) */
+ } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr));
+ } else if (type == BP_VAR_IS && not_found_exit_addr) {
+ ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ ir_ref if_def = ir_IF(type_ref);
+ ir_IF_FALSE(if_def);
+ ir_END_list(idx_not_found_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ }
+ if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
+ if (if_packed) {
+ ir_IF_FALSE(if_packed);
+ if_packed = IR_UNUSED;
+ }
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ op2_loaded = 1;
+ }
+ if (packed_loaded) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
+ }
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else if (type == BP_VAR_IS && not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else if (type == BP_VAR_IS && result_type_guard) {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*not_found_inputs);
+ ir_IF_TRUE(if_found);
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(idx_not_found_inputs);
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ } else if (!packed_loaded) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else if (type == BP_VAR_IS && not_found_exit_addr) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
+ } else if (type == BP_VAR_IS && result_type_guard) {
+ ir_END_list(*not_found_inputs);
+ } else {
+ ir_END_list(idx_not_found_inputs);
+ }
+ }
+
+ if (idx_not_found_inputs) {
+ ir_MERGE_list(idx_not_found_inputs);
+ switch (type) {
+ case BP_VAR_R:
+ ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
+ // JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
+ // JIT: retval = &EG(uninitialized_zval);
+ jit_SET_EX_OPLINE(jit, opline);
+ if (Z_MODE(op2_addr) == IS_REG) {
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ }
+ if (GCC_GLOBAL_REGS) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h);
+ } else {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h, jit_FP(jit));
+ }
+ } else {
+ ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
+ }
+ ir_END_list(*end_inputs);
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ if (!not_found_exit_addr) {
+ // JIT: retval = &EG(uninitialized_zval);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ ir_END_list(*end_inputs);
+ }
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ }
+ break;
+ case BP_VAR_RW:
+ if (packed_loaded) {
+ if (not_found_exit_addr) {
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ } else {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
+ ir_IF_TRUE(if_def);
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(idx_not_found_inputs);
+ }
+ }
+ if (!packed_loaded ||
+ !not_found_exit_addr ||
+ (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
+ if (if_packed) {
+ ir_IF_FALSE(if_packed);
+ if_packed = IR_UNUSED;
+ ir_END_list(idx_not_found_inputs);
+ } else if (!packed_loaded) {
+ ir_END_list(idx_not_found_inputs);
+ }
+
+ ir_MERGE_list(idx_not_found_inputs);
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ }
+ if (packed_loaded) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed),
+ ht_ref, h);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h);
+ }
+ if (not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ }
+ break;
+ case BP_VAR_W:
+ if (packed_loaded) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
+ ir_IF_TRUE_cold(if_def);
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ ir_IF_FALSE(if_def);
+ ir_END_list(idx_not_found_inputs);
+ }
+ if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
+ if (if_packed) {
+ ir_IF_FALSE(if_packed);
+ if_packed = IR_UNUSED;
+ ir_END_list(idx_not_found_inputs);
+ } else if (!packed_loaded) {
+ ir_END_list(idx_not_found_inputs);
+ }
+ ir_MERGE_list(idx_not_found_inputs);
+ if (!op2_loaded) {
+ // JIT: hval = Z_LVAL_P(dim);
+ h = jit_Z_LVAL(jit, op2_addr);
+ }
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h);
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ }
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ }
+
+ if (op2_info & MAY_BE_STRING) {
+ ir_ref key;
+
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ if_type = IS_UNUSED;
+ }
+
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
+ // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
+ if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
+ ir_IF_TRUE(if_type);
+ }
+
+ // JIT: offset_key = Z_STR_P(dim);
+ key = jit_Z_PTR(jit, op2_addr);
+
+ // JIT: retval = zend_hash_find(ht, offset_key);
+ switch (type) {
+ case BP_JIT_IS:
+ if (opline->op2_type != IS_CONST) {
+ ir_ref if_num, end1, ref2;
+
+ if_num = ir_IF(
+ ir_ULE(
+ ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
+ ir_CONST_CHAR('9')));
+ ir_IF_TRUE_cold(if_num);
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
+ end1 = ir_END();
+ ir_IF_FALSE(if_num);
+ ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
+ ir_MERGE_WITH(end1);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
+ }
+ if (not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(test_zval_values, ref);
+ ir_refs_add(test_zval_inputs, ir_END());
+ break;
+ case BP_VAR_R:
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ if (opline->op2_type != IS_CONST) {
+ ir_ref if_num, end1, ref2;
+
+ if_num = ir_IF(
+ ir_ULE(
+ ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
+ ir_CONST_CHAR('9')));
+ ir_IF_TRUE_cold(if_num);
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
+ end1 = ir_END();
+ ir_IF_FALSE(if_num);
+ ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
+ ir_MERGE_WITH(end1);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
+ }
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else if (type == BP_VAR_IS && not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else if (type == BP_VAR_IS && result_type_guard) {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*not_found_inputs);
+ ir_IF_TRUE(if_found);
+ } else {
+ if_found = ir_IF(ref);
+ switch (type) {
+ case BP_VAR_R:
+ ir_IF_FALSE_cold(if_found);
+ // JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
+ ir_END_list(*end_inputs);
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ ir_IF_FALSE(if_found);
+ // JIT: retval = &EG(uninitialized_zval);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ ir_END_list(*end_inputs);
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ break;
+ case BP_VAR_RW:
+ if (opline->op2_type != IS_CONST) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key);
+ }
+ if (not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_found);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ break;
+ case BP_VAR_W:
+ if (opline->op2_type != IS_CONST) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key);
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key);
+ }
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ }
+
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IS_UNDEF;
+ }
+ if (type != BP_VAR_RW) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ref = jit_ZVAL_ADDR(jit, op2_addr);
+ switch (type) {
+ case BP_VAR_R:
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper),
+ ht_ref,
+ ref,
+ jit_ZVAL_ADDR(jit, res_addr));
+ ir_END_list(*end_inputs);
+ break;
+ case BP_JIT_IS:
+ ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref);
+ if (not_found_exit_addr) {
+ ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
+ ir_refs_add(found_inputs, ir_END());
+ } else if (found_exit_addr) {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr));
+ ir_END_list(*end_inputs);
+ } else {
+ if_found = ir_IF(ref);
+ ir_IF_TRUE(if_found);
+ ir_refs_add(found_inputs, ir_END());
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ }
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper),
+ ht_ref,
+ ref,
+ jit_ZVAL_ADDR(jit, res_addr));
+ ir_END_list(*end_inputs);
+ break;
+ case BP_VAR_RW:
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref);
+ if_found = ir_IF(ref);
+ ir_IF_TRUE(if_found);
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ break;
+ case BP_VAR_W:
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref);
+ if_found = ir_IF(ref);
+ ir_IF_TRUE(if_found);
+ ir_refs_add(found_inputs, ir_END());
+ ir_refs_add(found_vals, ref);
+ ir_IF_FALSE(if_found);
+ ir_END_list(*end_inputs);
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ }
+
+ if (type == BP_JIT_IS
+ && !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+ /* dead code */
+ ir_END_list(*end_inputs);
+ } else if (type == BP_JIT_IS
+ && (op1_info & MAY_BE_ARRAY)
+ && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
+ && test_zval_inputs->count) {
+
+ ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs);
+
+ if (op1_info & MAY_BE_ARRAY_OF_REF) {
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ }
+ cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
+ if (not_found_exit_addr) {
+ ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
+ ir_refs_add(found_inputs, ir_END());
+ } else if (found_exit_addr) {
+ ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr));
+ ir_END_list(*end_inputs);
+ } else {
+ ir_ref if_set = ir_IF(cond);
+ ir_IF_FALSE(if_set);
+ ir_END_list(*end_inputs);
+ ir_IF_TRUE(if_set);
+ ir_refs_add(found_inputs, ir_END());
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_fetch_dim_read(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ bool op1_avoid_refcounting,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ uint32_t res_info,
+ zend_jit_addr res_addr,
+ uint8_t dim_type)
+{
+ zend_jit_addr orig_op1_addr;
+ const void *exit_addr = NULL;
+ const void *not_found_exit_addr = NULL;
+ bool result_type_guard = 0;
+ bool result_avoid_refcounting = 0;
+ uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
+ int may_throw = 0;
+ ir_ref if_type = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref not_found_inputs = IR_UNUSED;
+
+ orig_op1_addr = OP1_ADDR();
+
+ if (opline->opcode != ZEND_FETCH_DIM_IS
+ && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && !has_concrete_type(op1_info)) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ }
+
+ if ((res_info & MAY_BE_GUARD)
+ && JIT_G(current_frame)
+ && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
+
+ if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
+ result_type_guard = 1;
+ res_info &= ~MAY_BE_GUARD;
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+
+ if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
+ && (opline->opcode == ZEND_FETCH_LIST_R
+ || !(opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ || op1_avoid_refcounting)
+ && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
+ && (ssa_op+1)->op1_use == ssa_op->result_def
+ && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
+ && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
+ result_avoid_refcounting = 1;
+ ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
+ }
+
+ if (opline->opcode == ZEND_FETCH_DIM_IS
+ && !(res_info & MAY_BE_NULL)) {
+ uint32_t flags = 0;
+ uint32_t old_op1_info = 0;
+ uint32_t old_info;
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ int32_t exit_point;
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && !op1_avoid_refcounting) {
+ flags |= ZEND_JIT_EXIT_FREE_OP1;
+ }
+ if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
+ && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ flags |= ZEND_JIT_EXIT_FREE_OP2;
+ }
+
+ if (op1_avoid_refcounting) {
+ old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
+ SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
+ }
+
+ old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
+ SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY);
+ exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
+ not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!not_found_exit_addr) {
+ return 0;
+ }
+
+ if (op1_avoid_refcounting) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
+ }
+ }
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ ir_ref ht_ref, ref;
+ zend_jit_addr val_addr;
+ ir_refs *found_inputs, *found_vals;
+
+ ir_refs_init(found_inputs, 10);
+ ir_refs_init(found_vals, 10);
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
+ } else {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
+ ir_IF_TRUE(if_type);
+ }
+ }
+
+ ht_ref = jit_Z_PTR(jit, op1_addr);
+
+ if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
+ (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
+ may_throw = 1;
+ }
+
+ if (!zend_jit_fetch_dimension_address_inner(jit, opline,
+ (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS,
+ op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, exit_addr,
+ result_type_guard, ht_ref, found_inputs, found_vals,
+ &end_inputs, ¬_found_inputs)) {
+ return 0;
+ }
+
+ if (found_inputs->count) {
+ ir_MERGE_N(found_inputs->count, found_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
+ val_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ if (result_type_guard) {
+ uint8_t type = concrete_type(res_info);
+ uint32_t flags = 0;
+
+ if (opline->opcode != ZEND_FETCH_LIST_R
+ && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && !op1_avoid_refcounting) {
+ flags |= ZEND_JIT_EXIT_FREE_OP1;
+ }
+ if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
+ && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ flags |= ZEND_JIT_EXIT_FREE_OP2;
+ }
+
+ val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
+ (op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting);
+ if (!val_addr) {
+ return 0;
+ }
+
+ if (not_found_inputs) {
+ ir_END_list(not_found_inputs);
+ ir_MERGE_list(not_found_inputs);
+ }
+
+ // ZVAL_COPY
+ jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
+ if (Z_MODE(res_addr) != IS_REG) {
+ } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ } else if (op1_info & MAY_BE_ARRAY_OF_REF) {
+ // ZVAL_COPY_DEREF
+ ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
+ if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
+ return 0;
+ }
+ } else {
+ // ZVAL_COPY
+ jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
+ }
+
+ ir_END_list(end_inputs);
+ } else if (not_found_inputs) {
+ ir_MERGE_list(not_found_inputs);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ ir_END_list(end_inputs);
+ } else if (!end_inputs && jit->ctx.control) {
+ ir_END_list(end_inputs); /* dead code */
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IS_UNDEF;
+ }
+
+ if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
+ ir_ref str_ref;
+
+ may_throw = 1;
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
+ if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
+ } else {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_TRUE(if_type);
+ }
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ str_ref = jit_Z_PTR(jit, op1_addr);
+ if (opline->opcode != ZEND_FETCH_DIM_IS) {
+ ir_ref ref;
+
+ if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper),
+ str_ref, jit_Z_LVAL(jit, op2_addr));
+ } else {
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper),
+ str_ref, jit_ZVAL_ADDR(jit, op2_addr));
+ }
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
+ } else {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper),
+ str_ref,
+ jit_ZVAL_ADDR(jit, op2_addr),
+ jit_ZVAL_ADDR(jit, res_addr));
+ }
+ ir_END_list(end_inputs);
+ }
+
+ if (op1_info & MAY_BE_OBJECT) {
+ ir_ref arg2;
+
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IS_UNDEF;
+ }
+
+ may_throw = 1;
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
+ if (exit_addr) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
+ } else {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+ ir_IF_TRUE(if_type);
+ }
+ }
+
+ jit_SET_EX_OPLINE(jit, opline);
+ if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+
+ if (opline->opcode != ZEND_FETCH_DIM_IS) {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, res_addr));
+ } else {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, res_addr));
+ }
+
+ ir_END_list(end_inputs);
+ }
+
+ if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
+ && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
+
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IS_UNDEF;
+ }
+
+ if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
+ jit_SET_EX_OPLINE(jit, opline);
+ if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
+ may_throw = 1;
+ zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
+ }
+
+ if (op2_info & MAY_BE_UNDEF) {
+ may_throw = 1;
+ zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
+ }
+ }
+
+ if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
+ ir_ref ref;
+
+ may_throw = 1;
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
+ } else {
+ jit_SET_EX_OPLINE(jit, opline);
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref);
+ }
+
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
+ /* Magic offsetGet() may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
+ if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
+ may_throw = 1;
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ }
+ if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
+ if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
+ if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
+ may_throw = 1;
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ }
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+ } else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) {
+ ir_BEGIN(IR_UNUSED); /* unreachable tail */
+ }
+
+ return 1;
+}
+
+static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ ir_ref *if_type,
+ ir_ref *ht_ref,
+ int *may_throw)
+{
+ ir_ref ref = IR_UNUSED;
+ ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED;
+ ir_refs *array_inputs, *array_values;
+
+ ir_refs_init(array_inputs, 4);
+ ir_refs_init(array_values, 4);
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if (op1_info & MAY_BE_REF) {
+ ir_ref if_reference, if_array, end1, ref2;
+
+ *may_throw = 1;
+ if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_FALSE(if_reference);
+ end1 = ir_END();
+ ir_IF_TRUE_cold(if_reference);
+ array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
+ if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
+ ir_IF_TRUE(if_array);
+ array_reference_end = ir_END();
+ ir_IF_FALSE_cold(if_array);
+ if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref);
+ ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
+
+ ir_MERGE_WITH(end1);
+ ref = ir_PHI_2(IR_ADDR, ref2, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ ir_ref op1_ref = ref;
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
+ ir_IF_TRUE(*if_type);
+ }
+ if (array_reference_end) {
+ ir_MERGE_WITH(array_reference_end);
+ op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref);
+ }
+ // JIT: SEPARATE_ARRAY()
+ ref = jit_Z_PTR_ref(jit, op1_ref);
+ if (RC_MAY_BE_N(op1_info)) {
+ if (RC_MAY_BE_1(op1_info)) {
+ ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
+ ir_IF_TRUE(if_refcount_1);
+ ir_refs_add(array_inputs, ir_END());
+ ir_refs_add(array_values, ref);
+ ir_IF_FALSE(if_refcount_1);
+ }
+ ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref);
+ }
+ if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
+ ir_refs_add(array_inputs, ir_END());
+ ir_refs_add(array_values, ref);
+ }
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
+ if (*if_type) {
+ ir_IF_FALSE_cold(*if_type);
+ *if_type = IR_UNUSED;
+ }
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
+ *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
+ ir_IF_TRUE(*if_type);
+ }
+ if ((op1_info & MAY_BE_UNDEF)
+ && (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) {
+ ir_ref end1 = IR_UNUSED;
+
+ *may_throw = 1;
+ if (op1_info & MAY_BE_NULL) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
+ ir_IF_TRUE(if_def);
+ end1 = ir_END();
+ ir_IF_FALSE(if_def);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
+ if (end1) {
+ ir_MERGE_WITH(end1);
+ }
+ }
+ // JIT: ZVAL_ARR(container, zend_new_array(8));
+ ref = ir_CALL_1(IR_ADDR,
+ jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
+ jit_ZVAL_ADDR(jit, op1_addr));
+ if (array_inputs->count) {
+ ir_refs_add(array_inputs, ir_END());
+ ir_refs_add(array_values, ref);
+ }
+ }
+
+ if (array_inputs->count) {
+ ir_MERGE_N(array_inputs->count, array_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs);
+ }
+
+ *ht_ref = ref;
+ return op1_addr;
+}
+
+static int zend_jit_fetch_dim(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ zend_jit_addr res_addr,
+ uint8_t dim_type)
+{
+ int may_throw = 0;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref ref, if_type = IR_UNUSED, ht_ref;
+
+ if (opline->opcode == ZEND_FETCH_DIM_RW) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
+ ir_refs *found_inputs, *found_vals;
+
+ ir_refs_init(found_inputs, 8);
+ ir_refs_init(found_vals, 8);
+
+ if (opline->op2_type == IS_UNUSED) {
+ ir_ref if_ok;
+
+ may_throw = 1;
+ // JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
+ ht_ref, jit_EG(uninitialized_zval));
+
+ // JIT: if (UNEXPECTED(!var_ptr)) {
+ if_ok = ir_IF(ref);
+ ir_IF_FALSE_cold(if_ok);
+ if (opline->opcode != ZEND_FETCH_DIM_RW) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_ok);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
+
+ ir_END_list(end_inputs);
+ } else {
+ uint32_t type;
+
+ switch (opline->opcode) {
+ case ZEND_FETCH_DIM_W:
+ case ZEND_FETCH_LIST_W:
+ type = BP_VAR_W;
+ break;
+ case ZEND_FETCH_DIM_RW:
+ may_throw = 1;
+ type = BP_VAR_RW;
+ break;
+ case ZEND_FETCH_DIM_UNSET:
+ type = BP_VAR_UNSET;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
+ may_throw = 1;
+ }
+ if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
+ op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
+ 0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) {
+ return 0;
+ }
+
+ if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ end_inputs = ir_END();
+ }
+ } else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+ /* impossible dead path */
+ end_inputs = ir_END();
+ } else {
+ ZEND_ASSERT(end_inputs == IR_UNUSED);
+ }
+
+ if (found_inputs->count) {
+ ir_MERGE_N(found_inputs->count, found_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
+ ir_END_list(end_inputs);
+ }
+
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
+ ir_ref arg2;
+
+ may_throw = 1;
+
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ if_type = IR_UNUSED;
+ }
+
+ if (opline->opcode != ZEND_FETCH_DIM_RW) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ if (opline->op2_type == IS_UNUSED) {
+ arg2 = IR_NULL;
+ } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+
+ switch (opline->opcode) {
+ case ZEND_FETCH_DIM_W:
+ case ZEND_FETCH_LIST_W:
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, res_addr));
+ break;
+ case ZEND_FETCH_DIM_RW:
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, res_addr));
+ break;
+// case ZEND_FETCH_DIM_UNSET:
+// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
+// break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
+ ir_END_list(end_inputs);
+ }
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
+ /* ASSIGN_DIM may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
+ && (op2_info & MAY_HAVE_DTOR)
+ && (op2_info & MAY_BE_RC1)) {
+ may_throw = 1;
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ bool op1_avoid_refcounting,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ uint8_t dim_type,
+ int may_throw,
+ uint8_t smart_branch_opcode,
+ uint32_t target_label,
+ uint32_t target_label2,
+ const void *exit_addr)
+{
+ zend_jit_addr res_addr;
+ ir_ref if_type = IR_UNUSED;
+ ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
+ ir_refs *true_inputs;
+
+ ir_refs_init(true_inputs, 8);
+
+ // TODO: support for empty() ???
+ ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
+
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ref = jit_ZVAL_DEREF_ref(jit, ref);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ const void *found_exit_addr = NULL;
+ const void *not_found_exit_addr = NULL;
+ ir_ref ht_ref;
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
+ ir_IF_TRUE(if_type);
+ }
+
+ ht_ref = jit_Z_PTR(jit, op1_addr);
+
+ if (exit_addr
+ && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
+ && !may_throw
+ && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
+ && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ found_exit_addr = exit_addr;
+ } else {
+ not_found_exit_addr = exit_addr;
+ }
+ }
+ if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
+ op2_info, op2_addr, op2_range, dim_type, found_exit_addr, not_found_exit_addr, NULL,
+ 0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) {
+ return 0;
+ }
+
+ if (found_exit_addr) {
+ ir_MERGE_list(false_inputs);
+ return 1;
+ } else if (not_found_exit_addr) {
+ ir_MERGE_N(true_inputs->count, true_inputs->refs);
+ return 1;
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ if_type = IR_UNUSED;
+ }
+
+ if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
+ ir_ref ref, arg1, arg2, if_true;
+
+ jit_SET_EX_OPLINE(jit, opline);
+ arg1 = jit_ZVAL_ADDR(jit, op1_addr);
+ if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+ ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2);
+ if_true = ir_IF(ref);
+ ir_IF_TRUE(if_true);
+ ir_refs_add(true_inputs, ir_END());
+ ir_IF_FALSE(if_true);
+ ir_END_list(false_inputs);
+ } else {
+ if (op2_info & MAY_BE_UNDEF) {
+ ir_ref end1 = IR_UNUSED;
+
+ if (op2_info & MAY_BE_ANY) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
+ ir_IF_TRUE(if_def);
+ end1 = ir_END();
+ ir_IF_FALSE(if_def);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
+ if (end1) {
+ ir_MERGE_WITH(end1);
+ }
+ }
+ ir_END_list(false_inputs);
+ }
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
+ /* Magic offsetExists() may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ if (true_inputs->count) {
+ ir_MERGE_N(true_inputs->count, true_inputs->refs);
+
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ if (!op1_avoid_refcounting) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ if (may_throw) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ }
+ if (!(opline->extended_value & ZEND_ISEMPTY)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
+ } else if (smart_branch_opcode == ZEND_JMPNZ) {
+ _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ZEND_UNREACHABLE(); // TODO: support for empty()
+ }
+ }
+
+ ir_MERGE_list(false_inputs);
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+ if (!op1_avoid_refcounting) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ if (may_throw) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ }
+ if (!(opline->extended_value & ZEND_ISEMPTY)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ } else if (smart_branch_opcode) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
+ } else if (smart_branch_opcode == ZEND_JMPNZ) {
+ _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ZEND_UNREACHABLE(); // TODO: support for empty()
+ }
+
+ if (!exit_addr && smart_branch_opcode) {
+ jit->b = -1;
+ } else {
+ ir_MERGE_list(end_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_assign_dim(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ uint32_t val_info,
+ zend_jit_addr op3_addr,
+ zend_jit_addr op3_def_addr,
+ zend_jit_addr res_addr,
+ uint8_t dim_type,
+ int may_throw)
+{
+ ir_ref if_type = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED, ht_ref;
+
+ if (op3_addr != op3_def_addr && op3_def_addr) {
+ if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
+ return 0;
+ }
+ if (Z_MODE(op3_def_addr) == IS_REG && Z_MODE(op3_addr) != IS_REG) {
+ op3_addr = op3_def_addr;
+ }
+ }
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
+
+ val_info &= ~MAY_BE_UNDEF;
+ }
+
+ op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
+ if (opline->op2_type == IS_UNUSED) {
+ uint32_t var_info = MAY_BE_NULL;
+ ir_ref if_ok, ref;
+ zend_jit_addr var_addr;
+
+ // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
+ ht_ref, jit_EG(uninitialized_zval));
+
+ // JIT: if (UNEXPECTED(!var_ptr)) {
+ if_ok = ir_IF(ref);
+ ir_IF_FALSE_cold(if_ok);
+
+ // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
+
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_ok);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ } else {
+ uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
+ zend_jit_addr var_addr;
+ ir_ref ref;
+ ir_refs *found_inputs, *found_values;
+
+ ir_refs_init(found_inputs, 8);
+ ir_refs_init(found_values, 8);
+
+ if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
+ op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
+ 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
+ return 0;
+ }
+
+ if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
+ var_info |= MAY_BE_REF;
+ }
+ if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ var_info |= MAY_BE_RC1;
+ }
+
+ if (found_inputs->count) {
+ ir_MERGE_N(found_inputs->count, found_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ // JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
+ if (opline->op1_type == IS_VAR
+ && Z_MODE(op3_addr) != IS_REG
+ && (res_addr == 0 || Z_MODE(res_addr) != IS_REG)) {
+ ZEND_ASSERT(opline->result_type == IS_UNUSED);
+ if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ } else {
+ if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
+ return 0;
+ }
+ }
+ }
+ }
+
+ ir_END_list(end_inputs);
+ }
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
+ ir_ref arg2, arg4;
+
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IR_UNUSED;
+ }
+
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (opline->op2_type == IS_UNUSED) {
+ arg2 = IR_NULL;
+ } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+
+ if (opline->result_type == IS_UNUSED) {
+ arg4 = IR_NULL;
+ } else {
+ arg4 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, op3_addr),
+ arg4);
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
+ /* ASSIGN_DIM may increase refcount of the value */
+ val_info |= MAY_BE_RCN;
+ }
+#endif
+
+ jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
+
+ ir_END_list(end_inputs);
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
+ /* ASSIGN_DIM may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ ir_MERGE_list(end_inputs);
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_assign_dim_op(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint32_t op1_info,
+ uint32_t op1_def_info,
+ zend_jit_addr op1_addr,
+ uint32_t op2_info,
+ zend_jit_addr op2_addr,
+ zend_ssa_range *op2_range,
+ uint32_t op1_data_info,
+ zend_jit_addr op3_addr,
+ zend_ssa_range *op1_data_range,
+ uint8_t dim_type,
+ int may_throw)
+{
+ zend_jit_addr var_addr = IS_UNUSED;
+ const void *not_found_exit_addr = NULL;
+ uint32_t var_info = MAY_BE_NULL;
+ ir_ref if_type = IS_UNUSED;
+ ir_ref end_inputs = IR_UNUSED, ht_ref;
+ bool emit_fast_path = 1;
+
+ ZEND_ASSERT(opline->result_type == IS_UNUSED);
+
+ if (may_throw) {
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+
+ op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
+
+ if (Z_MODE(op3_addr) == IS_REG
+ && Z_LOAD(op3_addr)
+ && jit->ra[Z_SSA_VAR(op3_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, op3_addr);
+ }
+
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
+ uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
+
+ if (opline->op2_type == IS_UNUSED) {
+ var_info = MAY_BE_NULL;
+ ir_ref if_ok, ref;
+
+ // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
+ ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
+ ht_ref, jit_EG(uninitialized_zval));
+
+ // JIT: if (UNEXPECTED(!var_ptr)) {
+ if_ok = ir_IF(ref);
+ ir_IF_FALSE_cold(if_ok);
+
+ // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
+ ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
+
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_ok);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else {
+ ir_ref ref;
+ ir_refs *found_inputs, *found_values;
+
+ ir_refs_init(found_inputs, 8);
+ ir_refs_init(found_values, 8);
+
+ var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
+ if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
+ var_info |= MAY_BE_REF;
+ }
+ if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ var_info |= MAY_BE_RC1;
+ }
+
+ if (dim_type != IS_UNKNOWN
+ && dim_type != IS_UNDEF
+ && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
+ && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
+ && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!not_found_exit_addr) {
+ return 0;
+ }
+ }
+
+ if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
+ op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, NULL,
+ 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
+ return 0;
+ }
+
+ if (found_inputs->count) {
+ ir_MERGE_N(found_inputs->count, found_inputs->refs);
+ ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+
+ if (not_found_exit_addr && dim_type != IS_REFERENCE) {
+ jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
+ var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
+ }
+ if (var_info & MAY_BE_REF) {
+ binary_op_type binary_op = get_binary_op(opline->extended_value);
+ ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2;
+
+ ref = jit_ZVAL_ADDR(jit, var_addr);
+ if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
+ ir_IF_FALSE(if_ref);
+ noref_path = ir_END();
+ ir_IF_TRUE(if_ref);
+
+ reference = jit_Z_PTR_ref(jit, ref);
+ ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
+ if_typed = jit_if_TYPED_REF(jit, reference);
+ ir_IF_FALSE(if_typed);
+ ref_path = ir_END();
+ ir_IF_TRUE_cold(if_typed);
+
+ if (Z_MODE(op3_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, op3_addr, real_addr, op1_data_info)) {
+ return 0;
+ }
+ arg2 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op3_addr);
+ }
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
+ reference, arg2, ir_CONST_FC_FUNC(binary_op));
+
+ ir_END_list(end_inputs);
+
+ ir_MERGE_2(noref_path, ref_path);
+ ref = ir_PHI_2(IR_ADDR, ref, ref2);
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ }
+ } else {
+ emit_fast_path = 0;
+ }
+ }
+
+ if (emit_fast_path) {
+ uint8_t val_op_type = (opline+1)->op1_type;
+
+ if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
+ /* prevent FREE_OP in the helpers */
+ val_op_type = IS_CV;
+ }
+
+ switch (opline->extended_value) {
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ case ZEND_DIV:
+ if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
+ 1 /* may overflow */, may_throw)) {
+ return 0;
+ }
+ break;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
+ IS_CV, opline->op1, var_addr, var_info, NULL,
+ val_op_type, (opline+1)->op1, op3_addr, op1_data_info,
+ op1_data_range,
+ 0, var_addr, var_def_info, var_info, may_throw)) {
+ return 0;
+ }
+ break;
+ case ZEND_CONCAT:
+ if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
+ may_throw)) {
+ return 0;
+ }
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
+ binary_op_type binary_op;
+ ir_ref arg2;
+
+ if (if_type) {
+ ir_IF_FALSE_cold(if_type);
+ if_type = IS_UNUSED;
+ }
+
+ if (opline->op2_type == IS_UNUSED) {
+ arg2 = IR_NULL;
+ } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, op2_addr);
+ }
+ binary_op = get_binary_op(opline->extended_value);
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ arg2,
+ jit_ZVAL_ADDR(jit, op3_addr),
+ ir_CONST_FC_FUNC(binary_op));
+ ir_END_list(end_inputs);
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+
+ jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
+{
+ zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+
+ // JIT: ZVAL_COPY(res, value);
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+
+ jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
+ } else {
+ zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+
+ jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
+ }
+
+ // JIT: Z_FE_POS_P(res) = 0;
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0));
+
+ return 1;
+}
+
+static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ ir_ref ref;
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ref = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
+ ir_CONST_U32(HASH_FLAG_PACKED));
+ if (op_info & MAY_BE_ARRAY_PACKED) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
+ }
+
+ return 1;
+}
+
+static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
+{
+ zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED;
+ ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED;
+ ir_ref exit_inputs = IR_UNUSED;
+
+ if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
+ /* empty array */
+ if (exit_addr) {
+ if (exit_opcode == ZEND_JMP) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } else {
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
+ jit->b = -1;
+ }
+ return 1;
+ }
+
+ // JIT: array = EX_VAR(opline->op1.var);
+ // JIT: fe_ht = Z_ARRVAL_P(array);
+ ht_ref = jit_Z_PTR(jit, op1_addr);
+
+ if (op1_info & MAY_BE_PACKED_GUARD) {
+ if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
+ return 0;
+ }
+ }
+
+ // JIT: pos = Z_FE_POS_P(array);
+ hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)));
+
+ if (MAY_BE_HASH(op1_info)) {
+ ir_ref loop_ref, pos2_ref, p2_ref;
+
+ if (MAY_BE_PACKED(op1_info)) {
+ ref = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
+ ir_CONST_U32(HASH_FLAG_PACKED));
+ if_packed = ir_IF(ref);
+ ir_IF_FALSE(if_packed);
+ }
+
+ // JIT: p = fe_ht->arData + pos;
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(hash_pos_ref);
+ } else {
+ ref = ir_BITCAST_A(hash_pos_ref);
+ }
+ hash_p_ref = ir_ADD_A(
+ ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))),
+ ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData))));
+
+ loop_ref = ir_LOOP_BEGIN(ir_END());
+ hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED);
+ hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED);
+
+ // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
+ ref = ir_ULT(hash_pos_ref,
+ ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
+
+ // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
+ // JIT: ZEND_VM_CONTINUE();
+
+ if (exit_addr) {
+ if (exit_opcode == ZEND_JMP) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_fit = ir_IF(ref);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(exit_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+ } else {
+ ir_ref if_fit = ir_IF(ref);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(exit_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+
+ // JIT: pos++;
+ pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1));
+
+ // JIT: value_type = Z_TYPE_INFO_P(value);
+ // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
+ if (!exit_addr || exit_opcode == ZEND_JMP) {
+ if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
+ ir_IF_FALSE(if_def_hash);
+ } else {
+ ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
+ }
+
+ // JIT: p++;
+ p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket));
+
+ ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
+ ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref);
+ ir_PHI_SET_OP(hash_p_ref, 2, p2_ref);
+
+ if (MAY_BE_PACKED(op1_info)) {
+ ir_IF_TRUE(if_packed);
+ }
+ }
+ if (MAY_BE_PACKED(op1_info)) {
+ ir_ref loop_ref, pos2_ref, p2_ref;
+
+ // JIT: p = fe_ht->arPacked + pos;
+ if (sizeof(void*) == 8) {
+ ref = ir_ZEXT_A(packed_pos_ref);
+ } else {
+ ref = ir_BITCAST_A(packed_pos_ref);
+ }
+ packed_p_ref = ir_ADD_A(
+ ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))),
+ ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))));
+
+ loop_ref = ir_LOOP_BEGIN(ir_END());
+ packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED);
+ packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED);
+
+ // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
+ ref = ir_ULT(packed_pos_ref,
+ ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
+
+ // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
+ // JIT: ZEND_VM_CONTINUE();
+ if (exit_addr) {
+ if (exit_opcode == ZEND_JMP) {
+ ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_fit = ir_IF(ref);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(exit_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+ } else {
+ ir_ref if_fit = ir_IF(ref);
+ ir_IF_FALSE(if_fit);
+ ir_END_list(exit_inputs);
+ ir_IF_TRUE(if_fit);
+ }
+
+ // JIT: pos++;
+ pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1));
+
+ // JIT: value_type = Z_TYPE_INFO_P(value);
+ // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
+ if (!exit_addr || exit_opcode == ZEND_JMP) {
+ if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
+ ir_IF_FALSE(if_def_packed);
+ } else {
+ ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
+ }
+
+ // JIT: p++;
+ p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval));
+
+ ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
+ ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref);
+ ir_PHI_SET_OP(packed_p_ref, 2, p2_ref);
+ }
+
+ if (!exit_addr || exit_opcode == ZEND_JMP) {
+ zend_jit_addr val_addr;
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ uint32_t val_info;
+ ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED;
+
+ if (RETURN_VALUE_USED(opline)) {
+ zend_jit_addr res_addr = RES_ADDR();
+
+ if (MAY_BE_HASH(op1_info)) {
+ ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED;
+
+ ZEND_ASSERT(if_def_hash);
+ ir_IF_TRUE(if_def_hash);
+
+ // JIT: Z_FE_POS_P(array) = pos + 1;
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
+ ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)));
+
+ if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
+ key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key)));
+ }
+ if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
+ && (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
+ // JIT: if (!p->key) {
+ if_key = ir_IF(key_ref);
+ ir_IF_TRUE(if_key);
+ }
+ if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
+ ir_ref if_interned, interned_path;
+
+ // JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
+ jit_set_Z_PTR(jit, res_addr, key_ref);
+ ref = ir_AND_U32(
+ ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))),
+ ir_CONST_U32(IS_STR_INTERNED));
+ if_interned = ir_IF(ref);
+ ir_IF_TRUE(if_interned);
+
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
+
+ interned_path = ir_END();
+ ir_IF_FALSE(if_interned);
+
+ jit_GC_ADDREF(jit, key_ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
+
+ ir_MERGE_WITH(interned_path);
+
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ key_path = ir_END();
+ }
+ }
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
+ ir_IF_FALSE(if_key);
+ }
+ // JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h);
+ ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h)));
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+
+ if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
+ ir_MERGE_WITH(key_path);
+ }
+ }
+ if (MAY_BE_PACKED(op1_info)) {
+ hash_path = ir_END();
+ } else {
+ p_ref = hash_p_ref;
+ }
+ }
+ if (MAY_BE_PACKED(op1_info)) {
+ ZEND_ASSERT(if_def_packed);
+ ir_IF_TRUE(if_def_packed);
+
+ // JIT: Z_FE_POS_P(array) = pos + 1;
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
+ ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)));
+
+ // JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos);
+ if (sizeof(zend_long) == 8) {
+ packed_pos_ref = ir_ZEXT_L(packed_pos_ref);
+ } else {
+ packed_pos_ref = ir_BITCAST_L(packed_pos_ref);
+ }
+ jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+
+ if (MAY_BE_HASH(op1_info)) {
+ ir_MERGE_WITH(hash_path);
+ p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref);
+ } else {
+ p_ref = packed_p_ref;
+ }
+ }
+ } else {
+ ir_ref pos_ref = IR_UNUSED;
+
+ if (if_def_hash && if_def_packed) {
+ ir_IF_TRUE(if_def_hash);
+ ir_MERGE_WITH_EMPTY_TRUE(if_def_packed);
+ pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref);
+ p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref);
+ } else if (if_def_hash) {
+ ir_IF_TRUE(if_def_hash);
+ pos_ref = hash_pos_ref;
+ p_ref = hash_p_ref;
+ } else if (if_def_packed) {
+ ir_IF_TRUE(if_def_packed);
+ pos_ref = packed_pos_ref;
+ p_ref = packed_p_ref;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+
+ // JIT: Z_FE_POS_P(array) = pos + 1;
+ ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
+ ir_ADD_U32(pos_ref, ir_CONST_U32(1)));
+ }
+
+ val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
+ if (val_info & MAY_BE_ARRAY) {
+ val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ if (op1_info & MAY_BE_ARRAY_OF_REF) {
+ val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
+ MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ val_info |= MAY_BE_RC1 | MAY_BE_RCN;
+ }
+
+ val_addr = ZEND_ADDR_REF_ZVAL(p_ref);
+ if (opline->op2_type == IS_CV) {
+ // JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
+ if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) {
+ return 0;
+ }
+ } else {
+ // JIT: ZVAL_COPY(res, value);
+ jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
+ }
+
+ if (!exit_addr) {
+ zend_basic_block *bb;
+
+ ZEND_ASSERT(jit->b >= 0);
+ bb = &jit->ssa->cfg.blocks[jit->b];
+ _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
+ ZEND_ASSERT(exit_inputs);
+ if (!jit->ctx.ir_base[exit_inputs].op2) {
+ ref = exit_inputs;
+ } else {
+ ir_MERGE_list(exit_inputs);
+ ref = ir_END();
+ }
+ _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
+ jit->b = -1;
+ }
+ } else {
+ ZEND_ASSERT(exit_inputs);
+ ir_MERGE_list(exit_inputs);
+ }
+
+ return 1;
+}
+
+static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var)
+{
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+ ir_ref ref = jit_Z_PTR(jit, this_addr);
+
+ jit_set_Z_PTR(jit, var_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
+ jit_GC_ADDREF(jit, ref);
+
+ return 1;
+}
+
+static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only)
+{
+ if (!op_array->scope ||
+ (op_array->fn_flags & ZEND_ACC_STATIC) ||
+ ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ if (!JIT_G(current_frame) ||
+ !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
+
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
+
+ if (JIT_G(current_frame)) {
+ TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
+ }
+ }
+ } else {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
+
+ ir_IF_FALSE_cold(if_object);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
+
+ ir_IF_TRUE(if_object);
+ }
+ }
+
+ if (!check_only) {
+ if (!zend_jit_load_this(jit, opline->result.var)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)),
+ ir_CONST_ADDR(exit_addr));
+
+ return 1;
+}
+
+static int zend_jit_fetch_obj(zend_jit_ctx *jit,
+ const zend_op *opline,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ bool op1_indirect,
+ zend_class_entry *ce,
+ bool ce_is_instanceof,
+ bool on_this,
+ bool delayed_fetch_this,
+ bool op1_avoid_refcounting,
+ zend_class_entry *trace_ce,
+ zend_jit_addr res_addr,
+ uint8_t prop_type,
+ int may_throw)
+{
+ zval *member;
+ zend_property_info *prop_info;
+ bool may_be_dynamic = 1;
+ zend_jit_addr prop_addr;
+ uint32_t res_info = RES_INFO();
+ ir_ref prop_type_ref = IR_UNUSED;
+ ir_ref obj_ref = IR_UNUSED;
+ ir_ref prop_ref = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+ ir_ref end_values = IR_UNUSED;
+
+ ZEND_ASSERT(opline->op2_type == IS_CONST);
+ ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
+
+ member = RT_CONSTANT(opline, opline->op2);
+ ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
+ prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
+
+ if (on_this) {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ obj_ref = jit_Z_PTR(jit, this_addr);
+ } else {
+ if (opline->op1_type == IS_VAR
+ && opline->opcode == ZEND_FETCH_OBJ_W
+ && (op1_info & MAY_BE_INDIRECT)
+ && Z_REG(op1_addr) == ZREG_FP) {
+ op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
+ } else {
+ ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+
+ ir_IF_FALSE_cold(if_obj);
+ if (opline->opcode != ZEND_FETCH_OBJ_IS) {
+ ir_ref op1_ref = IR_UNUSED;
+
+ jit_SET_EX_OPLINE(jit, opline);
+ if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) {
+ zend_jit_addr orig_op1_addr = OP1_ADDR();
+ ir_ref fast_path = IR_UNUSED;
+
+ if (op1_info & MAY_BE_ANY) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
+ ir_IF_TRUE(if_def);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_def);
+ }
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
+ ir_CONST_U32(opline->op1.var));
+ if (fast_path) {
+ ir_MERGE_WITH(fast_path);
+ }
+ op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
+ } else {
+ op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
+ }
+ if (opline->opcode == ZEND_FETCH_OBJ_W) {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write),
+ op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
+ jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
+ } else {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read),
+ op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ }
+ } else {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ }
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_obj);
+ }
+ }
+ obj_ref = jit_Z_PTR(jit, op1_addr);
+ }
+
+ ZEND_ASSERT(obj_ref);
+ if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
+ prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
+ if (prop_info) {
+ ce = trace_ce;
+ ce_is_instanceof = 0;
+ if (!(op1_info & MAY_BE_CLASS_GUARD)) {
+ if (on_this && JIT_G(current_frame)
+ && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
+ ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
+ } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
+ if (on_this && JIT_G(current_frame)) {
+ JIT_G(current_frame)->ce = ce;
+ TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
+ }
+ } else {
+ return 0;
+ }
+ if (ssa->var_info && ssa_op->op1_use >= 0) {
+ ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_use].ce = ce;
+ ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
+ }
+ }
+ }
+ }
+
+ if (!prop_info) {
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
+ ir_ref if_same = ir_IF(ir_EQ(ref,
+ ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
+
+ ir_IF_FALSE_cold(if_same);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_same);
+ ir_ref offset_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
+
+ may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array);
+ if (may_be_dynamic) {
+ ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
+ if (opline->opcode == ZEND_FETCH_OBJ_W) {
+ ir_IF_TRUE_cold(if_dynamic);
+ ir_END_list(slow_inputs);
+ } else {
+ ir_IF_TRUE_cold(if_dynamic);
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (opline->opcode != ZEND_FETCH_OBJ_IS) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic_ex),
+ obj_ref, offset_ref);
+ ir_END_PHI_list(end_values, val_addr);
+ } else {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic),
+ obj_ref, offset_ref);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ if (Z_MODE(res_addr) == IS_REG) {
+ ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic_ex),
+ obj_ref, offset_ref);
+ ir_END_PHI_list(end_values, val_addr);
+ } else {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic),
+ obj_ref, offset_ref);
+ ir_END_list(end_inputs);
+ }
+ }
+ }
+ ir_IF_FALSE(if_dynamic);
+ }
+ prop_ref = ir_ADD_A(obj_ref, offset_ref);
+ prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
+ ir_ref if_def = ir_IF(prop_type_ref);
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_def);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+ if (opline->opcode == ZEND_FETCH_OBJ_W
+ && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
+ uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
+
+ ir_ref allowed_inputs = IR_UNUSED;
+ ir_ref forbidden_inputs = IR_UNUSED;
+
+ ir_ref prop_info_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
+ ir_ref if_has_prop_info = ir_IF(prop_info_ref);
+
+ ir_IF_TRUE_cold(if_has_prop_info);
+
+ ir_ref prop_flags = ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags)));
+ ir_ref if_readonly_or_avis = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY|ZEND_ACC_PPP_SET_MASK)));
+
+ ir_IF_FALSE(if_readonly_or_avis);
+ ir_END_list(allowed_inputs);
+
+ ir_IF_TRUE_cold(if_readonly_or_avis);
+
+ ir_ref if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY)));
+ ir_IF_TRUE(if_readonly);
+ ir_END_list(forbidden_inputs);
+
+ ir_IF_FALSE(if_readonly);
+ ir_ref has_avis_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), prop_info_ref);
+ ir_ref if_avis_access = ir_IF(has_avis_access);
+ ir_IF_TRUE(if_avis_access);
+ ir_END_list(allowed_inputs);
+
+ ir_IF_FALSE(if_avis_access);
+ ir_END_list(forbidden_inputs);
+
+ ir_MERGE_list(forbidden_inputs);
+
+ ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
+ ir_IF_TRUE(if_prop_obj);
+ ref = jit_Z_PTR(jit, prop_addr);
+ jit_GC_ADDREF(jit, ref);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE_cold(if_prop_obj);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY)));
+ ir_IF_TRUE(if_readonly);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), prop_info_ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE(if_readonly);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error),
+ prop_info_ref, ir_CONST_ADDR("indirectly modify"));
+ jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
+ ir_END_list(end_inputs);
+
+ ir_MERGE_list(allowed_inputs);
+
+ if (flags == ZEND_FETCH_DIM_WRITE) {
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
+ prop_ref, prop_info_ref);
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_has_prop_info);
+ } else if (flags == ZEND_FETCH_REF) {
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
+ prop_ref,
+ prop_info_ref,
+ jit_ZVAL_ADDR(jit, res_addr));
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_has_prop_info);
+ } else {
+ ZEND_ASSERT(flags == 0);
+ ir_MERGE_WITH_EMPTY_FALSE(if_has_prop_info);
+ }
+ }
+ } else {
+ prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
+ /* perform IS_UNDEF check only after result type guard (during deoptimization) */
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+ ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr));
+ }
+ } else {
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+ ir_ref if_def = ir_IF(prop_type_ref);
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
+ ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
+ ir_IF_TRUE(if_prop_obj);
+ ir_ref ref = jit_Z_PTR(jit, prop_addr);
+ jit_GC_ADDREF(jit, ref);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE_cold(if_prop_obj);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), ir_CONST_ADDR(prop_info));
+ jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
+ ir_END_list(end_inputs);
+
+ goto result_fetched;
+ } else if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_PPP_SET_MASK)) {
+ /* Readonly properties which are also asymmetric are never mutable indirectly, which is
+ * handled by the previous branch. */
+ ir_ref has_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), ir_CONST_ADDR(prop_info));
+
+ ir_ref if_access = ir_IF(has_access);
+ ir_IF_FALSE_cold(if_access);
+
+ ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
+ ir_IF_TRUE(if_prop_obj);
+ ir_ref ref = jit_Z_PTR(jit, prop_addr);
+ jit_GC_ADDREF(jit, ref);
+ jit_set_Z_PTR(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
+ ir_END_list(end_inputs);
+
+ ir_IF_FALSE_cold(if_prop_obj);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error),
+ ir_CONST_ADDR(prop_info), ir_CONST_ADDR("indirectly modify"));
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_access);
+ }
+
+ if (opline->opcode == ZEND_FETCH_OBJ_W
+ && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
+ && ZEND_TYPE_IS_SET(prop_info->type)) {
+ uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
+
+ if (flags == ZEND_FETCH_DIM_WRITE) {
+ if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
+ if (!prop_type_ref) {
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+ }
+ ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE)));
+ ir_IF_TRUE_cold(if_null_or_flase);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
+ prop_ref, ir_CONST_ADDR(prop_info));
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_null_or_flase);
+ }
+ } else if (flags == ZEND_FETCH_REF) {
+ ir_ref ref;
+
+ if (!prop_type_ref) {
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+ }
+
+ ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX)));
+ ir_IF_FALSE(if_reference);
+ if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
+ ref = ir_CONST_ADDR(prop_info);
+ } else {
+ int prop_info_offset =
+ (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
+
+ ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
+ }
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
+ prop_ref,
+ ref,
+ jit_ZVAL_ADDR(jit, res_addr));
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_reference);
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ }
+ }
+
+ if (opline->opcode == ZEND_FETCH_OBJ_W) {
+ ZEND_ASSERT(prop_ref);
+ jit_set_Z_PTR(jit, res_addr, prop_ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
+ ssa->var_info[ssa_op->result_def].indirect_reference = 1;
+ }
+ ir_END_list(end_inputs);
+ } else {
+ if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
+ ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
+ } else if ((res_info & MAY_BE_GUARD) && Z_MODE(res_addr) == IS_REG) {
+ ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+
+ if (!zend_jit_zval_copy_deref_reg(jit, res_addr, res_info & ~MAY_BE_GUARD, prop_addr, prop_type_ref, &end_values)) {
+ return 0;
+ }
+ } else {
+ prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
+
+ if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
+ return 0;
+ }
+ ir_END_list(end_inputs);
+ }
+ }
+
+result_fetched:
+ if (op1_avoid_refcounting) {
+ SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
+ }
+
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
+ ir_MERGE_list(slow_inputs);
+ jit_SET_EX_OPLINE(jit, opline);
+
+ op1_info |= MAY_BE_RC1 | MAY_BE_RCN; /* object may be captured/released in magic handler */
+ if (opline->opcode == ZEND_FETCH_OBJ_W) {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref);
+ ir_END_list(end_inputs);
+ } else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ ir_ref val_ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow_ex), obj_ref);
+ ir_END_PHI_list(end_values, val_ref);
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref);
+ ir_END_list(end_inputs);
+ }
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref);
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (end_values) {
+ ir_ref val_ref = ir_PHI_list(end_values);
+ zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val_ref);
+ bool result_avoid_refcounting = 0;
+
+ ZEND_ASSERT(opline->opcode == ZEND_FETCH_OBJ_R
+ || opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG
+ || opline->opcode == ZEND_FETCH_OBJ_IS);
+ ZEND_ASSERT(end_inputs == IR_UNUSED);
+ if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
+ uint8_t type = concrete_type(res_info);
+ uint32_t flags = ZEND_JIT_EXIT_CHECK_EXCEPTION;
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && !delayed_fetch_this
+ && !op1_avoid_refcounting) {
+ flags |= ZEND_JIT_EXIT_FREE_OP1;
+ }
+
+ if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
+ && !(flags & ZEND_JIT_EXIT_FREE_OP1)
+ && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
+ && (ssa_op+1)->op1_use == ssa_op->result_def
+ && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
+ result_avoid_refcounting = 1;
+ ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
+ }
+
+ val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
+ 1, flags, op1_avoid_refcounting);
+ if (!val_addr) {
+ return 0;
+ }
+
+ res_info &= ~MAY_BE_GUARD;
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+
+ // ZVAL_COPY
+ jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
+
+ if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
+ return 0;
+ }
+ } else {
+ ir_MERGE_list(end_inputs);
+ }
+
+ if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
+ if (opline->op1_type == IS_VAR
+ && opline->opcode == ZEND_FETCH_OBJ_W
+ && (op1_info & MAY_BE_RC1)) {
+ zend_jit_addr orig_op1_addr = OP1_ADDR();
+ ir_ref if_refcounted, ptr, refcount, if_non_zero;
+ ir_ref merge_inputs = IR_UNUSED;
+
+ if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
+ ir_IF_FALSE( if_refcounted);
+ ir_END_list(merge_inputs);
+ ir_IF_TRUE( if_refcounted);
+ ptr = jit_Z_PTR(jit, orig_op1_addr);
+ refcount = jit_GC_DELREF(jit, ptr);
+ if_non_zero = ir_IF(refcount);
+ ir_IF_TRUE( if_non_zero);
+ ir_END_list(merge_inputs);
+ ir_IF_FALSE( if_non_zero);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr);
+ ir_END_list(merge_inputs);
+ ir_MERGE_list(merge_inputs);
+ } else if (!op1_avoid_refcounting) {
+ if (on_this) {
+ op1_info &= ~MAY_BE_RC1;
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+ }
+
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
+ && prop_info
+ && (opline->opcode != ZEND_FETCH_OBJ_W ||
+ !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
+ !ZEND_TYPE_IS_SET(prop_info->type))
+ && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
+ may_throw = 0;
+ }
+
+ if (may_throw) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_check_exception_undef_result(jit, opline);
+ } else {
+ zend_jit_check_exception(jit);
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_fetch_obj_func_arg(zend_jit_ctx *jit, const zend_op *opline,
+ const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op,
+ uint32_t op1_info, zend_jit_addr op1_addr, zend_class_entry *ce,
+ bool ce_is_instanceof, bool on_this, zend_jit_addr res_addr)
+{
+ ir_ref rx, call_info, if_by_ref, end_by_ref;
+
+ /* Both runtime paths must observe a consistent frame state. The delayed
+ * call chain would otherwise only be flushed inside the by-ref branch (by
+ * zend_jit_set_ip() in zend_jit_handler()), leaving EX(call) stale on the
+ * by-val path and after the merge. Flush it before branching. */
+ if (jit->delayed_call_level) {
+ if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ /* JIT: if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_SEND_ARG_BY_REF) */
+ if (jit->reuse_ip) {
+ rx = jit_IP(jit);
+ } else {
+ rx = ir_LOAD_A(jit_EX(call));
+ }
+ call_info = ir_LOAD_U32(jit_CALL(rx, This.u1.type_info));
+ if_by_ref = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
+
+ /* by-ref path: the FUNC_ARG handler re-checks the flag and dispatches
+ * into FETCH_OBJ_W */
+ ir_IF_TRUE_cold(if_by_ref);
+ if (!zend_jit_handler(jit, opline, zend_may_throw(opline, ssa_op, op_array, ssa))) {
+ return 0;
+ }
+ end_by_ref = ir_END();
+
+ /* zend_jit_handler() stored IP = opline + 1 on the by-ref path only;
+ * that compile-time knowledge is invalid for the by-val path and after
+ * the merge. */
+ zend_jit_reset_last_valid_opline(jit);
+
+ /* by-val path */
+ ir_IF_FALSE(if_by_ref);
+ if (!zend_jit_fetch_obj(jit, opline, op_array, ssa, ssa_op,
+ op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
+ res_addr, IS_UNKNOWN,
+ zend_may_throw(opline, ssa_op, op_array, ssa))) {
+ return 0;
+ }
+ ir_MERGE_WITH(end_by_ref);
+
+ return 1;
+}
+
+
+static int zend_jit_assign_obj(zend_jit_ctx *jit,
+ const zend_op *opline,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ uint32_t val_info,
+ zend_jit_addr val_addr,
+ zend_jit_addr val_def_addr,
+ zend_jit_addr res_addr,
+ bool op1_indirect,
+ zend_class_entry *ce,
+ bool ce_is_instanceof,
+ bool on_this,
+ bool delayed_fetch_this,
+ zend_class_entry *trace_ce,
+ uint8_t prop_type,
+ int may_throw)
+{
+ zval *member;
+ zend_string *name;
+ zend_property_info *prop_info;
+ zend_jit_addr prop_addr;
+ ir_ref obj_ref = IR_UNUSED;
+ ir_ref prop_ref = IR_UNUSED;
+ ir_ref delayed_end_input = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+ uint32_t res_info = RES_INFO();
+
+ if (Z_MODE(val_addr) == IS_REG
+ && Z_LOAD(val_addr)
+ && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, val_addr);
+ }
+
+ if (val_addr != val_def_addr && val_def_addr) {
+ if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
+ return 0;
+ }
+ if (Z_MODE(val_def_addr) == IS_REG && Z_MODE(val_addr) != IS_REG) {
+ val_addr = val_def_addr;
+ }
+ }
+
+ ZEND_ASSERT(opline->op2_type == IS_CONST);
+ ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
+
+ member = RT_CONSTANT(opline, opline->op2);
+ ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
+ name = Z_STR_P(member);
+ prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
+
+ if (on_this) {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ obj_ref = jit_Z_PTR(jit, this_addr);
+ } else {
+ if (opline->op1_type == IS_VAR
+ && (op1_info & MAY_BE_INDIRECT)
+ && Z_REG(op1_addr) == ZREG_FP) {
+ op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
+ } else {
+ ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+ ir_IF_FALSE_cold(if_obj);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ ir_CONST_ADDR(ZSTR_VAL(name)));
+
+ if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) != IS_REG) {
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ }
+
+ ir_END_list(end_inputs);
+
+ ir_IF_TRUE(if_obj);
+ }
+ }
+ obj_ref = jit_Z_PTR(jit, op1_addr);
+ }
+
+ ZEND_ASSERT(obj_ref);
+ if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
+ prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
+ if (prop_info) {
+ ce = trace_ce;
+ ce_is_instanceof = 0;
+ if (!(op1_info & MAY_BE_CLASS_GUARD)) {
+ if (on_this && JIT_G(current_frame)
+ && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
+ ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
+ } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
+ if (on_this && JIT_G(current_frame)) {
+ JIT_G(current_frame)->ce = ce;
+ TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
+ }
+ } else {
+ return 0;
+ }
+ if (ssa->var_info && ssa_op->op1_use >= 0) {
+ ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_use].ce = ce;
+ ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
+ }
+ if (ssa->var_info && ssa_op->op1_def >= 0) {
+ ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_def].ce = ce;
+ ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
+ }
+ }
+ }
+ }
+
+ if (!prop_info) {
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
+ ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
+
+ ir_IF_FALSE_cold(if_same);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_same);
+ ir_ref offset_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
+
+ ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
+ ir_IF_TRUE_cold(if_dynamic);
+ ir_END_list(slow_inputs);
+
+ ir_IF_FALSE(if_dynamic);
+ prop_ref = ir_ADD_A(obj_ref, offset_ref);
+ ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_def);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
+ ir_ref arg3, arg4;
+ ir_ref prop_info_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
+ ir_ref if_has_prop_info = ir_IF(prop_info_ref);
+ ir_IF_TRUE_cold(if_has_prop_info);
+
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg3 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg3 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+
+ if (!RETURN_VALUE_USED(opline)) {
+ arg4 = IR_NULL;
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ arg4 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg4 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+ // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
+ prop_ref,
+ prop_info_ref,
+ arg3,
+ arg4);
+
+ if ((opline+1)->op1_type == IS_CONST) {
+ // TODO: ???
+ // if (Z_TYPE_P(value) == orig_type) {
+ // CACHE_PTR_EX(cache_slot + 2, NULL);
+ }
+
+ ir_END_list(end_inputs);
+ ir_IF_FALSE(if_has_prop_info);
+ }
+ } else {
+ prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+ /* With the exception of __clone(), readonly assignment always happens on IS_UNDEF, doding
+ * the fast path. Thus, the fast path is not useful. */
+ if (prop_info->flags & ZEND_ACC_READONLY) {
+ ZEND_ASSERT(slow_inputs == IR_UNUSED);
+ goto slow_path;
+ }
+ // Undefined property with potential magic __get()/__set() or lazy object
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ if (ZEND_TYPE_IS_SET(prop_info->type)) {
+ ir_ref ref, arg3, arg4;
+
+ // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
+ jit_SET_EX_OPLINE(jit, opline);
+ if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
+ ref = ir_CONST_ADDR(prop_info);
+ } else {
+ int prop_info_offset =
+ (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
+
+ ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
+ }
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg3 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg3 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+ if (!RETURN_VALUE_USED(opline)) {
+ arg4 = IR_NULL;
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ arg4 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg4 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
+ prop_ref,
+ ref,
+ arg3,
+ arg4);
+
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
+ if (Z_MODE(val_addr) != IS_REG
+ && (res_addr == 0 || Z_MODE(res_addr) != IS_REG)
+ && opline->result_type == IS_UNUSED) {
+ if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ } else {
+ zend_jit_addr real_res_addr;
+
+ if (res_addr && Z_MODE(res_addr) == IS_REG) {
+ real_res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ } else {
+ real_res_addr = res_addr;
+ }
+ if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, real_res_addr, 0, 0)) {
+ return 0;
+ }
+ }
+ if (end_inputs || slow_inputs) {
+ if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
+ && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ /* skip FREE_OP_DATA() */
+ delayed_end_input = ir_END();
+ } else {
+ ir_END_list(end_inputs);
+ }
+ }
+ }
+
+ if (slow_inputs) {
+ ir_ref arg3, arg5;
+
+ ir_MERGE_list(slow_inputs);
+
+slow_path:
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg3 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg3 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+ if (!RETURN_VALUE_USED(opline)) {
+ arg5 = IR_NULL;
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ arg5 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg5 = jit_ZVAL_ADDR(jit, res_addr);
+ }
+
+ // JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper),
+ obj_ref,
+ ir_CONST_ADDR(name),
+ arg3,
+ ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
+ arg5);
+
+ ir_END_list(end_inputs);
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+
+ if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ val_info |= MAY_BE_RC1|MAY_BE_RCN;
+ }
+ jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
+
+ if (delayed_end_input) {
+ ir_MERGE_WITH(delayed_end_input);
+ }
+ }
+
+ if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+
+ if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
+ return 0;
+ }
+ }
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_assign_obj_op(zend_jit_ctx *jit,
+ const zend_op *opline,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ uint32_t val_info,
+ zend_jit_addr val_addr,
+ zend_ssa_range *val_range,
+ bool op1_indirect,
+ zend_class_entry *ce,
+ bool ce_is_instanceof,
+ bool on_this,
+ bool delayed_fetch_this,
+ zend_class_entry *trace_ce,
+ uint8_t prop_type)
+{
+ zval *member;
+ zend_string *name;
+ zend_property_info *prop_info;
+ zend_jit_addr prop_addr;
+ bool use_prop_guard = 0;
+ bool may_throw = 0;
+ binary_op_type binary_op = get_binary_op(opline->extended_value);
+ ir_ref obj_ref = IR_UNUSED;
+ ir_ref prop_ref = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+
+ ZEND_ASSERT(opline->op2_type == IS_CONST);
+ ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
+ ZEND_ASSERT(opline->result_type == IS_UNUSED);
+
+ member = RT_CONSTANT(opline, opline->op2);
+ ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
+ name = Z_STR_P(member);
+ prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
+
+ if (on_this) {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ obj_ref = jit_Z_PTR(jit, this_addr);
+ } else {
+ if (opline->op1_type == IS_VAR
+ && (op1_info & MAY_BE_INDIRECT)
+ && Z_REG(op1_addr) == ZREG_FP) {
+ op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
+ } else {
+ ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+ ir_IF_FALSE_cold(if_obj);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID,
+ (op1_info & MAY_BE_UNDEF) ?
+ ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) :
+ ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ ir_CONST_ADDR(ZSTR_VAL(name)));
+
+ may_throw = 1;
+
+ ir_END_list(end_inputs);
+ ir_IF_TRUE(if_obj);
+ }
+ }
+ obj_ref = jit_Z_PTR(jit, op1_addr);
+ }
+
+ ZEND_ASSERT(obj_ref);
+ if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
+ prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
+ if (prop_info) {
+ ce = trace_ce;
+ ce_is_instanceof = 0;
+ if (!(op1_info & MAY_BE_CLASS_GUARD)) {
+ if (on_this && JIT_G(current_frame)
+ && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
+ ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
+ } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
+ if (on_this && JIT_G(current_frame)) {
+ JIT_G(current_frame)->ce = ce;
+ TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
+ }
+ } else {
+ return 0;
+ }
+ if (ssa->var_info && ssa_op->op1_use >= 0) {
+ ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_use].ce = ce;
+ ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
+ }
+ if (ssa->var_info && ssa_op->op1_def >= 0) {
+ ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_def].ce = ce;
+ ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
+ }
+ }
+ }
+ }
+
+ use_prop_guard = (prop_type != IS_UNKNOWN
+ && prop_type != IS_UNDEF
+ && prop_type != IS_REFERENCE
+ && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
+
+ if (Z_MODE(val_addr) == IS_REG
+ && Z_LOAD(val_addr)
+ && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
+ /* Force load */
+ zend_jit_use_reg(jit, val_addr);
+ }
+
+ if (!prop_info) {
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
+ ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
+
+ ir_IF_FALSE_cold(if_same);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_same);
+ if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
+ ir_ref prop_info_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
+ ir_ref if_has_prop_info = ir_IF(prop_info_ref);
+ ir_IF_TRUE_cold(if_has_prop_info);
+ ir_END_list(slow_inputs);
+
+ ir_IF_FALSE(if_has_prop_info);
+ }
+ ir_ref offset_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
+
+ ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
+ ir_IF_TRUE_cold(if_dynamic);
+ ir_END_list(slow_inputs);
+
+ ir_IF_FALSE(if_dynamic);
+
+ prop_ref = ir_ADD_A(obj_ref, offset_ref);
+ if (!use_prop_guard) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_def);
+ }
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+ } else {
+ prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ }
+ if (ZEND_TYPE_IS_SET(prop_info->type)) {
+ ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
+
+ may_throw = 1;
+
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg2 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+
+ if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
+ ir_IF_FALSE(if_ref);
+ noref_path = ir_END();
+ ir_IF_TRUE(if_ref);
+
+ reference = jit_Z_PTR(jit, prop_addr);
+ ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
+ if_typed = jit_if_TYPED_REF(jit, reference);
+ ir_IF_FALSE(if_typed);
+ ref_path = ir_END();
+ ir_IF_TRUE_cold(if_typed);
+
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
+ reference,
+ arg2,
+ ir_CONST_FC_FUNC(binary_op));
+
+ ir_END_list(end_inputs);
+
+ ir_MERGE_2(noref_path, ref_path);
+ prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
+ if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
+ ref = ir_CONST_ADDR(prop_info);
+ } else {
+ int prop_info_offset =
+ (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
+
+ ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
+ }
+
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop),
+ prop_ref,
+ ref,
+ arg2,
+ ir_CONST_FC_FUNC(binary_op));
+
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
+ zend_jit_addr var_addr = prop_addr;
+ uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
+ uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
+
+ if (use_prop_guard) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
+ var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
+ }
+
+ if (var_info & MAY_BE_REF) {
+ ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
+
+ may_throw = 1;
+
+ if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
+ ir_IF_FALSE(if_ref);
+ noref_path = ir_END();
+ ir_IF_TRUE(if_ref);
+
+ reference = jit_Z_PTR(jit, var_addr);
+ ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
+ if_typed = jit_if_TYPED_REF(jit, reference);
+ ir_IF_FALSE(if_typed);
+ ref_path = ir_END();
+ ir_IF_TRUE_cold(if_typed);
+
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg2 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg2 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
+ reference,
+ arg2,
+ ir_CONST_FC_FUNC(binary_op));
+
+ ir_END_list(end_inputs);
+
+ ir_MERGE_2(noref_path, ref_path);
+ prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
+ var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ var_info &= ~MAY_BE_REF;
+ }
+
+ uint8_t val_op_type = (opline+1)->op1_type;
+ if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
+ /* prevent FREE_OP in the helpers */
+ val_op_type = IS_CV;
+ }
+
+ switch (opline->extended_value) {
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
+ (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if (opline->extended_value != ZEND_ADD ||
+ (var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
+ (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
+ may_throw = 1;
+ }
+ }
+ if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
+ 1 /* may overflow */, 0)) {
+ return 0;
+ }
+ break;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
+ (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
+ (val_info & MAY_BE_ANY) != MAY_BE_STRING) {
+ may_throw = 1;
+ }
+ }
+ goto long_math;
+ case ZEND_SL:
+ case ZEND_SR:
+ if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
+ (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ may_throw = 1;
+ }
+ if (val_op_type != IS_CONST ||
+ Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
+ Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
+ may_throw = 1;
+ }
+ goto long_math;
+ case ZEND_MOD:
+ if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
+ (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ may_throw = 1;
+ }
+ if (val_op_type != IS_CONST ||
+ Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
+ Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
+ may_throw = 1;
+ }
+long_math:
+ if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
+ IS_CV, opline->op1, var_addr, var_info, NULL,
+ val_op_type, (opline+1)->op1, val_addr, val_info,
+ val_range,
+ 0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
+ return 0;
+ }
+ break;
+ case ZEND_CONCAT:
+ may_throw = 1;
+ if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr,
+ 0)) {
+ return 0;
+ }
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ if (end_inputs || slow_inputs) {
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (slow_inputs) {
+ ir_ref arg3;
+
+ ir_MERGE_list(slow_inputs);
+
+ may_throw = 1;
+
+ if (Z_MODE(val_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
+ if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
+ return 0;
+ }
+ arg3 = jit_ZVAL_ADDR(jit, real_addr);
+ } else {
+ arg3 = jit_ZVAL_ADDR(jit, val_addr);
+ }
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper),
+ obj_ref,
+ ir_CONST_ADDR(name),
+ arg3,
+ ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
+ ir_CONST_FC_FUNC(binary_op));
+
+ ir_END_list(end_inputs);
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+
+ if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ val_info |= MAY_BE_RC1|MAY_BE_RCN;
+ }
+
+ // JIT: FREE_OP_DATA();
+ jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
+
+ if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
+ if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
+ may_throw = 1;
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_incdec_obj(zend_jit_ctx *jit,
+ const zend_op *opline,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ uint32_t op1_info,
+ zend_jit_addr op1_addr,
+ bool op1_indirect,
+ zend_class_entry *ce,
+ bool ce_is_instanceof,
+ bool on_this,
+ bool delayed_fetch_this,
+ zend_class_entry *trace_ce,
+ uint8_t prop_type)
+{
+ zval *member;
+ zend_string *name;
+ zend_property_info *prop_info;
+ zend_jit_addr res_addr = 0;
+ zend_jit_addr prop_addr;
+ bool use_prop_guard = 0;
+ bool may_throw = 0;
+ uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
+ ir_ref obj_ref = IR_UNUSED;
+ ir_ref prop_ref = IR_UNUSED;
+ ir_ref end_inputs = IR_UNUSED;
+ ir_ref slow_inputs = IR_UNUSED;
+
+ ZEND_ASSERT(opline->op2_type == IS_CONST);
+ ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
+
+ if (opline->result_type != IS_UNUSED) {
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ }
+
+ member = RT_CONSTANT(opline, opline->op2);
+ ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
+ name = Z_STR_P(member);
+ prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
+
+ if (on_this) {
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ obj_ref = jit_Z_PTR(jit, this_addr);
+ } else {
+ if (opline->op1_type == IS_VAR
+ && (op1_info & MAY_BE_INDIRECT)
+ && Z_REG(op1_addr) == ZREG_FP) {
+ op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
+ } else {
+ ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
+ ir_IF_FALSE_cold(if_obj);
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec),
+ jit_ZVAL_ADDR(jit, op1_addr),
+ ir_CONST_ADDR(ZSTR_VAL(name)));
+
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
+ ir_IF_TRUE(if_obj);
+ }
+ }
+ obj_ref = jit_Z_PTR(jit, op1_addr);
+ }
+
+ ZEND_ASSERT(obj_ref);
+ if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
+ prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
+ if (prop_info) {
+ ce = trace_ce;
+ ce_is_instanceof = 0;
+ if (!(op1_info & MAY_BE_CLASS_GUARD)) {
+ if (on_this && JIT_G(current_frame)
+ && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
+ ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
+ } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
+ if (on_this && JIT_G(current_frame)) {
+ JIT_G(current_frame)->ce = ce;
+ TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
+ }
+ } else {
+ return 0;
+ }
+ if (ssa->var_info && ssa_op->op1_use >= 0) {
+ ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_use].ce = ce;
+ ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
+ }
+ if (ssa->var_info && ssa_op->op1_def >= 0) {
+ ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op1_def].ce = ce;
+ ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
+ }
+ }
+ }
+ }
+
+ use_prop_guard = (prop_type != IS_UNKNOWN
+ && prop_type != IS_UNDEF
+ && prop_type != IS_REFERENCE
+ && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
+
+ if (!prop_info) {
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
+ ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
+
+ ir_IF_FALSE_cold(if_same);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_same);
+ if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
+ ir_ref prop_info_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
+ ir_ref if_has_prop_info = ir_IF(prop_info_ref);
+ ir_IF_TRUE_cold(if_has_prop_info);
+ ir_END_list(slow_inputs);
+
+ ir_IF_FALSE(if_has_prop_info);
+ }
+ ir_ref offset_ref = ir_LOAD_A(
+ ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
+
+ ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
+ ir_IF_TRUE_cold(if_dynamic);
+ ir_END_list(slow_inputs);
+
+ ir_IF_FALSE(if_dynamic);
+
+ prop_ref = ir_ADD_A(obj_ref, offset_ref);
+ if (!use_prop_guard) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+
+ ir_IF_TRUE(if_def);
+ }
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+ } else {
+ prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
+ prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
+ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
+ ir_IF_FALSE_cold(if_def);
+ ir_END_list(slow_inputs);
+ ir_IF_TRUE(if_def);
+ }
+ }
+
+ if (ZEND_TYPE_IS_SET(prop_info->type)) {
+ const void *func;
+ ir_ref ref;
+
+ may_throw = 1;
+ jit_SET_EX_OPLINE(jit, opline);
+
+ if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
+ ref = ir_CONST_ADDR(prop_info);
+ } else {
+ int prop_info_offset =
+ (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
+
+ ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
+ ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
+ }
+
+ if (opline->result_type == IS_UNUSED) {
+ switch (opline->opcode) {
+ case ZEND_PRE_INC_OBJ:
+ case ZEND_POST_INC_OBJ:
+ func = zend_jit_inc_typed_prop;
+ break;
+ case ZEND_PRE_DEC_OBJ:
+ case ZEND_POST_DEC_OBJ:
+ func = zend_jit_dec_typed_prop;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref);
+ } else {
+ switch (opline->opcode) {
+ case ZEND_PRE_INC_OBJ:
+ func = zend_jit_pre_inc_typed_prop;
+ break;
+ case ZEND_PRE_DEC_OBJ:
+ func = zend_jit_pre_dec_typed_prop;
+ break;
+ case ZEND_POST_INC_OBJ:
+ func = zend_jit_post_inc_typed_prop;
+ break;
+ case ZEND_POST_DEC_OBJ:
+ func = zend_jit_post_dec_typed_prop;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+ ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func),
+ prop_ref,
+ ref,
+ jit_ZVAL_ADDR(jit, res_addr));
+ }
+ ir_END_list(end_inputs);
+ }
+ }
+
+ if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
+ uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
+ zend_jit_addr var_addr = prop_addr;
+ ir_ref if_long = IR_UNUSED;
+ ir_ref if_overflow = IR_UNUSED;
+
+ if (use_prop_guard) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
+ var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
+ }
+
+ if (var_info & MAY_BE_REF) {
+ const void *func;
+ ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
+
+ if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
+ ir_IF_FALSE(if_ref);
+ noref_path = ir_END();
+ ir_IF_TRUE(if_ref);
+
+ reference = jit_Z_PTR(jit, var_addr);
+ ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
+ if_typed = jit_if_TYPED_REF(jit, reference);
+ ir_IF_FALSE(if_typed);
+ ref_path = ir_END();
+ ir_IF_TRUE_cold(if_typed);
+
+ switch (opline->opcode) {
+ case ZEND_PRE_INC_OBJ:
+ func = zend_jit_pre_inc_typed_ref;
+ break;
+ case ZEND_PRE_DEC_OBJ:
+ func = zend_jit_pre_dec_typed_ref;
+ break;
+ case ZEND_POST_INC_OBJ:
+ func = zend_jit_post_inc_typed_ref;
+ break;
+ case ZEND_POST_DEC_OBJ:
+ func = zend_jit_post_dec_typed_ref;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ may_throw = 1;
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func),
+ reference,
+ (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
+
+ ir_END_list(end_inputs);
+
+ ir_MERGE_2(noref_path, ref_path);
+ prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
+ var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
+
+ var_info &= ~MAY_BE_REF;
+ }
+
+ if (var_info & MAY_BE_LONG) {
+ ir_ref addr, ref;
+
+ if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
+ if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
+ ir_IF_TRUE(if_long);
+ }
+
+ addr = jit_ZVAL_ADDR(jit, var_addr);
+ ref = ir_LOAD_L(addr);
+ if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
+ if (opline->result_type != IS_UNUSED) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
+ ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1));
+ } else {
+ ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1));
+ }
+
+ ir_STORE(addr, ref);
+ if_overflow = ir_IF(ir_OVERFLOW(ref));
+ ir_IF_FALSE(if_overflow);
+
+ if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
+ if (opline->result_type != IS_UNUSED) {
+ jit_set_Z_LVAL(jit, res_addr, ref);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
+ }
+ }
+ ir_END_list(end_inputs);
+ }
+
+ if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
+ if (if_long) {
+ ir_IF_FALSE_cold(if_long);
+ }
+ if (var_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ may_throw = 1;
+ jit_SET_EX_OPLINE(jit, opline);
+ }
+ if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
+ jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
+ }
+ if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
+ if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc),
+ jit_ZVAL_ADDR(jit, var_addr),
+ jit_ZVAL_ADDR(jit, res_addr));
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function),
+ jit_ZVAL_ADDR(jit, var_addr));
+ }
+ } else {
+ if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
+ ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec),
+ jit_ZVAL_ADDR(jit, var_addr),
+ jit_ZVAL_ADDR(jit, res_addr));
+ } else {
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function),
+ jit_ZVAL_ADDR(jit, var_addr));
+ }
+ }
+
+ ir_END_list(end_inputs);
+ }
+ if (var_info & MAY_BE_LONG) {
+ ir_IF_TRUE_cold(if_overflow);
+ if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
+ jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
+#else
+ jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
+ if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ } else {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
+ jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
+#else
+ jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
+ if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
+#if SIZEOF_ZEND_LONG == 4
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
+ jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
+#else
+ jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
+#endif
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
+ }
+ }
+ if (opline->result_type != IS_UNUSED
+ && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
+ && prop_info
+ && !ZEND_TYPE_IS_SET(prop_info->type)
+ && (res_info & MAY_BE_GUARD)
+ && (res_info & MAY_BE_LONG)) {
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+ uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ int32_t exit_point;
+ const void *exit_addr;
+
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ } else {
+ ir_END_list(end_inputs);
+ }
+ }
+ }
+
+ if (slow_inputs) {
+ const void *func;
+
+ ir_MERGE_list(slow_inputs);
+
+ // JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result);
+ switch (opline->opcode) {
+ case ZEND_PRE_INC_OBJ:
+ func = zend_jit_pre_inc_obj_helper;
+ break;
+ case ZEND_PRE_DEC_OBJ:
+ func = zend_jit_pre_dec_obj_helper;
+ break;
+ case ZEND_POST_INC_OBJ:
+ func = zend_jit_post_inc_obj_helper;
+ break;
+ case ZEND_POST_DEC_OBJ:
+ func = zend_jit_post_dec_obj_helper;
+ break;
+ default:
+ ZEND_UNREACHABLE();
+ }
+
+ may_throw = 1;
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
+ ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func),
+ obj_ref,
+ ir_CONST_ADDR(name),
+ ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
+ (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
+
+ ir_END_list(end_inputs);
+ }
+
+ if (end_inputs) {
+ ir_MERGE_list(end_inputs);
+ }
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) {
+ if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
+ may_throw = 1;
+ }
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
+ }
+
+ if (may_throw) {
+ zend_jit_check_exception(jit);
+ }
+
+ return 1;
+}
+
+static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
+{
+ HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
+ const zend_op *next_opline = NULL;
+ ir_refs *slow_inputs;
+
+ ir_refs_init(slow_inputs, 8);
+
+ if (trace) {
+ ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
+ ZEND_ASSERT(trace->opline != NULL);
+ next_opline = trace->opline;
+ }
+
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ zval *jump_zv = NULL;
+ const zend_op *target;
+ int b;
+
+ /* Wrong type for SWITCH_LONG/SWITCH_STRING falls back to the ZEND_CASE chain */
+ target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
+ if (opline->opcode == ZEND_SWITCH_LONG) {
+ if (Z_TYPE_P(zv) == IS_LONG) {
+ jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
+ } else {
+ target = opline + 1;
+ }
+ } else if (opline->opcode == ZEND_SWITCH_STRING) {
+ if (Z_TYPE_P(zv) == IS_STRING) {
+ jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
+ } else {
+ target = opline + 1;
+ }
+ } else if (opline->opcode == ZEND_MATCH) {
+ if (Z_TYPE_P(zv) == IS_LONG) {
+ jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
+ } else if (Z_TYPE_P(zv) == IS_STRING) {
+ jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ if (jump_zv != NULL) {
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
+ }
+ if (next_opline) {
+ ZEND_ASSERT(target == next_opline);
+ } else {
+ b = ssa->cfg.map[target - op_array->opcodes];
+ _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
+ jit->b = -1;
+ }
+ } else {
+ zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
+ uint32_t op1_info = OP1_INFO();
+ zend_jit_addr op1_addr = OP1_ADDR();
+ const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
+ const zend_op *target;
+ int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
+ int b;
+ int32_t exit_point;
+ const void *exit_addr;
+ const void *default_label = NULL;
+ zval *zv;
+
+ if (next_opline) {
+ if (next_opline != default_opline) {
+ exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
+ default_label = zend_jit_trace_get_exit_addr(exit_point);
+ if (!default_label) {
+ return 0;
+ }
+ }
+ }
+
+ if (opline->opcode == ZEND_SWITCH_LONG) {
+ if (op1_info & MAY_BE_LONG) {
+ const void *fallback_label = NULL;
+
+ if (next_opline) {
+ if (next_opline != opline + 1) {
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ fallback_label = zend_jit_trace_get_exit_addr(exit_point);
+ if (!fallback_label) {
+ return 0;
+ }
+ }
+ }
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref, if_long, fast_path, ref2;
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_long);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_long);
+
+ // JIT: ZVAL_DEREF(op)
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
+ } else {
+ ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
+ ir_IF_FALSE_cold(if_ref);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_ref);
+ }
+
+ ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
+
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
+ } else {
+ if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_long);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_long);
+ }
+
+ ir_MERGE_2(fast_path, ir_END());
+ ref = ir_PHI_2(IR_ADDR, ref, ref2);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
+ } else {
+ ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE_cold(if_long);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_long);
+ }
+ }
+ ir_ref ref = jit_Z_LVAL(jit, op1_addr);
+
+ if (!HT_IS_PACKED(jumptable)) {
+ ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
+ ir_CONST_ADDR(jumptable), ref);
+ ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
+ /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
+ if (sizeof(Bucket) == 32) {
+ ref = ir_SHR_L(ref, ir_CONST_LONG(5));
+ } else {
+ ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
+ }
+ }
+ ref = ir_SWITCH(ref);
+
+ if (next_opline) {
+ ir_ref continue_list = IR_UNUSED;
+
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ ir_ref idx;
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+
+ if (HT_IS_PACKED(jumptable)) {
+ idx = ir_CONST_LONG(zv - jumptable->arPacked);
+ } else {
+ idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
+ }
+ ir_CASE_VAL(ref, idx);
+ if (target == next_opline) {
+ ir_END_list(continue_list);
+ } else {
+ exit_point = zend_jit_trace_get_exit_point(target, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } ZEND_HASH_FOREACH_END();
+
+ ir_CASE_DEFAULT(ref);
+ if (next_opline == default_opline) {
+ ir_END_list(continue_list);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
+ }
+ if (continue_list) {
+ ir_MERGE_list(continue_list);
+ } else {
+ ZEND_ASSERT(slow_inputs->count);
+ ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
+ }
+ } else {
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+ b = ssa->cfg.map[target - op_array->opcodes];
+ _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
+ } ZEND_HASH_FOREACH_END();
+
+ _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
+ if (slow_inputs->count) {
+ ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
+ _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
+ }
+ jit->b = -1;
+ }
+ } else if (!next_opline) {
+ _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
+ jit->b = -1;
+ }
+ } else if (opline->opcode == ZEND_SWITCH_STRING) {
+ if (op1_info & MAY_BE_STRING) {
+ const void *fallback_label = NULL;
+
+ if (next_opline) {
+ if (next_opline != opline + 1) {
+ exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
+ fallback_label = zend_jit_trace_get_exit_addr(exit_point);
+ if (!fallback_label) {
+ return 0;
+ }
+ }
+ }
+ if (op1_info & MAY_BE_REF) {
+ ir_ref ref, if_string, fast_path, ref2;
+
+ ref = jit_ZVAL_ADDR(jit, op1_addr);
+ if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_TRUE(if_string);
+ fast_path = ir_END();
+ ir_IF_FALSE_cold(if_string);
+
+ // JIT: ZVAL_DEREF(op)
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
+ } else {
+ ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_FALSE_cold(if_ref);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_ref);
+ }
+
+ ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
+
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
+ } else {
+ if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_FALSE_cold(if_string);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_string);
+ }
+
+ ir_MERGE_2(fast_path, ir_END());
+ ref = ir_PHI_2(IR_ADDR, ref, ref2);
+ op1_addr = ZEND_ADDR_REF_ZVAL(ref);
+ } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
+ if (fallback_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
+ } else {
+ ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_FALSE_cold(if_string);
+ ir_refs_add(slow_inputs, ir_END());
+ ir_IF_TRUE(if_string);
+ }
+ }
+
+ ir_ref ref = jit_Z_PTR(jit, op1_addr);
+ ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
+ ir_CONST_ADDR(jumptable), ref);
+ ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
+ /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
+ if (sizeof(Bucket) == 32) {
+ ref = ir_SHR_L(ref, ir_CONST_LONG(5));
+ } else {
+ ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
+ }
+ ref = ir_SWITCH(ref);
+
+ if (next_opline) {
+ ir_ref continue_list = IR_UNUSED;
+
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ ir_ref idx;
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+
+ if (HT_IS_PACKED(jumptable)) {
+ idx = ir_CONST_LONG(zv - jumptable->arPacked);
+ } else {
+ idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
+ }
+ ir_CASE_VAL(ref, idx);
+ if (target == next_opline) {
+ ir_END_list(continue_list);
+ } else {
+ exit_point = zend_jit_trace_get_exit_point(target, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } ZEND_HASH_FOREACH_END();
+
+ ir_CASE_DEFAULT(ref);
+ if (next_opline == default_opline) {
+ ir_END_list(continue_list);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
+ }
+ if (continue_list) {
+ ir_MERGE_list(continue_list);
+ } else {
+ ZEND_ASSERT(slow_inputs->count);
+ ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
+ }
+ } else {
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+ b = ssa->cfg.map[target - op_array->opcodes];
+ _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
+ } ZEND_HASH_FOREACH_END();
+ _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
+ if (slow_inputs->count) {
+ ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
+ _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
+ }
+ jit->b = -1;
+ }
+ } else if (!next_opline) {
+ _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
+ jit->b = -1;
+ }
+ } else if (opline->opcode == ZEND_MATCH) {
+ ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED;
+ ir_ref continue_list = IR_UNUSED;
+
+ if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
+ ir_ref long_path = IR_UNUSED;
+
+ if (op1_info & MAY_BE_REF) {
+ op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
+ }
+ if (op1_info & MAY_BE_LONG) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_TRUE(if_type);
+ } else if (default_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
+ } else if (next_opline) {
+ ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE(if_type);
+ ir_END_list(continue_list);
+ ir_IF_TRUE(if_type);
+ } else {
+ ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
+ ir_IF_FALSE(if_type);
+ ir_END_list(default_input_list);
+ ir_IF_TRUE(if_type);
+ }
+ }
+ ref = jit_Z_LVAL(jit, op1_addr);
+ ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
+ ir_CONST_ADDR(jumptable), ref);
+ if (op1_info & MAY_BE_STRING) {
+ long_path = ir_END();
+ }
+ }
+ if (op1_info & MAY_BE_STRING) {
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ if_type = IS_UNUSED;
+ }
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
+ if (op1_info & MAY_BE_UNDEF) {
+ if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_TRUE(if_type);
+ } else if (default_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
+ } else if (next_opline) {
+ ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_FALSE(if_type);
+ ir_END_list(continue_list);
+ ir_IF_TRUE(if_type);
+ } else {
+ ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
+ ir_IF_FALSE(if_type);
+ ir_END_list(default_input_list);
+ ir_IF_TRUE(if_type);
+ }
+ }
+ ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
+ ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
+ ir_CONST_ADDR(jumptable), ref2);
+ if (op1_info & MAY_BE_LONG) {
+ ir_MERGE_WITH(long_path);
+ ref = ir_PHI_2(IR_LONG, ref2, ref);
+ } else {
+ ref = ref2;
+ }
+ }
+
+ ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
+ /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
+ if (HT_IS_PACKED(jumptable)) {
+ ZEND_ASSERT(sizeof(zval) == 16);
+ ref = ir_SHR_L(ref, ir_CONST_LONG(4));
+ } else {
+ if (sizeof(Bucket) == 32) {
+ ref = ir_SHR_L(ref, ir_CONST_LONG(5));
+ } else {
+ ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
+ }
+ }
+ ref = ir_SWITCH(ref);
+
+ if (next_opline) {
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ ir_ref idx;
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+
+ if (HT_IS_PACKED(jumptable)) {
+ idx = ir_CONST_LONG(zv - jumptable->arPacked);
+ } else {
+ idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
+ }
+ ir_CASE_VAL(ref, idx);
+ if (target == next_opline) {
+ ir_END_list(continue_list);
+ } else {
+ exit_point = zend_jit_trace_get_exit_point(target, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
+ }
+ } ZEND_HASH_FOREACH_END();
+
+ ir_CASE_DEFAULT(ref);
+ if (next_opline == default_opline) {
+ ir_END_list(continue_list);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
+ }
+ } else {
+ ZEND_HASH_FOREACH_VAL(jumptable, zv) {
+ target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
+ b = ssa->cfg.map[target - op_array->opcodes];
+ _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
+ } ZEND_HASH_FOREACH_END();
+ _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
+ }
+ } else if (!(op1_info & MAY_BE_UNDEF)) {
+ if (next_opline) {
+ if (next_opline == default_opline) {
+ ir_END_list(continue_list);
+ } else {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
+ }
+ } else {
+ _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
+ }
+ }
+
+ if (op1_info & MAY_BE_UNDEF) {
+ if (if_type) {
+ ir_IF_FALSE(if_type);
+ if_type = IS_UNUSED;
+ }
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
+ if (default_label) {
+ jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
+ } else if (next_opline) {
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
+ ir_IF_TRUE(if_def);
+ ir_END_list(continue_list);
+ ir_IF_FALSE_cold(if_def);
+ } else {
+ ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
+ ir_IF_TRUE(if_def);
+ ir_END_list(default_input_list);
+ ir_IF_FALSE_cold(if_def);
+ }
+ }
+
+ jit_SET_EX_OPLINE(jit, opline);
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
+ ir_CONST_U32(opline->op1.var));
+ zend_jit_check_exception_undef_result(jit, opline);
+ if (default_label) {
+ jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
+ } else if (next_opline) {
+ ir_END_list(continue_list);
+ } else {
+ ir_END_list(default_input_list);
+ }
+ }
+ if (next_opline) {
+ ZEND_ASSERT(continue_list);
+ ir_MERGE_list(continue_list);
+ } else {
+ if (default_input_list) {
+ if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
+ ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
+ jit->ctx.ir_base[ref].op3 = default_input_list;
+ } else {
+ ir_MERGE_list(default_input_list);
+ _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
+ }
+ }
+ jit->b = -1;
+ }
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa)
+{
+ int i, count;
+ zend_basic_block *bb;
+
+ zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
+
+ jit->ctx.spill_base = ZREG_FP;
+
+ jit->op_array = jit->current_op_array = op_array;
+ jit->ssa = ssa;
+ jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
+ jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
+
+ count = 0;
+ for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) {
+ jit->bb_predecessors[i] = count;
+ count += bb->predecessors_count;
+ }
+ jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
+
+ if (!GCC_GLOBAL_REGS) {
+ ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
+ jit_STORE_FP(jit, ref);
+ jit->ctx.flags |= IR_FASTCALL_FUNC;
+ }
+
+ return 1;
+}
+
+static void *zend_jit_finish(zend_jit_ctx *jit)
+{
+ void *entry;
+ size_t size;
+ zend_string *str = NULL;
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP|
+ ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP|
+ ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
+ if (jit->name) {
+ str = zend_string_copy(jit->name);
+ } else {
+ str = zend_jit_func_name(jit->op_array);
+ }
+ }
+
+ if (jit->op_array) {
+ /* Only for function JIT */
+ _zend_jit_fix_merges(jit);
+#if defined(IR_TARGET_AARCH64)
+ } else if (jit->trace) {
+ jit->ctx.deoptimization_exits = jit->trace->exit_count;
+ jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
+#endif
+ } else {
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+ jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
+#endif
+ }
+
+ entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
+ if (entry) {
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
+#ifdef HAVE_CAPSTONE
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
+ if (str) {
+ ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size);
+ }
+ ir_disasm(str ? ZSTR_VAL(str) : "unknown",
+ entry, size,
+ (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0,
+ &jit->ctx, stderr);
+ }
+#endif
+#ifndef _WIN32
+ if (str) {
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
+ uintptr_t sp_offset = 0;
+
+// ir_mem_unprotect(entry, size);
+ if (!(jit->ctx.flags & IR_FUNCTION)
+ && zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
+ sp_offset = zend_jit_hybrid_vm_sp_adj;
+#else
+ sp_offset = sizeof(void*);
+#endif
+ } else {
+ sp_offset = sizeof(void*);
+ }
+ ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0);
+// ir_mem_protect(entry, size);
+ }
+
+ if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ ir_perf_map_register(ZSTR_VAL(str), entry, size);
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
+ ir_perf_jitdump_register(ZSTR_VAL(str), entry, size);
+ }
+ }
+ }
+#endif
+ }
+
+ if (jit->op_array) {
+ /* Only for function JIT */
+ const zend_op_array *op_array = jit->op_array;
+ zend_op *opline = (zend_op*)op_array->opcodes;
+
+ if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ while (opline->opcode == ZEND_RECV) {
+ opline++;
+ }
+ }
+ opline->handler = entry;
+
+ if (jit->ctx.entries_count) {
+ /* For all entries */
+ int i = jit->ctx.entries_count;
+ do {
+ ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
+ op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3;
+ } while (i != 0);
+ }
+ } else {
+ /* Only for tracing JIT */
+ zend_jit_trace_info *t = jit->trace;
+ zend_jit_trace_stack *stack;
+ uint32_t i;
+
+ if (t) {
+ for (i = 0; i < t->stack_map_size; i++) {
+ stack = t->stack_map + i;
+ if (stack->flags & ZREG_SPILL_SLOT) {
+ stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
+ stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
+ }
+ }
+ }
+
+ zend_jit_trace_add_code(entry, size);
+ }
+ }
+
+ if (str) {
+ zend_string_release(str);
+ }
+
+ return entry;
+}
+
+static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
+{
+ const void *entry;
+ size_t size;
+ ir_code_buffer code_buffer;
+
+ code_buffer.start = dasm_buf;
+ code_buffer.end = dasm_end;
+ code_buffer.pos = *dasm_ptr;
+
+ entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit],
+ &code_buffer, &size);
+
+ *dasm_ptr = code_buffer.pos;
+
+ if (entry) {
+#ifdef HAVE_CAPSTONE
+ if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
+ uint32_t i;
+ char name[32];
+
+ for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
+ snprintf(name, sizeof(name), "jit$$trace_exit_%d", n + i);
+ ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
+ }
+ }
+#endif
+ }
+
+ return entry;
+}
+
+static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
+
+ return 1;
+}
+
+static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
+
+ return 1;
+}
+
+static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr)
+{
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
+
+ return 1;
+}
+
+static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline)
+{
+ uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
+ zend_jit_set_last_valid_opline(jit, opline);
+
+ return 1;
+}
+
+static bool zend_jit_guard_reference(zend_jit_ctx *jit,
+ const zend_op *opline,
+ zend_jit_addr *var_addr_ptr,
+ zend_jit_addr *ref_addr_ptr,
+ bool add_ref_guard)
+{
+ zend_jit_addr var_addr = *var_addr_ptr;
+ const void *exit_addr = NULL;
+ ir_ref ref;
+
+ if (add_ref_guard) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ ref = jit_Z_TYPE(jit, var_addr);
+ ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
+ }
+
+ ref = jit_Z_PTR(jit, var_addr);
+ *ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref);
+ ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ *var_addr_ptr = var_addr;
+
+ return 1;
+}
+
+static bool zend_jit_fetch_reference(zend_jit_ctx *jit,
+ const zend_op *opline,
+ uint8_t var_type,
+ uint32_t *var_info_ptr,
+ zend_jit_addr *var_addr_ptr,
+ bool add_ref_guard,
+ bool add_type_guard)
+{
+ zend_jit_addr var_addr = *var_addr_ptr;
+ uint32_t var_info = *var_info_ptr;
+ const void *exit_addr = NULL;
+ ir_ref ref;
+
+ if (add_ref_guard || add_type_guard) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ }
+
+ if (add_ref_guard) {
+ ref = jit_Z_TYPE(jit, var_addr);
+ ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
+ }
+ if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
+ /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
+ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper),
+ jit_ZVAL_ADDR(jit, var_addr));
+ *var_addr_ptr = var_addr;
+ } else {
+ ref = jit_Z_PTR(jit, var_addr);
+ ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ *var_addr_ptr = var_addr;
+ }
+
+ if (var_type != IS_UNKNOWN) {
+ var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
+ }
+ if (add_type_guard
+ && var_type != IS_UNKNOWN
+ && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
+ ref = jit_Z_TYPE(jit, var_addr);
+ ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr));
+
+ ZEND_ASSERT(var_info & (1 << var_type));
+ if (var_type < IS_STRING) {
+ var_info = (1 << var_type);
+ } else if (var_type != IS_ARRAY) {
+ var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
+ } else {
+ var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
+ }
+
+ *var_info_ptr = var_info;
+ } else {
+ var_info &= ~MAY_BE_REF;
+ *var_info_ptr = var_info;
+ }
+ *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
+
+ return 1;
+}
+
+static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
+{
+ zend_jit_addr var_addr = *var_addr_ptr;
+ uint32_t var_info = *var_info_ptr;
+ int32_t exit_point;
+ const void *exit_addr;
+ ir_ref ref = IR_UNUSED;
+
+ if (add_indirect_guard) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
+ ref = jit_Z_PTR(jit, var_addr);
+ } else {
+ /* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W,
+ * is eliminated by store forwarding (S2L) */
+ ref = jit_Z_PTR(jit, var_addr);
+ }
+ *var_info_ptr &= ~MAY_BE_INDIRECT;
+ var_addr = ZEND_ADDR_REF_ZVAL(ref);
+ *var_addr_ptr = var_addr;
+
+ if (var_type != IS_UNKNOWN) {
+ var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
+ }
+ if (!(var_type & IS_TRACE_REFERENCE)
+ && var_type != IS_UNKNOWN
+ && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
+ exit_point = zend_jit_trace_get_exit_point(opline, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
+
+ //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
+ ZEND_ASSERT(var_info & (1 << var_type));
+ if (var_type < IS_STRING) {
+ var_info = (1 << var_type);
+ } else if (var_type != IS_ARRAY) {
+ var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
+ } else {
+ var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
+ }
+
+ *var_info_ptr = var_info;
+ }
+
+ return 1;
+}
+
+static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
+{
+ zend_jit_op_array_trace_extension *jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ size_t offset = jit_extension->offset;
+ const void *handler =
+ (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
+ ir_ref ref;
+
+ zend_jit_set_ip(jit, opline);
+ if (GCC_GLOBAL_REGS) {
+ ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
+ } else {
+ ref = jit_FP(jit);
+ ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
+ }
+ if (may_throw
+ && opline->opcode != ZEND_RETURN
+ && opline->opcode != ZEND_RETURN_BY_REF) {
+ zend_jit_check_exception(jit);
+ }
+
+ while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
+ trace++;
+ }
+
+ if (!GCC_GLOBAL_REGS
+ && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
+ if (opline->opcode == ZEND_RETURN ||
+ opline->opcode == ZEND_RETURN_BY_REF ||
+ opline->opcode == ZEND_DO_UCALL ||
+ opline->opcode == ZEND_DO_FCALL_BY_NAME ||
+ opline->opcode == ZEND_DO_FCALL ||
+ opline->opcode == ZEND_GENERATOR_CREATE) {
+
+ ir_ref addr = jit_EG(current_execute_data);
+
+ jit_STORE_FP(jit, ir_LOAD_A(addr));
+ }
+ }
+
+ if (zend_jit_trace_may_exit(op_array, opline)) {
+ if (opline->opcode == ZEND_RETURN ||
+ opline->opcode == ZEND_RETURN_BY_REF ||
+ opline->opcode == ZEND_GENERATOR_CREATE) {
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ if (trace->op != ZEND_JIT_TRACE_END ||
+ (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
+ trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
+ /* this check may be handled by the following OPLINE guard or jmp [IP] */
+ ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
+ jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ }
+ } else if (GCC_GLOBAL_REGS) {
+ ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ } else {
+ ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ }
+ } else if (opline->opcode == ZEND_GENERATOR_RETURN ||
+ opline->opcode == ZEND_YIELD ||
+ opline->opcode == ZEND_YIELD_FROM) {
+ ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
+ ir_BEGIN(IR_UNUSED); /* unreachable block */
+ }
+ if (trace->op != ZEND_JIT_TRACE_END ||
+ (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
+ trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
+
+ const zend_op *next_opline = trace->opline;
+ const zend_op *exit_opline = NULL;
+ uint32_t exit_point;
+ const void *exit_addr;
+ uint32_t old_info = 0;
+ uint32_t old_res_info = 0;
+ zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
+
+ if (zend_is_smart_branch(opline)) {
+ bool exit_if_true = 0;
+ exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
+ } else {
+ switch (opline->opcode) {
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_JMP_NULL:
+ case ZEND_FE_RESET_R:
+ case ZEND_FE_RESET_RW:
+ exit_opline = (trace->opline == opline + 1) ?
+ OP_JMP_ADDR(opline, opline->op2) :
+ opline + 1;
+ break;
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ exit_opline = (trace->opline == opline + 1) ?
+ ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
+ opline + 1;
+ break;
+
+ }
+ }
+
+ switch (opline->opcode) {
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ if (opline->op2_type != IS_UNUSED) {
+ old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
+ }
+ break;
+ case ZEND_FE_RESET_RW:
+ case ZEND_BIND_INIT_STATIC_OR_JMP:
+ if (opline->op1_type == IS_CV) {
+ old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
+ }
+ break;
+ }
+ if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
+ old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
+ SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
+ }
+ exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
+ }
+ switch (opline->opcode) {
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ if (opline->op2_type != IS_UNUSED) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
+ }
+ break;
+ case ZEND_FE_RESET_RW:
+ case ZEND_BIND_INIT_STATIC_OR_JMP:
+ if (opline->op1_type == IS_CV) {
+ SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
+ }
+ break;
+ }
+
+ if (!exit_addr) {
+ return 0;
+ }
+ ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
+ }
+ }
+
+ zend_jit_set_last_valid_opline(jit, trace->opline);
+
+ return 1;
+}
+
+static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
+ zend_string *name,
+ uint32_t trace_num,
+ zend_jit_trace_info *parent,
+ uint32_t exit_num)
+{
+ zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
+
+ jit->ctx.spill_base = ZREG_FP;
+
+ jit->op_array = NULL;
+ jit->ssa = NULL;
+ jit->name = zend_string_copy(name);
+
+ jit->ctx.flags |= IR_SKIP_PROLOGUE;
+
+ zend_jit_preserve_parent_regs(jit, NULL, parent, exit_num);
+
+ return 1;
+}
+
+static int zend_jit_trace_start(zend_jit_ctx *jit,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_string *name,
+ uint32_t trace_num,
+ zend_jit_trace_info *parent,
+ uint32_t exit_num)
+{
+ zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
+
+ jit->ctx.spill_base = ZREG_FP;
+
+ jit->op_array = NULL;
+ jit->current_op_array = op_array;
+ jit->ssa = ssa;
+ jit->name = zend_string_copy(name);
+
+ if (!GCC_GLOBAL_REGS) {
+ if (!parent) {
+ ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
+ jit_STORE_FP(jit, ref);
+ jit->ctx.flags |= IR_FASTCALL_FUNC;
+ }
+ }
+
+ if (parent) {
+ jit->ctx.flags |= IR_SKIP_PROLOGUE;
+ }
+
+ zend_jit_preserve_parent_regs(jit, ssa, parent, exit_num);
+
+ ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
+
+ return 1;
+}
+
+static void zend_jit_preserve_parent_regs(zend_jit_ctx *jit,
+ zend_ssa *ssa,
+ zend_jit_trace_info *parent,
+ uint32_t exit_num)
+{
+ /* Emit early RLOADs of registers used for deoptimization to prevent
+ * clobbering. zend_jit_deopt_rload() will reference these. */
+
+ if (parent) {
+ int i;
+ int parent_vars_count = parent->exit_info[exit_num].stack_size;
+ zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL :
+ parent->stack_map +
+ parent->exit_info[exit_num].stack_offset;
+
+ for (i = 0; i < parent_vars_count; i++) {
+ if (STACK_FLAGS(parent_stack, i) != ZREG_CONST
+ && STACK_REG(parent_stack, i) != ZREG_NONE) {
+ int32_t reg = STACK_REG(parent_stack, i);
+ ir_type type;
+
+ if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
+ type = IR_ADDR;
+ } else if (STACK_TYPE(parent_stack, i) == IS_LONG) {
+ type = IR_LONG;
+ } else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) {
+ type = IR_DOUBLE;
+ } else {
+ ZEND_UNREACHABLE();
+ }
+ if (ssa && ssa->vars[i].no_val) {
+ /* pass */
+ } else {
+ ir_ref ref = ir_RLOAD(type, reg);
+
+ if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
+ /* op3 is used as a flag that the value is already stored in memory.
+ * In case the IR framework decides to spill the result of IR_LOAD,
+ * it doesn't have to store the value once again.
+ *
+ * See: insn->op3 check in ir_emit_rload()
+ */
+ ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
+ }
+ }
+ }
+ }
+ }
+
+ if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
+ ZEND_ASSERT(parent->exit_info[exit_num].poly_func.reg >= 0 && parent->exit_info[exit_num].poly_this.reg >= 0);
+ if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func.reg)) {
+ ir_RLOAD_A(parent->exit_info[exit_num].poly_func.reg);
+ }
+ if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this.reg)) {
+ ir_RLOAD_A(parent->exit_info[exit_num].poly_this.reg);
+ }
+ }
+}
+
+static int zend_jit_trace_begin_loop(zend_jit_ctx *jit)
+{
+ return ir_LOOP_BEGIN(ir_END());
+}
+
+static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
+{
+ int dst_var = phi->ssa_var;
+ int src_var = phi->sources[0];
+ ir_ref ref;
+
+ ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
+ ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
+
+ ref = ir_PHI_2(
+ (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
+ zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
+
+ src_var = phi->sources[1];
+ ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
+ jit->ra[src_var].flags |= ZREG_FORWARD;
+
+ zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
+}
+
+static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr)
+{
+ if (timeout_exit_addr) {
+ zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
+ }
+ ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
+ ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
+ return 1;
+}
+
+static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
+{
+ if (GCC_GLOBAL_REGS) {
+ if (!original_handler) {
+ ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
+ } else {
+ ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
+ }
+ } else {
+ if (original_handler) {
+ ir_ref ref;
+ ir_ref addr = zend_jit_orig_opline_handler(jit);
+
+#if defined(IR_TARGET_X86)
+ addr = ir_CAST_FC_FUNC(addr);
+#endif
+ ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
+ if (opline &&
+ (opline->opcode == ZEND_RETURN
+ || opline->opcode == ZEND_RETURN_BY_REF
+ || opline->opcode == ZEND_GENERATOR_RETURN
+ || opline->opcode == ZEND_GENERATOR_CREATE
+ || opline->opcode == ZEND_YIELD
+ || opline->opcode == ZEND_YIELD_FROM)) {
+ ir_RETURN(ref);
+ return 1;
+ }
+ }
+ ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
+ }
+ return 1;
+}
+
+static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
+{
+ return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
+}
+
+static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr)
+{
+ const void *link_addr;
+
+ /* Skip prologue. */
+ ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1);
+ link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size);
+
+ if (timeout_exit_addr) {
+ zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
+ }
+ ir_IJMP(ir_CONST_ADDR(link_addr));
+
+ return 1;
+}
+
+static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
+{
+ uint32_t op1_info, op2_info;
+
+ switch (opline->opcode) {
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM);
+ case ZEND_QM_ASSIGN:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ case ZEND_CASE:
+ return 1;
+ case ZEND_RETURN:
+ return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
+ case ZEND_ASSIGN:
+ return (opline->op1_type == IS_CV);
+ case ZEND_ASSIGN_OP:
+ if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
+ return 0;
+ }
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ return zend_jit_supported_binary_op(opline->extended_value, op1_info, op2_info);
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ return 0;
+ }
+ if (trace && trace->op1_type != IS_UNKNOWN) {
+ op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
+ }
+ if (trace && trace->op2_type != IS_UNKNOWN) {
+ op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
+ }
+ return !(op1_info & MAY_BE_UNDEF)
+ && !(op2_info & MAY_BE_UNDEF)
+ && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
+ && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if (trace && trace->op1_type != IS_UNKNOWN) {
+ op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
+ }
+ if (trace && trace->op2_type != IS_UNKNOWN) {
+ op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
+ }
+ return (op1_info & MAY_BE_LONG)
+ && (op2_info & MAY_BE_LONG);
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ op1_info = OP1_INFO();
+ return opline->op1_type == IS_CV
+ && (op1_info & MAY_BE_LONG)
+ && !(op1_info & MAY_BE_REF);
+ case ZEND_STRLEN:
+ op1_info = OP1_INFO();
+ return (opline->op1_type & (IS_CV|IS_CONST))
+ && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
+ case ZEND_COUNT:
+ op1_info = OP1_INFO();
+ return (opline->op1_type & (IS_CV|IS_CONST))
+ && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
+ if (!ssa->cfg.map) {
+ return 0;
+ }
+ if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
+ ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ return 0;
+ }
+ }
+ ZEND_FALLTHROUGH;
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ return 1;
+ case ZEND_FETCH_CONSTANT:
+ return 1;
+ case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+ if ((opline->extended_value & ZEND_ISEMPTY)) {
+ return 0;
+ }
+ ZEND_FALLTHROUGH;
+ case ZEND_FETCH_DIM_R:
+ case ZEND_FETCH_DIM_IS:
+ case ZEND_FETCH_LIST_R:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if (trace
+ && trace->op1_type != IS_UNKNOWN
+ && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
+ op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
+ }
+ return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
+ (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
+ ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
+ case ZEND_ASSIGN_DIM_OP:
+ if (opline->result_type != IS_UNUSED) {
+ return 0;
+ }
+ if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
+ return 0;
+ }
+ ZEND_FALLTHROUGH;
+ case ZEND_ASSIGN_DIM:
+ case ZEND_FETCH_DIM_W:
+ case ZEND_FETCH_DIM_RW:
+ case ZEND_FETCH_LIST_W:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if (trace) {
+ if (opline->op1_type == IS_CV) {
+ if ((opline->opcode == ZEND_ASSIGN_DIM
+ || opline->opcode == ZEND_ASSIGN_DIM_OP)
+ && (opline+1)->op1_type == IS_CV
+ && (opline+1)->op1.var == opline->op1.var) {
+ /* skip $a[x] = $a; */
+ return 0;
+ }
+ } else if (opline->op1_type == IS_VAR) {
+ if (trace->op1_type == IS_UNKNOWN
+ || !(trace->op1_type & IS_TRACE_INDIRECT)
+ || opline->result_type != IS_UNUSED) {
+ return 0;
+ }
+ }
+ if (trace->op1_type != IS_UNKNOWN
+ && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
+ op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
+ }
+ } else {
+ if (opline->op1_type != IS_CV) {
+ return 0;
+ }
+ }
+ return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
+ (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
+ ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
+ case ZEND_ASSIGN_OBJ_OP:
+ if (opline->result_type != IS_UNUSED) {
+ return 0;
+ }
+ if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
+ return 0;
+ }
+ ZEND_FALLTHROUGH;
+ case ZEND_FETCH_OBJ_R:
+ case ZEND_ASSIGN_OBJ:
+ if (opline->op2_type != IS_CONST
+ || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
+ || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
+ return 0;
+ }
+ op1_info = OP1_INFO();
+ return opline->op1_type == IS_UNUSED || (op1_info & MAY_BE_OBJECT);
+ }
+ return 0;
+}
+
+static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
+{
+ if (ssa->vars[var].no_val) {
+ /* we don't need the value */
+ return 0;
+ }
+
+ if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
+ /* Disable global register allocation,
+ * register allocation for SSA variables connected through Phi functions
+ */
+ if (ssa->vars[var].definition_phi) {
+ return 0;
+ }
+ if (ssa->vars[var].phi_use_chain) {
+ zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
+ do {
+ if (!ssa->vars[phi->ssa_var].no_val) {
+ return 0;
+ }
+ phi = zend_ssa_next_use_phi(ssa, var, phi);
+ } while (phi);
+ }
+ }
+
+ if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
+ ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
+ /* bad type */
+ return 0;
+ }
+
+ return 1;
+}
+
+static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
+{
+ if (!zend_jit_var_supports_reg(ssa, var)) {
+ return 0;
+ }
+
+ if (ssa->vars[var].definition >= 0) {
+ uint32_t def = ssa->vars[var].definition;
+ if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
+ return 0;
+ }
+ }
+
+ if (ssa->vars[var].use_chain >= 0) {
+ int use = ssa->vars[var].use_chain;
+
+ do {
+ if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
+ !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
+ return 0;
+ }
+ use = zend_ssa_next_use(ssa->ops, var, use);
+ } while (use >= 0);
+ }
+
+ if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
+ int def_block, use_block, b, use, j;
+ zend_basic_block *bb;
+ zend_ssa_phi *p;
+ bool ret = 1;
+ zend_worklist worklist;
+ ALLOCA_FLAG(use_heap)
+
+ /* Check if live range is split by ENTRY block */
+ if (ssa->vars[var].definition >= 0) {
+ def_block =ssa->cfg.map[ssa->vars[var].definition];
+ } else {
+ ZEND_ASSERT(ssa->vars[var].definition_phi);
+ def_block = ssa->vars[var].definition_phi->block;
+ }
+
+ ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
+
+ if (ssa->vars[var].use_chain >= 0) {
+ use = ssa->vars[var].use_chain;
+ do {
+ use_block = ssa->cfg.map[use];
+ if (use_block != def_block) {
+ zend_worklist_push(&worklist, use_block);
+ }
+ use = zend_ssa_next_use(ssa->ops, var, use);
+ } while (use >= 0);
+ }
+
+ p = ssa->vars[var].phi_use_chain;
+ while (p) {
+ use_block = p->block;
+ if (use_block != def_block) {
+ bb = &ssa->cfg.blocks[use_block];
+ for (j = 0; j < bb->predecessors_count; j++) {
+ if (p->sources[j] == var) {
+ use_block = ssa->cfg.predecessors[bb->predecessor_offset + j];
+ if (use_block != def_block) {
+ zend_worklist_push(&worklist, use_block);
+ }
+ }
+ }
+ }
+ p = zend_ssa_next_use_phi(ssa, var, p);
+ }
+
+ while (zend_worklist_len(&worklist) != 0) {
+ b = zend_worklist_pop(&worklist);
+ bb = &ssa->cfg.blocks[b];
+ if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
+ ret = 0;
+ break;
+ }
+ for (j = 0; j < bb->predecessors_count; j++) {
+ b = ssa->cfg.predecessors[bb->predecessor_offset + j];
+ if (b != def_block) {
+ zend_worklist_push(&worklist, b);
+ }
+ }
+ }
+
+ ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
+
+ return ret;
+ }
+
+ return 1;
+}
+
+static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) {
+ // JIT: zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(fbc))
+ ir_ref observer_handler;
+ zend_function *fbc = ZEND_FLF_FUNC(opline);
+ // Not need for runtime cache or generator checks here, we just need if_unobserved
+ ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUSED, IR_UNUSED).if_unobserved;
+
+ // Call zend_frameless_observed_call for the main logic.
+ ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit));
+
+ ir_ref skip = ir_END();
+ ir_IF_TRUE(if_unobserved);
+ return skip;
+}
+
- static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
++static bool jit_frameless_interrupt_check(zend_jit_ctx *jit, const zend_op *next_opline)
++{
++ if (zend_interrupt_function) {
++ /* Mirrors ZEND_VM_FCALL_INTERRUPT_CHECK(), call before advancing opline. */
++ // JIT: if (EG(vm_interrupt)) zend_fcall_interrupt(execute_data);
++ ir_ref if_interrupt = ir_IF(ir_LOAD_U8(jit_EG(vm_interrupt)));
++ ir_IF_TRUE_cold(if_interrupt);
++ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_fcall_interrupt), jit_FP(jit));
++ ir_MERGE_WITH_EMPTY_FALSE(if_interrupt);
++ } else {
++ const void *exit_addr = NULL;
++
++ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
++ uint32_t exit_point = zend_jit_trace_get_exit_point(next_opline, ZEND_JIT_EXIT_TO_VM);
++ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
++ if (!exit_addr) {
++ return false;
++ }
++ }
++
++ zend_jit_check_timeout(jit, next_opline, exit_addr);
++ }
++
++ return true;
++}
++
++static bool jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
+{
+ jit_SET_EX_OPLINE(jit, opline);
+
+ void *function = ZEND_FLF_HANDLER(opline);
+ zend_jit_addr res_addr = RES_ADDR();
+ ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+
+ ir_ref skip_observer = IR_UNUSED;
+ if (ZEND_OBSERVER_ENABLED) {
+ skip_observer = jit_frameless_observer(jit, opline);
+ }
+
+ ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref);
+
+ if (skip_observer != IR_UNUSED) {
+ ir_MERGE_WITH(skip_observer);
+ }
+
++ if (!jit_frameless_interrupt_check(jit, opline+1)) {
++ return false;
++ }
+ zend_jit_check_exception(jit);
++ return true;
+}
+
- static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
++static bool jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
+{
+ jit_SET_EX_OPLINE(jit, opline);
+
+ /* Avoid dropping RC check in case op escapes. */
+ if (op1_info & MAY_BE_RC1) {
+ op1_info |= MAY_BE_RCN;
+ }
+
+ void *function = ZEND_FLF_HANDLER(opline);
+ zend_jit_addr res_addr = RES_ADDR();
+ zend_jit_addr op1_addr = OP1_ADDR();
+ ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
+ op1_info &= ~MAY_BE_UNDEF;
+ op1_info |= MAY_BE_NULL;
+ op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
+ }
+
+ ir_ref skip_observer = IR_UNUSED;
+ if (ZEND_OBSERVER_ENABLED) {
+ skip_observer = jit_frameless_observer(jit, opline);
+ }
+
+ ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref);
+
+ if (skip_observer != IR_UNUSED) {
+ ir_MERGE_WITH(skip_observer);
+ }
+
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
++ if (!jit_frameless_interrupt_check(jit, opline+1)) {
++ return false;
++ }
+ zend_jit_check_exception(jit);
++ return true;
+}
+
- static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
++static bool jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
+{
+ jit_SET_EX_OPLINE(jit, opline);
+
+ /* Avoid dropping RC check in case op escapes. */
+ if (op1_info & MAY_BE_RC1) {
+ op1_info |= MAY_BE_RCN;
+ }
+ if (op2_info & MAY_BE_RC1) {
+ op2_info |= MAY_BE_RCN;
+ }
+
+ void *function = ZEND_FLF_HANDLER(opline);
+ zend_jit_addr res_addr = RES_ADDR();
+ zend_jit_addr op1_addr = OP1_ADDR();
+ zend_jit_addr op2_addr = OP2_ADDR();
+ ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
+ op1_info &= ~MAY_BE_UNDEF;
+ op1_info |= MAY_BE_NULL;
+ op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
+ }
+ if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
+ op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
+ op2_info &= ~MAY_BE_UNDEF;
+ op2_info |= MAY_BE_NULL;
+ op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
+ }
+ if (op2_info & MAY_BE_REF) {
+ op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
+ }
+
+ ir_ref skip_observer = IR_UNUSED;
+ if (ZEND_OBSERVER_ENABLED) {
+ skip_observer = jit_frameless_observer(jit, opline);
+ }
+
+ ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref);
+
+ if (skip_observer != IR_UNUSED) {
+ ir_MERGE_WITH(skip_observer);
+ }
+
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
+ /* Set OP1 to UNDEF in case FREE_OP2() throws. */
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0
+ && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
+ && (op2_info & MAY_BE_RC1)
+ && (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
+ if (JIT_G(current_frame)) {
+ SET_STACK_TYPE(JIT_G(current_frame)->stack,
+ EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
+ }
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
++ if (!jit_frameless_interrupt_check(jit, opline+1)) {
++ return false;
++ }
+ zend_jit_check_exception(jit);
++ return true;
+}
+
- static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
++static bool jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
+{
+ jit_SET_EX_OPLINE(jit, opline);
+
+ /* Avoid dropping RC check in case op escapes. */
+ if (op1_info & MAY_BE_RC1) {
+ op1_info |= MAY_BE_RCN;
+ }
+ if (op2_info & MAY_BE_RC1) {
+ op2_info |= MAY_BE_RCN;
+ }
+ if (op1_data_info & MAY_BE_RC1) {
+ op1_data_info |= MAY_BE_RCN;
+ }
+
+ void *function = ZEND_FLF_HANDLER(opline);
+ uint8_t op_data_type = (opline + 1)->op1_type;
+ zend_jit_addr res_addr = RES_ADDR();
+ zend_jit_addr op1_addr = OP1_ADDR();
+ zend_jit_addr op2_addr = OP2_ADDR();
+ zend_jit_addr op3_addr = OP1_DATA_ADDR();
+ ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
+ ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
+ ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
+ ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
+ jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
+ op1_info &= ~MAY_BE_UNDEF;
+ op1_info |= MAY_BE_NULL;
+ op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
+ }
+ if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
+ op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
+ op2_info &= ~MAY_BE_UNDEF;
+ op2_info |= MAY_BE_NULL;
+ op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref);
+ }
+ if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) {
+ op3_ref = zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
+ op1_data_info &= ~MAY_BE_UNDEF;
+ op1_data_info |= MAY_BE_NULL;
+ op3_addr = ZEND_ADDR_REF_ZVAL(op3_ref);
+ }
+ if (op1_info & MAY_BE_REF) {
+ op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
+ }
+ if (op2_info & MAY_BE_REF) {
+ op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
+ }
+ if (op1_data_info & MAY_BE_REF) {
+ op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
+ }
+
+ ir_ref skip_observer = IR_UNUSED;
+ if (ZEND_OBSERVER_ENABLED) {
+ skip_observer = jit_frameless_observer(jit, opline);
+ }
+
+ ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref);
+
+ if (skip_observer != IR_UNUSED) {
+ ir_MERGE_WITH(skip_observer);
+ }
+
+ jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
+ /* Set OP1 to UNDEF in case FREE_OP2() throws. */
+ bool op1_undef = false;
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
+ && (((opline->op2_type & (IS_VAR|IS_TMP_VAR))
+ && (op2_info & MAY_BE_RC1)
+ && (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
+ || ((op_data_type & (IS_VAR|IS_TMP_VAR))
+ && (op1_data_info & MAY_BE_RC1)
+ && (op1_data_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))))) {
+ op1_undef = true;
+ jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
+ if (JIT_G(current_frame)) {
+ SET_STACK_TYPE(JIT_G(current_frame)->stack,
+ EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
+ }
+ }
+ jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
+ /* If OP1 is set to UNDEF, we don't need to set OP2 to UNDEF on free because
+ * zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */
+ if (!op1_undef
+ && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
+ && (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0
+ && (op1_data_info & MAY_BE_RC1)
+ && (op1_data_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
+ jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
+ if (JIT_G(current_frame)) {
+ SET_STACK_TYPE(JIT_G(current_frame)->stack,
+ EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
+ }
+ }
+ jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
++ if (!jit_frameless_interrupt_check(jit, opline+2)) {
++ return false;
++ }
+ zend_jit_check_exception(jit);
++ return true;
+}
+
+/*
+ * Local variables:
+ * tab-width: 4
+ * c-basic-offset: 4
+ * indent-tabs-mode: t
+ * End:
+ */
diff --cc ext/opcache/jit/zend_jit_trace.c
index 6eef49baf9c,33c554a3f78..b179cced8d2
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@@ -6393,26 -6214,9 +6393,34 @@@ static const void *zend_jit_trace(zend_
if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
break;
}
- if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
+ if (!zend_jit_rope(&ctx, opline, op2_info)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_0:
- jit_frameless_icall0(jit, opline);
++ if (!jit_frameless_icall0(jit, opline)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_1:
+ op1_info = OP1_INFO();
- jit_frameless_icall1(jit, opline, op1_info);
++ if (!jit_frameless_icall1(jit, opline, op1_info)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_2:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
- jit_frameless_icall2(jit, opline, op1_info, op2_info);
++ if (!jit_frameless_icall2(jit, opline, op1_info, op2_info)) {
++ goto jit_failure;
++ }
+ goto done;
+ case ZEND_FRAMELESS_ICALL_3:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
- jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
++ if (!jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO())) {
+ goto jit_failure;
+ }
goto done;
default:
break;
diff --cc ext/opcache/tests/jit/frameless_interrupt_001.phpt
index 00000000000,00000000000..04840f75e27
new file mode 100644
--- /dev/null
+++ b/ext/opcache/tests/jit/frameless_interrupt_001.phpt
@@@ -1,0 -1,0 +1,31 @@@
++--TEST--
++JIT: an interrupt after a frameless call does not run the call again (function JIT)
++--EXTENSIONS--
++opcache
++zend_test
++--INI--
++opcache.enable=1
++opcache.enable_cli=1
++opcache.jit=function
++opcache.jit_buffer_size=32M
++zend_test.observer.enabled=1
++zend_test.observer.show_output=0
++zend_test.observer.observe_function_names=str_contains
++zend_test.observer.set_vm_interrupt_on_begin=1
++--FILE--
++<?php
++function f($s) {
++ $c = 0;
++ for ($i = 0; $i < 3000; $i++) {
++ if (str_contains($s . $i, '7')) {
++ $c++;
++ }
++ }
++ return $c;
++}
++var_dump(f("abc"));
++var_dump(f("abc"));
++?>
++--EXPECT--
++int(813)
++int(813)
diff --cc ext/opcache/tests/jit/frameless_interrupt_002.phpt
index 00000000000,00000000000..8fe1a0747d5
new file mode 100644
--- /dev/null
+++ b/ext/opcache/tests/jit/frameless_interrupt_002.phpt
@@@ -1,0 -1,0 +1,31 @@@
++--TEST--
++JIT: an interrupt after a frameless call does not run the call again (tracing JIT)
++--EXTENSIONS--
++opcache
++zend_test
++--INI--
++opcache.enable=1
++opcache.enable_cli=1
++opcache.jit=tracing
++opcache.jit_buffer_size=32M
++zend_test.observer.enabled=1
++zend_test.observer.show_output=0
++zend_test.observer.observe_function_names=str_contains
++zend_test.observer.set_vm_interrupt_on_begin=1
++--FILE--
++<?php
++function f($s) {
++ $c = 0;
++ for ($i = 0; $i < 3000; $i++) {
++ if (str_contains($s . $i, '7')) {
++ $c++;
++ }
++ }
++ return $c;
++}
++var_dump(f("abc"));
++var_dump(f("abc"));
++?>
++--EXPECT--
++int(813)
++int(813)
diff --cc ext/opcache/tests/jit/frameless_interrupt_003.phpt
index 00000000000,00000000000..be2577632e9
new file mode 100644
--- /dev/null
+++ b/ext/opcache/tests/jit/frameless_interrupt_003.phpt
@@@ -1,0 -1,0 +1,31 @@@
++--TEST--
++Properly deoptimize trace before frameless call interrupt
++--EXTENSIONS--
++opcache
++zend_test
++--INI--
++opcache.enable=1
++opcache.enable_cli=1
++opcache.jit=tracing
++opcache.jit_buffer_size=32M
++zend_test.observer.enabled=1
++zend_test.observer.show_output=0
++zend_test.observer.observe_function_names=str_contains
++zend_test.observer.set_vm_interrupt_on_begin_without_callback=1
++--FILE--
++<?php
++function test() {
++ $sum = 0;
++ for ($i = 0; $i < 3000; $i++) {
++ $sum += $i;
++ str_contains($i, 'x');
++ }
++ return $sum;
++}
++
++var_dump(test());
++var_dump(test());
++?>
++--EXPECT--
++int(4498500)
++int(4498500)
diff --cc ext/opcache/zend_persist.c
index 3f80b7d65d9,ac189b67fa0..8b8312ff81e
--- a/ext/opcache/zend_persist.c
+++ b/ext/opcache/zend_persist.c
@@@ -1455,7 -1433,7 +1505,8 @@@ zend_persistent_script *zend_accel_scri
if (for_shm) {
ZCSG(map_ptr_last) = CG(map_ptr_last);
+ ZCSG(map_ptr_static_last) = zend_map_ptr_static_last;
+ zend_accel_pending_ce_cache_commit();
}
#ifdef HAVE_JIT
diff --cc ext/pcre/php_pcre.c
index 6a24691a170,9cc2a23f61b..5146260b323
--- a/ext/pcre/php_pcre.c
+++ b/ext/pcre/php_pcre.c
@@@ -1040,24 -1024,29 +1040,30 @@@ static inline void add_offset_pair
}
/* }}} */
- static void populate_subpat_array(
+ static zend_result populate_subpat_array(
zval *subpats, const char *subject, PCRE2_SIZE *offsets, zend_string **subpat_names,
uint32_t num_subpats, int count, const PCRE2_SPTR mark, zend_long flags) {
- bool offset_capture = (flags & PREG_OFFSET_CAPTURE) != 0;
- bool unmatched_as_null = (flags & PREG_UNMATCHED_AS_NULL) != 0;
+ zend_long offset_capture = flags & PREG_OFFSET_CAPTURE;
+ zend_long unmatched_as_null = flags & PREG_UNMATCHED_AS_NULL;
zval val;
int i;
+ HashTable *subpats_ht = Z_ARRVAL_P(subpats);
if (subpat_names) {
if (offset_capture) {
for (i = 0; i < count; i++) {
add_offset_pair(
- subpats, subject, offsets[2*i], offsets[2*i+1],
+ subpats_ht, subject, offsets[2*i], offsets[2*i+1],
subpat_names[i], unmatched_as_null);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
if (unmatched_as_null) {
for (i = count; i < num_subpats; i++) {
- add_offset_pair(subpats, NULL, PCRE2_UNSET, PCRE2_UNSET, subpat_names[i], 1);
+ add_offset_pair(subpats_ht, NULL, PCRE2_UNSET, PCRE2_UNSET, subpat_names[i], 1);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
}
} else {
@@@ -1065,17 -1054,23 +1071,23 @@@
populate_match_value(
&val, subject, offsets[2*i], offsets[2*i+1], unmatched_as_null);
if (subpat_names[i]) {
- add_named(subpats, subpat_names[i], &val, offsets[2*i] == PCRE2_UNSET);
+ add_named(subpats_ht, subpat_names[i], &val, offsets[2*i] == PCRE2_UNSET);
}
- zend_hash_next_index_insert(Z_ARRVAL_P(subpats), &val);
+ zend_hash_next_index_insert_new(subpats_ht, &val);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
if (unmatched_as_null) {
for (i = count; i < num_subpats; i++) {
ZVAL_NULL(&val);
if (subpat_names[i]) {
- zend_hash_add(Z_ARRVAL_P(subpats), subpat_names[i], &val);
+ zend_hash_add(subpats_ht, subpat_names[i], &val);
}
- zend_hash_next_index_insert(Z_ARRVAL_P(subpats), &val);
+ zend_hash_next_index_insert_new(subpats_ht, &val);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
}
}
@@@ -1083,18 -1078,27 +1095,27 @@@
if (offset_capture) {
for (i = 0; i < count; i++) {
add_offset_pair(
- subpats, subject, offsets[2*i], offsets[2*i+1], NULL, unmatched_as_null);
+ subpats_ht, subject, offsets[2*i], offsets[2*i+1], NULL, unmatched_as_null);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
if (unmatched_as_null) {
for (i = count; i < num_subpats; i++) {
- add_offset_pair(subpats, NULL, PCRE2_UNSET, PCRE2_UNSET, NULL, 1);
+ add_offset_pair(subpats_ht, NULL, PCRE2_UNSET, PCRE2_UNSET, NULL, 1);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
}
} else {
for (i = 0; i < count; i++) {
populate_match_value(
&val, subject, offsets[2*i], offsets[2*i+1], unmatched_as_null);
- zend_hash_next_index_insert(Z_ARRVAL_P(subpats), &val);
+ zend_hash_next_index_insert_new(subpats_ht, &val);
+ if (ZEND_HAS_OOM()) {
+ return FAILURE;
+ }
}
if (unmatched_as_null) {
for (i = count; i < num_subpats; i++) {
@@@ -1107,9 -1114,10 +1131,10 @@@
if (mark) {
add_assoc_string_ex(subpats, "MARK", sizeof("MARK") - 1, (char *)mark);
}
+ return ZEND_HAS_OOM() ? FAILURE : SUCCESS;
}
-static void php_do_pcre_match(INTERNAL_FUNCTION_PARAMETERS, int global) /* {{{ */
+static void php_do_pcre_match(INTERNAL_FUNCTION_PARAMETERS, bool global) /* {{{ */
{
/* parameters */
zend_string *regex; /* Regular expression */
@@@ -1328,15 -1337,21 +1357,21 @@@ matched
if (offset_capture) {
for (i = 0; i < count; i++) {
add_offset_pair(
- &match_sets[i], subject, offsets[2*i], offsets[2*i+1],
+ match_sets[i], subject, offsets[2*i], offsets[2*i+1],
NULL, unmatched_as_null);
+ if (ZEND_HAS_OOM()) {
+ goto oom;
+ }
}
} else {
for (i = 0; i < count; i++) {
zval val;
populate_match_value(
&val, subject, offsets[2*i], offsets[2*i+1], unmatched_as_null);
- zend_hash_next_index_insert_new(Z_ARRVAL(match_sets[i]), &val);
+ zend_hash_next_index_insert_new(match_sets[i], &val);
+ if (ZEND_HAS_OOM()) {
+ goto oom;
+ }
}
}
mark = pcre2_get_mark(match_data);
@@@ -1355,31 -1368,34 +1390,38 @@@
* arrays with NULLs or empty strings.
*/
if (count < num_subpats) {
- for (; i < num_subpats; i++) {
+ for (int i = count; i < num_subpats; i++) {
if (offset_capture) {
add_offset_pair(
- &match_sets[i], NULL, PCRE2_UNSET, PCRE2_UNSET,
+ match_sets[i], NULL, PCRE2_UNSET, PCRE2_UNSET,
NULL, unmatched_as_null);
} else if (unmatched_as_null) {
- add_next_index_null(&match_sets[i]);
+ zval tmp;
+ ZVAL_NULL(&tmp);
+ zend_hash_next_index_insert_new(match_sets[i], &tmp);
} else {
- add_next_index_str(&match_sets[i], ZSTR_EMPTY_ALLOC());
+ zval tmp;
+ ZVAL_EMPTY_STRING(&tmp);
+ zend_hash_next_index_insert_new(match_sets[i], &tmp);
}
+ if (ZEND_HAS_OOM()) {
+ goto oom;
+ }
}
}
} else {
/* Allocate and populate the result set array */
- array_init_size(&result_set, count + (mark ? 1 : 0));
mark = pcre2_get_mark(match_data);
+ array_init_size(&result_set, count + (mark ? 1 : 0));
- populate_subpat_array(
+ zend_result result = populate_subpat_array(
&result_set, subject, offsets, subpat_names,
num_subpats, count, mark, flags);
+ if (UNEXPECTED(result == FAILURE)) {
+ zval_ptr_dtor(&result_set);
+ goto oom;
+ }
/* And add it to the output array */
- zend_hash_next_index_insert(Z_ARRVAL_P(subpats), &result_set);
+ zend_hash_next_index_insert_new(Z_ARRVAL_P(subpats), &result_set);
}
} else { /* single pattern matching */
/* For each subpattern, insert it into the subpatterns array. */
@@@ -1492,6 -1508,28 +1537,25 @@@ error
} else {
RETVAL_FALSE;
}
+ return;
+
+ oom:
+ if (match_data != mdata) {
+ pcre2_match_data_free(match_data);
+ }
+ if (match_sets) {
+ for (i = 0; i < num_subpats; i++) {
- zval_ptr_dtor(&match_sets[i]);
++ zend_array_destroy(match_sets[i]);
+ }
+ efree(match_sets);
+ }
- if (Z_TYPE(marks) != IS_UNDEF) {
- zval_ptr_dtor(&marks);
- }
- if (subpat_names) {
- free_subpats_table(subpat_names, num_subpats);
++ if (marks) {
++ zend_array_destroy(marks);
+ }
+ if (subpats) {
+ zend_hash_clean(Z_ARRVAL_P(subpats));
+ }
+ RETURN_OOM();
}
/* }}} */
diff --cc ext/pcre/tests/oom_preg_match_all_named_groups.phpt
index 00000000000,00000000000..de01fa6d85c
new file mode 100644
--- /dev/null
+++ b/ext/pcre/tests/oom_preg_match_all_named_groups.phpt
@@@ -1,0 -1,0 +1,17 @@@
++--TEST--
++preg_match_all() with named groups keeps the cached subpattern names on OOM
++--INI--
++memory_limit=16M
++--FILE--
++<?php
++register_shutdown_function(function () {
++ preg_match_all('/(?<l>[a-z])(?<d>\d)/', 'a1b2', $matches);
++ echo json_encode($matches), "\n";
++});
++
++$subject = str_repeat("ab12 ", 400000);
++preg_match_all('/(?<l>[a-z])(?<d>\d)/', $subject, $matches);
++?>
++--EXPECTF--
++Fatal error: Allowed memory size of 16777216 bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
++{"0":["a1","b2"],"l":["a","b"],"1":["a","b"],"d":["1","2"],"2":["1","2"]}
diff --cc ext/session/php_session.h
index b52cf43dfba,44483536654..7e53deb8483
--- a/ext/session/php_session.h
+++ b/ext/session/php_session.h
@@@ -202,8 -199,9 +202,10 @@@ typedef struct _php_ps_globals
bool lazy_write; /* omit session write when it is possible */
bool in_save_handler; /* state if session is in save handler or not */
bool set_handler; /* state if session module i setting handler or not */
+ bool random_seeded;
zend_string *session_vars; /* serialized original session data */
+
+ bool in_rfc1867_update; /* php_session_rfc1867_update() is running */
} php_ps_globals;
typedef php_ps_globals zend_ps_globals;
diff --cc ext/session/session.c
index a9838cafe66,1b35bf46e7e..7a57652f55d
--- a/ext/session/session.c
+++ b/ext/session/session.c
@@@ -1080,10 -1007,9 +1102,10 @@@ PS_SERIALIZER_ENCODE_FUNC(php) /* {{{ *
{
smart_str buf = {0};
php_serialize_data_t var_hash;
+ bool fail = false;
PS_ENCODE_VARS;
- PHP_VAR_SERIALIZE_INIT(var_hash);
+ php_session_serialize_init(&var_hash);
PS_ENCODE_LOOP(
smart_str_appendl(&buf, ZSTR_VAL(key), ZSTR_LEN(key));
diff --cc ext/shmop/shmop.c
index 640f595ea6e,089fbac59bd..ba69b5dc68d
--- a/ext/shmop/shmop.c
+++ b/ext/shmop/shmop.c
@@@ -243,9 -241,12 +243,12 @@@ PHP_FUNCTION(shmop_read
}
startaddr = shmop->addr + start;
- bytes = count ? count : shmop->size - start;
+ zend_long bytes = count ? count : shmop->size - start;
- return_string = zend_string_init(startaddr, bytes, 0);
+ return_string = zend_string_finit(startaddr, bytes, /* persistent */ false, /* fallible */ true);
+ if (UNEXPECTED(!return_string)) {
+ RETURN_OOM();
+ }
RETURN_NEW_STR(return_string);
}
diff --cc ext/simplexml/simplexml.c
index 9e5f293fe78,7ac685e36fc..7d4337ae384
--- a/ext/simplexml/simplexml.c
+++ b/ext/simplexml/simplexml.c
@@@ -939,18 -929,17 +939,17 @@@ static inline zend_string *sxe_xmlNodeL
}
/* }}} */
-/* {{{ _get_base_node_value() */
-static bool _get_base_node_value(php_sxe_object *sxe_ref, xmlNodePtr node, zval *value, xmlChar *nsprefix, int isprefix)
+/* {{{ get_base_node_value() */
- static void get_base_node_value(php_sxe_object *sxe_ref, xmlNodePtr node, zval *value, zend_string *nsprefix, int isprefix)
++static bool get_base_node_value(php_sxe_object *sxe_ref, xmlNodePtr node, zval *value, zend_string *nsprefix, int isprefix)
{
php_sxe_object *subnode;
- xmlChar *contents;
if (node->children && node->children->type == XML_TEXT_NODE && !xmlIsBlankNode(node->children)) {
- contents = xmlNodeListGetString(node->doc, node->children, 1);
- if (contents) {
- ZVAL_STRING(value, (char *)contents);
- xmlFree(contents);
+ zend_string *str = sxe_xmlNodeListGetString(node->doc, node->children, 1);
+ if (UNEXPECTED(!str)) {
+ return false;
}
+ ZVAL_STR(value, str);
} else {
subnode = php_sxe_object_new(sxe_ref->zo.ce, sxe_ref->fptr_count);
subnode->document = sxe_ref->document;
@@@ -1110,8 -1111,12 +1110,12 @@@ static HashTable *sxe_get_prop_hash(zen
ZVAL_UNDEF(&zattr);
test = sxe->iter.name && sxe->iter.type == SXE_ITER_ATTRLIST;
while (attr) {
- if ((!test || xmlStrEqual(attr->name, sxe->iter.name)) && match_ns(sxe, (xmlNodePtr)attr, sxe->iter.nsprefix, sxe->iter.isprefix)) {
+ if ((!test || xmlStrEqual(attr->name, BAD_CAST ZSTR_VAL(sxe->iter.name))) && match_ns((xmlNodePtr)attr, sxe->iter.nsprefix, sxe->iter.isprefix)) {
- ZVAL_STR(&value, sxe_xmlNodeListGetString((xmlDocPtr) sxe->document->ptr, attr->children, 1));
+ zend_string *str = sxe_xmlNodeListGetString((xmlDocPtr) sxe->document->ptr, attr->children, 1);
+ if (UNEXPECTED(!str)) {
+ goto oom;
+ }
+ ZVAL_STR(&value, str);
namelen = xmlStrlen(attr->name);
if (Z_ISUNDEF(zattr)) {
array_init(&zattr);
@@@ -1167,9 -1190,11 +1185,11 @@@
namelen = xmlStrlen(node->name);
}
- get_base_node_value(sxe, node, &value, sxe->iter.nsprefix, sxe->iter.isprefix);
- if (UNEXPECTED(!_get_base_node_value(sxe, node, &value, sxe->iter.nsprefix, sxe->iter.isprefix))) {
++ if (UNEXPECTED(!get_base_node_value(sxe, node, &value, sxe->iter.nsprefix, sxe->iter.isprefix))) {
+ goto oom;
+ }
- if (iter_data) {
+ if ( use_iter ) {
zend_hash_next_index_insert(rv, &value);
} else {
sxe_properties_add(rv, name, namelen, &value);
@@@ -1189,7 -1217,24 +1212,11 @@@ next_iter
}
}
- if (iter_data) {
- if (!Z_ISUNDEF(sxe->iter.data)) {
- zval_ptr_dtor(&sxe->iter.data);
- }
- ZVAL_COPY_VALUE(&sxe->iter.data, iter_data);
- }
-
return rv;
+
+ oom:
+ zend_hash_clean(rv);
- if (iter_data) {
- if (!Z_ISUNDEF(sxe->iter.data)) {
- zval_ptr_dtor(&sxe->iter.data);
- }
- ZVAL_COPY_VALUE(&sxe->iter.data, iter_data);
- }
+ return rv;
}
/* }}} */
@@@ -1633,10 -1700,14 +1672,14 @@@ PHP_METHOD(SimpleXMLElement, getName
sxe = Z_SXEOBJ_P(ZEND_THIS);
GET_NODE(sxe, node);
- node = php_sxe_get_first_node(sxe, node);
+ node = php_sxe_get_first_node_non_destructive(sxe, node);
if (node) {
namelen = xmlStrlen(node->name);
- RETURN_STRINGL_FAST((const char *) node->name, namelen);
+ zend_string *str = zend_string_finit_fast((const char *)node->name, namelen, /* fallible */ true);
+ if (!str) {
+ RETURN_OOM();
+ }
+ RETURN_STR(str);
} else {
RETURN_EMPTY_STRING();
}
diff --cc ext/simplexml/tests/oom_tostring.phpt
index 00000000000,00000000000..81d3daa758d
new file mode 100644
--- /dev/null
+++ b/ext/simplexml/tests/oom_tostring.phpt
@@@ -1,0 -1,0 +1,15 @@@
++--TEST--
++SimpleXMLElement::__toString() OOM
++--EXTENSIONS--
++simplexml
++--INI--
++memory_limit=16M
++--FILE--
++<?php
++$xml = simplexml_load_string('<r>' . str_repeat('x', 5500000) . '</r>', 'SimpleXMLElement', LIBXML_PARSEHUGE);
++$pad = str_repeat('p', 11500000);
++$value = $xml->__toString();
++echo strlen($value), "\n";
++?>
++--EXPECTF--
++Fatal error: Allowed memory size of 16777216 bytes exhausted %S(tried to allocate %d bytes) in %s on line 4
diff --cc ext/spl/spl_fixedarray.c
index 18025e0cbeb,1dc0f6334a7..16126ff31ba
--- a/ext/spl/spl_fixedarray.c
+++ b/ext/spl/spl_fixedarray.c
@@@ -177,30 -191,34 +191,34 @@@ static zend_result spl_fixedarray_resiz
{
if (size == array->size) {
/* nothing to do */
- return;
+ return SUCCESS;
}
- /* first initialization */
- if (array->size == 0) {
- return spl_fixedarray_init(array, size);
- }
-
if (UNEXPECTED(array->cached_resize >= 0)) {
/* We're already resizing, so just remember the desired size.
* The resize will happen later. */
array->cached_resize = size;
- return;
+ return SUCCESS;
}
+ /* first initialization */
+ if (array->size == 0) {
- spl_fixedarray_init(array, size);
- return;
++ return spl_fixedarray_init(array, size);
+ }
+
array->cached_resize = size;
/* clearing the array */
if (size == 0) {
+ /* Clears elements and size; resetting them afterwards would leak
+ * anything a destructor re-installed. */
spl_fixedarray_dtor(array);
- array->elements = NULL;
- array->size = 0;
} else if (size > array->size) {
- array->elements = safe_erealloc(array->elements, size, sizeof(zval), 0);
+ zval *elements = safe_ferealloc(array->elements, size, sizeof(zval), 0, /* fallible */ true);
+ if (UNEXPECTED(!elements)) {
+ array->cached_resize = -1;
+ return FAILURE;
+ }
+ array->elements = elements;
spl_fixedarray_init_elems(array, array->size, size);
array->size = size;
} else { /* size < array->size */
@@@ -209,16 -227,14 +227,18 @@@
array->elements = erealloc(array->elements, sizeof(zval) * size);
}
- /* If resized within the destructor, take the last resize command and perform it */
+ /* If resized within the destructor, take the last resize command and
+ * perform it. The sentinel is still set: re-initialising during a
+ * resize is refused. */
zend_long cached_resize = array->cached_resize;
+ ZEND_ASSERT(cached_resize >= 0);
+
array->cached_resize = -1;
if (cached_resize != size) {
- spl_fixedarray_resize(array, cached_resize);
+ return spl_fixedarray_resize(array, cached_resize);
}
+
+ return SUCCESS;
}
static HashTable* spl_fixedarray_object_get_gc(zend_object *obj, zval **table, int *n)
@@@ -311,6 -315,14 +323,17 @@@ static zend_object *spl_fixedarray_obje
inherited = true;
}
+ if (orig && clone_orig) {
+ spl_fixedarray_object *other = spl_fixed_array_from_obj(orig);
+ if (UNEXPECTED(spl_fixedarray_copy_ctor(&intern->array, &other->array) == FAILURE)) {
+ ZEND_ASSERT(intern->std.handlers);
+ return &intern->std;
+ }
++ } else {
++ /* The zeroed struct would mean "resizing"; set the sentinel. */
++ spl_fixedarray_default_ctor(&intern->array);
+ }
+
ZEND_ASSERT(parent);
if (UNEXPECTED(inherited)) {
@@@ -649,9 -668,11 +676,11 @@@ PHP_METHOD(SplFixedArray, __unserialize
RETURN_THROWS();
}
- if (intern->array.size == 0) {
+ if (EXPECTED(intern->array.size == 0 && !spl_fixedarray_resize_in_progress(&intern->array))) {
size = zend_hash_num_elements(data);
- spl_fixedarray_init_non_empty_struct(&intern->array, size);
+ if (UNEXPECTED(spl_fixedarray_init_non_empty_struct(&intern->array, size) == FAILURE)) {
+ RETURN_OOM();
+ }
if (!size) {
return;
}
@@@ -752,9 -773,11 +781,11 @@@ PHP_METHOD(SplFixedArray, fromArray
zend_throw_exception_ex(spl_ce_InvalidArgumentException, 0, "integer overflow detected");
RETURN_THROWS();
}
- spl_fixedarray_init(&array, tmp);
+ if (UNEXPECTED(spl_fixedarray_init(&array, tmp) == FAILURE)) {
+ RETURN_OOM();
+ }
- ZEND_HASH_FOREACH_KEY_VAL(Z_ARRVAL_P(data), num_index, str_index, element) {
+ ZEND_HASH_FOREACH_NUM_KEY_VAL(Z_ARRVAL_P(data), num_index, element) {
ZVAL_COPY_DEREF(&array.elements[num_index], element);
} ZEND_HASH_FOREACH_END();
diff --cc ext/standard/levenshtein.c
index d4688dcb9c1,477c27384f2..83e1feed710
--- a/ext/standard/levenshtein.c
+++ b/ext/standard/levenshtein.c
@@@ -25,23 -26,24 +25,33 @@@ static bool reference_levdist(const zen
size_t i1, i2;
if (ZSTR_LEN(string1) == 0) {
- return ZSTR_LEN(string2) * cost_ins;
+ *result = ZSTR_LEN(string2) * cost_ins;
+ return true;
}
if (ZSTR_LEN(string2) == 0) {
- return ZSTR_LEN(string1) * cost_del;
+ *result = ZSTR_LEN(string1) * cost_del;
+ return true;
}
+ /* When all costs are equal, levenshtein fulfills the requirements of a metric, which means
+ * that the distance is symmetric. If string1 is shorter than string 2 we can save memory (and CPU time)
+ * by having shorter rows (p1 & p2). */
+ if (ZSTR_LEN(string1) < ZSTR_LEN(string2) && cost_ins == cost_rep && cost_rep == cost_del) {
+ const zend_string *tmp = string1;
+ string1 = string2;
+ string2 = tmp;
+ }
+
- p1 = safe_emalloc((ZSTR_LEN(string2) + 1), sizeof(zend_long), 0);
- p2 = safe_emalloc((ZSTR_LEN(string2) + 1), sizeof(zend_long), 0);
+ /* Each row is eight times the length of string2. */
+ p1 = safe_femalloc((ZSTR_LEN(string2) + 1), sizeof(zend_long), 0, /* fallible */ true);
+ if (UNEXPECTED(!p1)) {
+ return false;
+ }
+ p2 = safe_femalloc((ZSTR_LEN(string2) + 1), sizeof(zend_long), 0, /* fallible */ true);
+ if (UNEXPECTED(!p2)) {
+ efree(p1);
+ return false;
+ }
for (i2 = 0; i2 <= ZSTR_LEN(string2); i2++) {
p1[i2] = i2 * cost_ins;
diff --cc ext/standard/string.c
index 40b05135a8e,779dd6359d5..9f8e45af329
--- a/ext/standard/string.c
+++ b/ext/standard/string.c
@@@ -2175,11 -2011,16 +2214,14 @@@ static zend_string *php_chunk_split(con
chunks++;
}
- dest = zend_string_safe_alloc(chunks, endlen, srclen, 0);
+ dest = zend_string_safe_falloc(chunks, endlen, srclen, 0, /* fallible */ true);
+ if (UNEXPECTED(!dest)) {
+ return NULL;
+ }
for (p = src, q = ZSTR_VAL(dest); p < (src + srclen - chunklen + 1); ) {
- memcpy(q, p, chunklen);
- q += chunklen;
- memcpy(q, end, endlen);
- q += endlen;
+ q = zend_mempcpy(q, p, chunklen);
+ q = zend_mempcpy(q, end, endlen);
p += chunklen;
}
@@@ -3392,53 -3244,6 +3465,66 @@@ PHPAPI zend_string *php_str_to_str(cons
}
/* }}} */
+static void php_strtr_array(zval *return_value, zend_string *str, HashTable *from_ht)
+{
+ if (zend_hash_num_elements(from_ht) < 1) {
+ RETURN_STR_COPY(str);
+ }
+
+ /* Converting a replacement may call __toString(), which can destroy from_ht. */
+ GC_TRY_ADDREF(from_ht);
+
+ if (zend_hash_num_elements(from_ht) == 1) {
+ zend_long num_key;
+ zend_string *str_key, *tmp_str, *replace, *tmp_replace;
+ zval *entry;
+
+ ZEND_HASH_FOREACH_KEY_VAL(from_ht, num_key, str_key, entry) {
+ tmp_str = NULL;
+ if (UNEXPECTED(!str_key)) {
+ str_key = tmp_str = zend_long_to_str(num_key);
+ }
+ replace = zval_get_tmp_string(entry, &tmp_replace);
+ if (ZSTR_LEN(str_key) < 1) {
+ php_error_docref(NULL, E_WARNING, "Ignoring replacement of empty string");
+ RETVAL_STR_COPY(str);
+ } else if (ZSTR_LEN(str_key) == 1) {
- RETVAL_STR(php_char_to_str_ex(str,
++ zend_string *res = php_char_to_str_ex(str,
+ ZSTR_VAL(str_key)[0],
+ ZSTR_VAL(replace),
+ ZSTR_LEN(replace),
+ /* case_sensitive */ true,
- NULL));
++ NULL);
++ if (UNEXPECTED(!res)) {
++ goto strtr_oom;
++ }
++ RETVAL_STR(res);
+ } else {
+ zend_long dummy = 0;
- RETVAL_STR(php_str_to_str_ex(str,
++ zend_string *res = php_str_to_str_ex(str,
+ ZSTR_VAL(str_key), ZSTR_LEN(str_key),
- ZSTR_VAL(replace), ZSTR_LEN(replace), &dummy));
++ ZSTR_VAL(replace), ZSTR_LEN(replace), &dummy);
++ if (UNEXPECTED(!res)) {
++ goto strtr_oom;
++ }
++ RETVAL_STR(res);
+ }
+ zend_tmp_string_release(tmp_str);
+ zend_tmp_string_release(tmp_replace);
+ break;
++
++strtr_oom:
++ zend_tmp_string_release(tmp_str);
++ zend_tmp_string_release(tmp_replace);
++ RETURN_OOM();
+ } ZEND_HASH_FOREACH_END();
+ } else {
+ php_strtr_array_ex(return_value, str, from_ht);
+ }
+
+ GC_TRY_DTOR_NO_REF(from_ht);
+}
+
/* {{{ Translates characters in str using given translation tables */
PHP_FUNCTION(strtr)
{
diff --cc ext/standard/tests/strings/oom_levenshtein.phpt
index 00000000000,53a6e00b9ce..0f58a8a0477
mode 000000,100644..100644
--- a/ext/standard/tests/strings/oom_levenshtein.phpt
+++ b/ext/standard/tests/strings/oom_levenshtein.phpt
@@@ -1,0 -1,15 +1,15 @@@
+ --TEST--
+ levenshtein() doesn't reach hard limit
+ --INI--
+ memory_limit=8M
+ --SKIPIF--
+ <?php
+ if (getenv("USE_ZEND_ALLOC") === "0") die("skip Zend MM disabled");
+ ?>
+ --FILE--
+ <?php
+ // Each workspace row is eight times the length of the second string.
-var_dump(levenshtein('a', str_repeat('b', 2*1024*1024)));
++var_dump(levenshtein('a', str_repeat('b', 8*1024*1024)));
+ ?>
+ --EXPECTF--
+ Fatal error: Allowed memory size of %d bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
diff --cc ext/standard/tests/strings/oom_str_getcsv.phpt
index 00000000000,24a9dde093e..39e019c54a2
mode 000000,100644..100644
--- a/ext/standard/tests/strings/oom_str_getcsv.phpt
+++ b/ext/standard/tests/strings/oom_str_getcsv.phpt
@@@ -1,0 -1,15 +1,17 @@@
+ --TEST--
+ str_getcsv() doesn't reach hard limit
+ --INI--
+ memory_limit=8M
+ --SKIPIF--
+ <?php
+ if (getenv("USE_ZEND_ALLOC") === "0") die("skip Zend MM disabled");
+ ?>
+ --FILE--
+ <?php
+ // Each field costs a zval plus a zend_string.
+ var_dump(str_getcsv(str_repeat('a,', 512*1024)));
+ ?>
+ --EXPECTF--
++Deprecated: str_getcsv(): the $escape parameter must be provided as its default value will change in %s on line %d
++
+ Fatal error: Allowed memory size of %d bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
diff --cc ext/standard/url_scanner_ex.h
index e6b29632429,aeb2f6c36d9..19ed7c39a8b
--- a/ext/standard/url_scanner_ex.h
+++ b/ext/standard/url_scanner_ex.h
@@@ -22,14 -22,15 +22,15 @@@ PHP_MSHUTDOWN_FUNCTION(url_scanner_ex)
PHP_RINIT_FUNCTION(url_scanner_ex);
PHP_RSHUTDOWN_FUNCTION(url_scanner_ex);
+ void php_url_scanner_early_rinit(void);
-PHPAPI char *php_url_scanner_adapt_single_url(const char *url, size_t urllen, const char *name, const char *value, size_t *newlen, int encode);
-PHPAPI int php_url_scanner_add_session_var(const char *name, size_t name_len, const char *value, size_t value_len, int encode);
-PHPAPI int php_url_scanner_reset_session_var(zend_string *name, int encode);
-PHPAPI int php_url_scanner_reset_session_vars(void);
-PHPAPI int php_url_scanner_add_var(const char *name, size_t name_len, const char *value, size_t value_len, int encode);
-PHPAPI int php_url_scanner_reset_var(zend_string *name, int encode);
-PHPAPI int php_url_scanner_reset_vars(void);
+PHPAPI char *php_url_scanner_adapt_single_url(const char *url, size_t urllen, const char *name, const char *value, size_t *newlen, bool encode);
+PHPAPI zend_result php_url_scanner_add_session_var(const char *name, size_t name_len, const char *value, size_t value_len, bool encode);
+PHPAPI zend_result php_url_scanner_reset_session_var(zend_string *name, int encode);
+PHPAPI zend_result php_url_scanner_reset_session_vars(void);
+PHPAPI zend_result php_url_scanner_add_var(const char *name, size_t name_len, const char *value, size_t value_len, bool encode);
+PHPAPI zend_result php_url_scanner_reset_var(zend_string *name, int encode);
+PHPAPI zend_result php_url_scanner_reset_vars(void);
#include "zend_smart_str_public.h"
diff --cc ext/standard/url_scanner_ex.re
index 61b55a6ef28,1eb1cb98c33..9e76c53b55e
--- a/ext/standard/url_scanner_ex.re
+++ b/ext/standard/url_scanner_ex.re
@@@ -681,9 -676,18 +681,16 @@@ static void php_url_scanner_ex_deactiva
smart_str_free(&ctx->tag);
smart_str_free(&ctx->arg);
smart_str_free(&ctx->attr_val);
-
- return SUCCESS;
}
+ void php_url_scanner_early_rinit(void)
+ {
+ /* See php_basic_early_rinit() */
+ php_url_scanner_ex_activate(1);
+ php_url_scanner_ex_activate(0);
+ }
+
-static inline void php_url_scanner_session_handler_impl(char *output, size_t output_len, char **handled_output, size_t *handled_output_len, int mode, int type)
+static inline void php_url_scanner_session_handler_impl(char *output, size_t output_len, char **handled_output, size_t *handled_output_len, int mode, bool is_session)
{
size_t len;
url_adapt_state_ex_t *url_state;
diff --cc ext/standard/var_unserializer.re
index aedc2518eb3,954ad08dbf8..43968dd40c9
--- a/ext/standard/var_unserializer.re
+++ b/ext/standard/var_unserializer.re
@@@ -323,9 -333,17 +333,12 @@@ PHPAPI void var_destroy(php_unserialize
static zend_string *unserialize_str(const unsigned char **p, size_t len, size_t maxlen)
{
size_t i, j;
- zend_string *str = zend_string_safe_alloc(1, len, 0, 0);
+ zend_string *str = zend_string_falloc(len, 0, /* fallible */ true);
+ if (UNEXPECTED(!str)) {
+ return NULL;
+ }
unsigned char *end = *(unsigned char **)p+maxlen;
- if (end < *p) {
- zend_string_efree(str);
- return NULL;
- }
-
for (i = 0; i < len; i++) {
if (*p >= end) {
zend_string_efree(str);
@@@ -1384,9 -1472,26 +1474,28 @@@ object ":" uiv ":" ["]
goto fail;
}
+ if ((*var_hash)->incomplete_classes_only) {
+ /* Do not load the enum: keep the case name, so that the value serializes
+ * back to the same bytes. */
+ zval case_zv;
+
+ YYCURSOR += 2;
+ *p = YYCURSOR;
+
+ object_init_ex(rval, PHP_IC_ENTRY);
+ if (ZEND_HAS_OOM()) {
+ goto fail;
+ }
+ php_store_class_name(rval, enum_name);
+ ZVAL_STR(&case_zv, case_name);
+ zend_hash_str_update(Z_OBJPROP_P(rval), MAGIC_MEMBER_ENUM_CASE, sizeof(MAGIC_MEMBER_ENUM_CASE) - 1, &case_zv);
+ zend_string_release_ex(enum_name, 0);
+ return 1;
+ }
+
+ BG(serialize_lock)++;
zend_class_entry *ce = zend_lookup_class(enum_name);
+ BG(serialize_lock)--;
if (!ce) {
php_error_docref(NULL, E_WARNING, "Class '%s' not found", ZSTR_VAL(enum_name));
goto fail;
diff --cc ext/xml/xml.c
index e863d0ca3c6,62286575904..a80f8ade6f6
--- a/ext/xml/xml.c
+++ b/ext/xml/xml.c
@@@ -614,8 -613,13 +617,13 @@@ static zval *xml_get_ctag(xml_parser *p
return NULL;
}
+ static zend_always_inline bool xml_handler_aborted(void)
+ {
+ return EG(exception) != NULL || ZEND_HAS_OOM();
+ }
+
-/* {{{ _xml_startElementHandler() */
-void _xml_startElementHandler(void *userData, const XML_Char *name, const XML_Char **attributes)
+/* {{{ xml_startElementHandler() */
+void xml_startElementHandler(void *userData, const XML_Char *name, const XML_Char **attributes)
{
xml_parser *parser = (xml_parser *)userData;
const char **attrs = (const char **) attributes;
@@@ -625,12 -630,15 +633,16 @@@
return;
}
+ if (UNEXPECTED(xml_handler_aborted())) {
+ return;
+ }
+
parser->level++;
- tag_name = _xml_decode_tag(parser, name);
+ tag_name = xml_decode_tag(parser, name);
- if (!Z_ISUNDEF(parser->startElementHandler)) {
+ if (ZEND_FCC_INITIALIZED(parser->startElementHandler)) {
+ zval args[3];
ZVAL_COPY(&args[0], &parser->index);
ZVAL_STRING(&args[1], SKIP_TAGSTART(ZSTR_VAL(tag_name)));
array_init(&args[2]);
@@@ -647,12 -655,18 +659,20 @@@
attributes += 2;
zend_string_release_ex(att, 0);
+
+ if (ZEND_HAS_OOM()) {
+ zend_string_release_ex(tag_name, false);
+ zval_ptr_dtor(&args[0]);
+ zval_ptr_dtor(&args[1]);
+ zval_ptr_dtor(&args[2]);
+ return;
+ }
}
- xml_call_handler(parser, &parser->startElementHandler, parser->startElementPtr, 3, args, &retval);
- zval_ptr_dtor(&retval);
+ zend_call_known_fcc(&parser->startElementHandler, /* retval */ NULL, /* param_count */ 3, args, /* named_params */ NULL);
+ zval_ptr_dtor(&args[0]);
+ zval_ptr_dtor(&args[1]);
+ zval_ptr_dtor(&args[2]);
}
if (!Z_ISUNDEF(parser->data) && !EG(exception)) {
@@@ -727,11 -748,16 +754,15 @@@ void xml_endElementHandler(void *userDa
if (!parser) {
return;
}
-
++
+ if (UNEXPECTED(xml_handler_aborted())) {
+ return;
+ }
- zval retval, args[2];
-
- zend_string *tag_name = _xml_decode_tag(parser, name);
+ zend_string *tag_name = xml_decode_tag(parser, name);
- if (!Z_ISUNDEF(parser->endElementHandler)) {
+ if (ZEND_FCC_INITIALIZED(parser->endElementHandler)) {
+ zval args[2];
ZVAL_COPY(&args[0], &parser->index);
ZVAL_STRING(&args[1], SKIP_TAGSTART(ZSTR_VAL(tag_name)));
@@@ -785,14 -811,17 +817,18 @@@ void xml_characterDataHandler(void *use
return;
}
+ if (UNEXPECTED(xml_handler_aborted())) {
+ return;
+ }
+
- zval retval, args[2];
-
- if (!Z_ISUNDEF(parser->characterDataHandler)) {
+ if (ZEND_FCC_INITIALIZED(parser->characterDataHandler)) {
+ zval args[2];
ZVAL_COPY(&args[0], &parser->index);
- _xml_xmlchar_zval(s, len, parser->target_encoding, &args[1]);
- xml_call_handler(parser, &parser->characterDataHandler, parser->characterDataPtr, 2, args, &retval);
- zval_ptr_dtor(&retval);
+ xml_xmlchar_zval(s, len, parser->target_encoding, &args[1]);
+
+ zend_call_known_fcc(&parser->characterDataHandler, /* retval */ NULL, /* param_count */ 2, args, /* named_params */ NULL);
+ zval_ptr_dtor(&args[0]);
+ zval_ptr_dtor(&args[1]);
}
if (Z_ISUNDEF(parser->data) || EG(exception)) {
diff --cc ext/zend_test/observer.c
index a7ab0c51e53,bff1d6ff934..6f3950e87fa
--- a/ext/zend_test/observer.c
+++ b/ext/zend_test/observer.c
@@@ -79,10 -78,6 +79,11 @@@ static void observer_begin(zend_execute
{
assert_observer_opline(execute_data);
- if (ZT_G(observer_set_vm_interrupt_on_begin)) {
++ if (ZT_G(observer_set_vm_interrupt_on_begin)
++ || ZT_G(observer_set_vm_interrupt_on_begin_without_callback)) {
+ zend_atomic_bool_store_ex(&EG(vm_interrupt), true);
+ }
+
if (!ZT_G(observer_show_output)) {
return;
}
@@@ -374,7 -357,6 +375,8 @@@ PHP_INI_BEGIN(
STD_PHP_INI_BOOLEAN("zend_test.observer.show_init_backtrace", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_show_init_backtrace, zend_zend_test_globals, zend_test_globals)
STD_PHP_INI_BOOLEAN("zend_test.observer.show_opcode", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_show_opcode, zend_zend_test_globals, zend_test_globals)
STD_PHP_INI_ENTRY("zend_test.observer.show_opcode_in_user_handler", "", PHP_INI_SYSTEM, OnUpdateString, observer_show_opcode_in_user_handler, zend_zend_test_globals, zend_test_globals)
+ STD_PHP_INI_BOOLEAN("zend_test.observer.set_vm_interrupt_on_begin", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_set_vm_interrupt_on_begin, zend_zend_test_globals, zend_test_globals)
++ STD_PHP_INI_BOOLEAN("zend_test.observer.set_vm_interrupt_on_begin_without_callback", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_set_vm_interrupt_on_begin_without_callback, zend_zend_test_globals, zend_test_globals)
STD_PHP_INI_BOOLEAN("zend_test.observer.fiber_init", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_fiber_init, zend_zend_test_globals, zend_test_globals)
STD_PHP_INI_BOOLEAN("zend_test.observer.fiber_switch", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_fiber_switch, zend_zend_test_globals, zend_test_globals)
STD_PHP_INI_BOOLEAN("zend_test.observer.fiber_destroy", "0", PHP_INI_SYSTEM, OnUpdateBool, observer_fiber_destroy, zend_zend_test_globals, zend_test_globals)
diff --cc ext/zend_test/php_test.h
index f32c6338d5c,fc6407955c2..9ab27f5a54f
--- a/ext/zend_test/php_test.h
+++ b/ext/zend_test/php_test.h
@@@ -45,7 -45,6 +45,8 @@@ ZEND_BEGIN_MODULE_GLOBALS(zend_test
int observer_show_init_backtrace;
int observer_show_opcode;
char *observer_show_opcode_in_user_handler;
+ int observer_set_vm_interrupt_on_begin;
++ int observer_set_vm_interrupt_on_begin_without_callback;
int observer_nesting_depth;
int observer_fiber_init;
int observer_fiber_switch;
diff --cc ext/zend_test/test.c
index 718def0b31c,0d493eeecf0..42619c62602
--- a/ext/zend_test/test.c
+++ b/ext/zend_test/test.c
@@@ -175,37 -164,44 +177,75 @@@ static ZEND_FUNCTION(zend_leak_bytes
emalloc(leakbytes);
}
+/* Reallocate a block of old_size bytes to new_size bytes and return its block
+ * size, along with the block size of a fresh allocation of new_size bytes. */
+static ZEND_FUNCTION(zend_test_erealloc_block_size)
+{
+ zend_long old_size, new_size;
+
+ ZEND_PARSE_PARAMETERS_START(2, 2)
+ Z_PARAM_LONG(old_size)
+ Z_PARAM_LONG(new_size)
+ ZEND_PARSE_PARAMETERS_END();
+
+ if (old_size < 1) {
+ zend_argument_value_error(1, "must be greater than 0");
+ RETURN_THROWS();
+ }
+ if (new_size < 1) {
+ zend_argument_value_error(2, "must be greater than 0");
+ RETURN_THROWS();
+ }
+
+ void *ptr = erealloc(emalloc(old_size), new_size);
+ void *fresh = emalloc(new_size);
+
+ array_init(return_value);
+ add_next_index_long(return_value, zend_mem_block_size(ptr));
+ add_next_index_long(return_value, zend_mem_block_size(fresh));
+
+ efree(fresh);
+ efree(ptr);
+}
+
+ static ZEND_FUNCTION(zend_test_emalloc)
+ {
+ zend_long size;
+
+ ZEND_PARSE_PARAMETERS_START(1, 1)
+ Z_PARAM_LONG(size)
+ ZEND_PARSE_PARAMETERS_END();
+
+ void *ptr = emalloc((size_t)size);
+
+ ZEND_ASSERT(ptr);
+ memset(ptr, 0, (size_t)size);
+
+ efree(ptr);
+ }
+
+ static ZEND_FUNCTION(zend_test_callback)
+ {
+ zend_fcall_info fci = empty_fcall_info;
+ zend_fcall_info_cache fci_cache = empty_fcall_info_cache;
+
+ ZEND_PARSE_PARAMETERS_START(1, 1)
+ Z_PARAM_FUNC(fci, fci_cache)
+ ZEND_PARSE_PARAMETERS_END();
+
+ void *p = malloc(1024);
+
+ zval result;
+ fci.retval = &result;
+ fci.param_count = 0;
+ fci.params = NULL;
+ zend_call_function(&fci, &fci_cache);
+ zval_ptr_dtor(&result);
+
+ /* Reachable via unwind exit. */
+ free(p);
+ }
+
/* Leak a refcounted variable */
static ZEND_FUNCTION(zend_leak_variable)
{
diff --cc ext/zend_test/test.stub.php
index c595677fc4a,02ba9d52a01..caf3fbac08e
--- a/ext/zend_test/test.stub.php
+++ b/ext/zend_test/test.stub.php
@@@ -246,8 -192,10 +246,12 @@@ namespace
function zend_leak_bytes(int $bytes = 3): void {}
+ function zend_test_erealloc_block_size(int $old_size, int $new_size): array {}
+
+ function zend_test_emalloc(int $size): void {}
+
+ function zend_test_callback(callable $callback): void {}
+
function zend_string_or_object(object|string $param): object|string {}
function zend_string_or_object_or_null(object|string|null $param): object|string|null {}
diff --cc ext/zend_test/test_arginfo.h
index 93792ac7c66,e8cc9c23544..e87fd554522
Binary files differ
diff --cc ext/zend_test/tests/observer_fiber_functions_03.phpt
index 8e267ea65a0,151f50d8b5f..a987242fafd
--- a/ext/zend_test/tests/observer_fiber_functions_03.phpt
+++ b/ext/zend_test/tests/observer_fiber_functions_03.phpt
@@@ -76,12 -71,14 +71,14 @@@ int(2
<!-- switching from fiber %s to %s -->
<resume '%s'>
</Fiber::suspend>
- <!-- init str_repeat() -->
- <str_repeat>
+ <var_dump>
+ <!-- init class@anonymous::__debugInfo() -->
+ <class@anonymous::__debugInfo>
+ </class@anonymous::__debugInfo>
- Fatal error: Allowed memory size of 104857600 bytes exhausted %s on line %d
- </str_repeat>
+ Fatal error: __debuginfo() must return an array in %s on line %d
+ </var_dump>
- </{closure}>
+ </{closure:%s:%d}>
<!-- switching from fiber %s to %s -->
<returned '%s'>
<!-- destroy: %s -->
diff --cc main/main.c
index 4870349ab3f,303985f1cfc..bc1725fa9bf
--- a/main/main.c
+++ b/main/main.c
@@@ -1005,17 -984,23 +1030,21 @@@ PHPAPI ZEND_COLD void php_verror(const
zend_string *message;
int is_function = 0;
+ if (ZEND_HAS_OOM()) {
+ return;
+ }
+
/* get error text into buffer and escape for html if necessary */
- buffer_len = (int)vspprintf(&buffer, 0, format, args);
+ zend_string *buffer = vstrpprintf(0, format, args);
if (PG(html_errors)) {
- replace_buffer = escape_html(buffer, buffer_len);
- efree(buffer);
+ zend_string *replace_buffer = escape_html(ZSTR_VAL(buffer), ZSTR_LEN(buffer));
+ zend_string_free(buffer);
if (replace_buffer) {
- buffer = ZSTR_VAL(replace_buffer);
- buffer_len = (int)ZSTR_LEN(replace_buffer);
+ buffer = replace_buffer;
} else {
- buffer = "";
- buffer_len = 0;
+ buffer = zend_empty_string;
}
}
diff --cc main/rfc1867.c
index 208f013627f,eabc8fb9984..39733e52c47
--- a/main/rfc1867.c
+++ b/main/rfc1867.c
@@@ -221,6 -223,10 +224,11 @@@ static int fill_buffer(multipart_buffe
self->buf_begin = self->buffer;
- if (SG(post_max_size) > 0 && SG(read_post_bytes) > SG(post_max_size)) {
++ zend_long post_max_size = REQUEST_PARSE_BODY_OPTION_GET(post_max_size, SG(post_max_size));
++ if (post_max_size > 0 && SG(read_post_bytes) > post_max_size) {
+ return 0;
+ }
+
/* calculate the free space in the buffer */
bytes_to_read = self->bufsize - self->bytes_in_buffer;
@@@ -867,7 -936,13 +947,13 @@@ SAPI_API SAPI_POST_HANDLER_FUNC(rfc1867
}
}
- if (++count <= PG(max_input_vars) && sapi_module.input_filter(PARSE_POST, param, &value, value_len, &new_val_len)) {
+ if (++count <= max_input_vars && sapi_module.input_filter(PARSE_POST, param, &value, value_len, &new_val_len)) {
+ if (ZEND_HAS_OOM()) {
+ efree(param);
+ efree(value);
+ goto fileupload_done;
+ }
+
if (php_rfc1867_callback != NULL) {
multipart_event_formdata event_formdata;
size_t newlength = new_val_len;
@@@ -882,12 -957,21 +968,21 @@@
efree(value);
continue;
}
+ if (ZEND_HAS_OOM()) {
+ efree(param);
+ efree(value);
+ goto fileupload_done;
+ }
new_val_len = newlength;
}
- safe_php_register_variable(param, value, new_val_len, array_ptr, 0);
+ if (!safe_php_register_variable(param, value, new_val_len, array_ptr, 0)) {
+ efree(param);
+ efree(value);
+ goto fileupload_done;
+ }
} else {
- if (count == PG(max_input_vars) + 1) {
- php_error_docref(NULL, E_WARNING, "Input variables exceeded " ZEND_LONG_FMT ". To increase the limit change max_input_vars in php.ini.", PG(max_input_vars));
+ if (count == max_input_vars + 1) {
+ EMIT_WARNING_OR_ERROR("Input variables exceeded " ZEND_LONG_FMT ". To increase the limit change max_input_vars in php.ini.", max_input_vars);
}
if (php_rfc1867_callback != NULL) {
@@@ -1026,11 -1120,15 +1131,15 @@@
cancel_upload = PHP_UPLOAD_ERROR_X;
continue;
}
+ if (ZEND_HAS_OOM()) {
+ cancel_upload = PHP_UPLOAD_ERROR_X;
+ continue;
+ }
}
- if (PG(upload_max_filesize) > 0 && (zend_long)(total_bytes+blen) > PG(upload_max_filesize)) {
+ if (upload_max_filesize > 0 && (zend_long)(total_bytes+blen) > upload_max_filesize) {
#if DEBUG_FILE_UPLOAD
- sapi_module.sapi_error(E_NOTICE, "upload_max_filesize of " ZEND_LONG_FMT " bytes exceeded - file [%s=%s] not saved", PG(upload_max_filesize), param, filename);
+ sapi_module.sapi_error(E_NOTICE, "upload_max_filesize of " ZEND_LONG_FMT " bytes exceeded - file [%s=%s] not saved", upload_max_filesize, param, filename);
#endif
cancel_upload = PHP_UPLOAD_ERROR_A;
} else if (max_file_size && ((zend_long)(total_bytes+blen) > max_file_size)) {
@@@ -1266,6 -1414,9 +1425,9 @@@ oom_with_param
}
fileupload_done:
- if (SG(post_max_size) > 0 && SG(read_post_bytes) > SG(post_max_size)) {
- sapi_module.sapi_error(E_WARNING, "Actual POST length does not match Content-Length, and exceeds " ZEND_LONG_FMT " bytes", SG(post_max_size));
++ if (post_max_size > 0 && SG(read_post_bytes) > post_max_size) {
++ sapi_module.sapi_error(E_WARNING, "Actual POST length does not match Content-Length, and exceeds " ZEND_LONG_FMT " bytes", post_max_size);
+ }
if (php_rfc1867_callback != NULL) {
multipart_event_end event_end;
diff --cc main/streams/memory.c
index cdc6c60bd97,33d4b8eae56..5e6c3d11bad
--- a/main/streams/memory.c
+++ b/main/streams/memory.c
@@@ -245,10 -253,14 +253,14 @@@ static int php_stream_memory_set_option
}
} else {
size_t old_size = ZSTR_LEN(ms->data);
- ms->data = zend_string_realloc(ms->data, newsize, 0);
+ zend_string *new_data = zend_string_frealloc(ms->data, newsize, 0, /* fallible */ true);
+ if (UNEXPECTED(!new_data)) {
+ return PHP_STREAM_OPTION_RETURN_ERR;
+ }
+ ms->data = new_data;
memset(ZSTR_VAL(ms->data) + old_size, 0, newsize - old_size);
- ZSTR_VAL(ms->data)[ZSTR_LEN(ms->data)] = '\0';
}
+ ZSTR_VAL(ms->data)[ZSTR_LEN(ms->data)] = '\0';
return PHP_STREAM_OPTION_RETURN_OK;
}
}
diff --cc sapi/fpm/tests/request_parse_body_multipart_options_post_max_size.phpt
index 00000000000,00000000000..6f8dccbb41f
new file mode 100644
--- /dev/null
+++ b/sapi/fpm/tests/request_parse_body_multipart_options_post_max_size.phpt
@@@ -1,0 -1,0 +1,52 @@@
++--TEST--
++request_parse_body() post_max_size option
++--SKIPIF--
++<?php include "skipif.inc"; ?>
++--FILE--
++<?php
++
++require_once "tester.inc";
++
++$cfg = <<<'EOT'
++[global]
++error_log = {{FILE:LOG}}
++[unconfined]
++listen = {{ADDR}}
++pm = dynamic
++pm.max_children = 5
++pm.start_servers = 1
++pm.min_spare_servers = 1
++pm.max_spare_servers = 3
++php_admin_value[post_max_size] = 10K
++php_admin_value[html_errors] = 0
++EOT;
++
++$code = <<<'EOT'
++<?php
++[$post, $files] = request_parse_body(['post_max_size' => '100K']);
++var_dump(strlen($post['field']));
++EOT;
++
++$tester = new FPM\Tester($cfg, $code);
++$tester->start();
++$tester->expectLogStartNotices();
++echo $tester
++ ->request(method: 'PUT', stdin: [
++ 'parts' => [[
++ 'name' => 'field',
++ 'value' => str_repeat('x', 50 * 1024),
++ ]],
++ ])
++ ->getBody();
++$tester->terminate();
++$tester->expectLogTerminatingNotices();
++$tester->close();
++
++?>
++--EXPECT--
++int(51200)
++--CLEAN--
++<?php
++require_once "tester.inc";
++FPM\Tester::clean();
++?>