Commit 0499935dfa3 for php
commit 0499935dfa3872e236467d0096fd3e5e84e63e50
Author: Jorg Adam Sowa <jorg.sowa@gmail.com>
Date: Sat Oct 24 21:28:45 2026 +0200
Update IR (#24298)
* Update IR
IR commit: ed32c881a0d7c14ce7ace6ac00dead75647b69c6
* Update IR
IR commit: 563b51a8c2c07aadb13db897abc04b5cf08d151b
* Update IR
IR commit: a4d13e263dd3010de06cd05235e006bfefa927a5
* Update IR
IR commit: bb044213cc1bc835613d0172ee95c52360a00524
diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh
index bc9c0a428b7..594203b7021 100755
--- a/.github/scripts/download-bundled/jit-ir.sh
+++ b/.github/scripts/download-bundled/jit-ir.sh
@@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.."
tmp_dir=/tmp/php-src-download-bundled/jit-ir
rm -rf "$tmp_dir"
-revision=93f92f5b5a49a02295685e340ebb0b83562cbe79
+revision=bb044213cc1bc835613d0172ee95c52360a00524
git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir"
diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4
index 2a1ac4e332f..224e9e54a2e 100644
--- a/ext/opcache/config.m4
+++ b/ext/opcache/config.m4
@@ -48,6 +48,7 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [
jit/ir/ir_patch.c
jit/ir/ir_perf.c
jit/ir/ir_ra.c
+ jit/ir/ir_ra_simple.c
jit/ir/ir_save.c
jit/ir/ir_sccp.c
jit/ir/ir_strtab.c
diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32
index cd22b363db0..f92b02e5967 100644
--- a/ext/opcache/config.w32
+++ b/ext/opcache/config.w32
@@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") {
} else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) {
var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1";
var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86");
- var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_save.c \
+ var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_save.c \
ir_dump.c ir_check.c ir_patch.c";
DEFINE("IR_TARGET", ir_target);
diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32
index 7caf8145887..b5273a1b86e 100644
--- a/ext/opcache/jit/Makefile.frag.w32
+++ b/ext/opcache/jit/Makefile.frag.w32
@@ -19,6 +19,11 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra.obj: \
ext\opcache\jit\ir\ir_private.h \
ext\opcache\jit\ir\ir_x86.h
+$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \
+ ext\opcache\jit\ir\ir.h \
+ ext\opcache\jit\ir\ir_private.h \
+ ext\opcache\jit\ir\ir_x86.h
+
$(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \
ext\opcache\jit\ir\ir.h \
ext\opcache\jit\ir\ir_private.h \
diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.h b/ext/opcache/jit/ir/dynasm/dasm_arm.h
index eaa94d9c2f7..97f541891d5 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_arm.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_arm.h
@@ -1,6 +1,6 @@
/*
** DynASM ARM encoding engine.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.lua b/ext/opcache/jit/ir/dynasm/dasm_arm.lua
index edb575366ac..7d95e453064 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_arm.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_arm.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM ARM module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.h b/ext/opcache/jit/ir/dynasm/dasm_arm64.h
index 8a22974ad3f..df8d5434d3a 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_arm64.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.h
@@ -1,6 +1,6 @@
/*
** DynASM ARM64 encoding engine.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
@@ -178,44 +178,46 @@ static int dasm_imm12(unsigned int n)
return -1;
}
-static int dasm_ffs(unsigned long long x)
+#if defined(__GNUC__) || defined(__clang__)
+#define dasm_ctz(x) __builtin_ctzll(x)
+#define dasm_clz(x) __builtin_clzll(x)
+#else
+static int dasm_ctz(unsigned long long x)
+{
+ int n = 0;
+ x = ~x;
+ while (x & 1) { x >>= 1; n++; }
+ return n;
+}
+
+static int dasm_clz(unsigned long long x)
{
- int n = -1;
- while (x) { x >>= 1; n++; }
+ int n = 0;
+ x = ~x;
+ while ((long long)x < 0) { x <<= 1; n++; }
return n;
}
+#endif
+
+static unsigned long long dasm_ror(unsigned long long x, int n)
+{
+ return (x << (-n & 63)) | (x >> n);
+}
static int dasm_imm13(int lo, int hi)
{
- int inv = 0, w = 64, s = 0xfff, xa, xb;
+ /* Thanks to: https://dougallj.wordpress.com/2021/10/30/ */
+ int rot, ones, size, immr, imms;
unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo;
- unsigned long long m = 1ULL, a, b, c;
- if (n & 1) { n = ~n; inv = 1; }
- a = n & (unsigned long long)-(long long)n;
- b = (n+a)&(unsigned long long)-(long long)(n+a);
- c = (n+a-b)&(unsigned long long)-(long long)(n+a-b);
- xa = dasm_ffs(a); xb = dasm_ffs(b);
- if (c) {
- w = dasm_ffs(c) - xa;
- if (w == 32) m = 0x0000000100000001UL;
- else if (w == 16) m = 0x0001000100010001UL;
- else if (w == 8) m = 0x0101010101010101UL;
- else if (w == 4) m = 0x1111111111111111UL;
- else if (w == 2) m = 0x5555555555555555UL;
- else return -1;
- s = (-2*w & 0x3f) - 1;
- } else if (!a) {
- return -1;
- } else if (xb == -1) {
- xb = 64;
- }
- if ((b-a) * m != n) return -1;
- if (inv) {
- return ((w - xb) << 6) | (s+w+xa-xb);
- } else {
- return ((w - xa) << 6) | (s+xb-xa);
- }
- return -1;
+ if ((n+1u) <= 1u) return -1; /* Neither all-zero nor all-ones are allowed. */
+ rot = (n & (n+1u)) ? dasm_ctz(n & (n+1u)) : 64;
+ n = dasm_ror(n, rot & 63);
+ ones = dasm_ctz(~n);
+ size = dasm_clz(n) + ones;
+ if (dasm_ror(n, size & 63) != n) return -1; /* Non-repeating? */
+ immr = -rot & (size - 1);
+ imms = (-(size << 1) | (ones - 1)) & 63;
+ return ((immr | (size & 64)) << 6) | imms;
}
/* Pass 1: Store actions and args, link branches/labels, estimate offsets. */
diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua
index 501c4bd2abf..2d6fe202bcc 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM ARM64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
@@ -244,6 +244,10 @@ local map_cond = {
hs = 2, lo = 3,
}
+local map_bti = {
+ c = 0x40, j = 0x80, jc = 0xc0,
+}
+
------------------------------------------------------------------------------
local parse_reg_type
@@ -475,6 +479,12 @@ local function parse_cond(expr, inv)
return shl(bit.bxor(c, inv), 12)
end
+local function parse_map(expr, map)
+ local x = map[expr]
+ if not x then werror("bad operand") end
+ return x
+end
+
local function parse_load(params, nparams, n, op)
if params[n+2] then werror("too many operands") end
local scale = shr(op, 30)
@@ -823,11 +833,21 @@ map_op = {
tbz_3 = "36000000DTBw|36000000DTBx",
tbnz_3 = "37000000DTBw|37000000DTBx",
+ -- Branch Target Identification.
+ bti_1 = "d503241ft",
+
-- ARM64e: Pointer authentication codes (PAC).
blraaz_1 = "d63f081fNx",
+ blrabz_1 = "d63f0c1fNx",
braa_2 = "d71f0800NDx",
+ brab_2 = "d71f0c00NDx",
braaz_1 = "d61f081fNx",
+ brabz_1 = "d61f0c1fNx",
+ paciasp_0 = "d503233f",
pacibsp_0 = "d503237f",
+ autiasp_0 = "d50323bf",
+ autibsp_0 = "d50323ff",
+ retaa_0 = "d65f0bff",
retab_0 = "d65f0fff",
-- Miscellaneous instructions.
@@ -996,6 +1016,8 @@ local function parse_template(params, template, nparams, pos)
op = op + parse_cond(q, 0); n = n + 1
elseif p == "c" then
op = op + parse_cond(q, 1); n = n + 1
+ elseif p == "t" then
+ op = op + parse_map(q, map_bti); n = n + 1
else
assert(false)
diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.h b/ext/opcache/jit/ir/dynasm/dasm_mips.h
index 7800e933877..e80cf8e758d 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_mips.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_mips.h
@@ -1,6 +1,6 @@
/*
** DynASM MIPS encoding engine.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.lua b/ext/opcache/jit/ir/dynasm/dasm_mips.lua
index 1c605b681f1..4d7f5069e44 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_mips.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_mips.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM MIPS32/MIPS64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua
index c97d666b319..c0db89650ce 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM MIPS64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
-- This module just sets 64 bit mode for the combined MIPS/MIPS64 module.
diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.h b/ext/opcache/jit/ir/dynasm/dasm_ppc.h
index 4c7d7289f9d..85324e3c316 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_ppc.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.h
@@ -1,6 +1,6 @@
/*
** DynASM PPC/PPC64 encoding engine.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua
index d66ae4a0d61..88aa193395a 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM PPC/PPC64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
--
-- Support for various extensions contributed by Caio Souza Oliveira.
diff --git a/ext/opcache/jit/ir/dynasm/dasm_proto.h b/ext/opcache/jit/ir/dynasm/dasm_proto.h
index 3f50f5028f3..f7b379dfc33 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_proto.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_proto.h
@@ -1,6 +1,6 @@
/*
** DynASM encoding engine prototypes.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
diff --git a/ext/opcache/jit/ir/dynasm/dasm_x64.lua b/ext/opcache/jit/ir/dynasm/dasm_x64.lua
index 72d9bb87bff..688e1a33180 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_x64.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_x64.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM x64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
-- This module just sets 64 bit mode for the combined x86/x64 module.
diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.h b/ext/opcache/jit/ir/dynasm/dasm_x86.h
index d50299b13c1..55d6f8386e6 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_x86.h
+++ b/ext/opcache/jit/ir/dynasm/dasm_x86.h
@@ -1,6 +1,6 @@
/*
** DynASM x86 encoding engine.
-** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+** Copyright (C) 2005-2026 Mike Pall. All rights reserved.
** Released under the MIT license. See dynasm.lua for full copyright notice.
*/
diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.lua b/ext/opcache/jit/ir/dynasm/dasm_x86.lua
index 0467bf91329..e339b316163 100644
--- a/ext/opcache/jit/ir/dynasm/dasm_x86.lua
+++ b/ext/opcache/jit/ir/dynasm/dasm_x86.lua
@@ -1,7 +1,7 @@
------------------------------------------------------------------------------
-- DynASM x86/x64 module.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See dynasm.lua for full copyright notice.
------------------------------------------------------------------------------
diff --git a/ext/opcache/jit/ir/dynasm/dynasm.lua b/ext/opcache/jit/ir/dynasm/dynasm.lua
index 0d15a87282f..dc5d99de066 100644
--- a/ext/opcache/jit/ir/dynasm/dynasm.lua
+++ b/ext/opcache/jit/ir/dynasm/dynasm.lua
@@ -2,7 +2,7 @@
-- DynASM. A dynamic assembler for code generation engines.
-- Originally designed and implemented for LuaJIT.
--
--- Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+-- Copyright (C) 2005-2026 Mike Pall. All rights reserved.
-- See below for full copyright notice.
------------------------------------------------------------------------------
@@ -17,7 +17,7 @@ local _info = {
url = "https://luajit.org/dynasm.html",
license = "MIT",
copyright = [[
-Copyright (C) 2005-2023 Mike Pall. All rights reserved.
+Copyright (C) 2005-2026 Mike Pall. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c
index 120722b2dca..fd56dd8067c 100644
--- a/ext/opcache/jit/ir/ir.c
+++ b/ext/opcache/jit/ir/ir.c
@@ -379,6 +379,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted
#define ir_op_flag_xN (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | IR_OP_FLAG_VAR_INPUTS)
#define ir_op_flag_a1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
#define ir_op_flag_a2 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
+#define ir_op_flag_a2X1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
#define ir_op_kind____ IR_OPND_UNUSED
#define ir_op_kind_def IR_OPND_DATA
@@ -391,8 +392,9 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted
#define ir_op_kind_num IR_OPND_NUM
#define ir_op_kind_fld IR_OPND_STR
#define ir_op_kind_var IR_OPND_DATA
-#define ir_op_kind_prb IR_OPND_PROB
-#define ir_op_kind_opt IR_OPND_PROB
+#define ir_op_kind_opt IR_OPND_OPT
+#define ir_op_kind_prb IR_OPND_OPT
+#define ir_op_kind_alg IR_OPND_OPT
#define ir_op_kind_pro IR_OPND_PROTO
#define ir_op_kind_lbl IR_OPND_LABEL_REF
@@ -620,7 +622,7 @@ static IR_NEVER_INLINE void ir_const_hash_rehash(ir_ctx *ctx)
ctx->const_hash = ir_mem_calloc(ctx->const_hash_mask + 1, sizeof(ir_ref));
for (ref = 1 - ctx->consts_count, insn = ctx->ir_base + ref; ref < IR_TRUE; ref++, insn++) {
if (insn->op == IR_LONG_CONST) {
- hash = insn->val.u64;
+ hash = insn->val.addr;
insn->prev_const = ctx->const_hash[hash & ctx->const_hash_mask];
ctx->const_hash[hash & ctx->const_hash_mask] = ref;
ref += IR_ALIGNED_SIZE(insn->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn);
@@ -1299,6 +1301,11 @@ ir_ref ir_fold(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
if (UNEXPECTED(!(ctx->flags & IR_OPT_FOLDING))) {
if ((opt & IR_OPT_OP_MASK) == IR_PHI) {
opt |= (3 << IR_OPT_INPUTS_SHIFT);
+ } else if ((ir_op_flags[(opt & IR_OPT_OP_MASK)] & IR_OP_FLAG_COMMUTATIVE)
+ && op1 < op2
+ && IR_IS_CONST_REF(op1)) {
+ /* Move constants to op2 for better instruction selection with (this is for -O0) */
+ SWAP_REFS(op1, op2);
}
return ir_emit(ctx, opt, op1, op2, op3);
}
diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h
index 1470606a101..1d7bd07b381 100644
--- a/ext/opcache/jit/ir/ir.h
+++ b/ext/opcache/jit/ir/ir.h
@@ -65,6 +65,8 @@ extern "C" {
# define IR_TARGET_TRIPLET "x86-freebsd-sysv"
# elif defined(__NetBSD__)
# define IR_TARGET_TRIPLET "x86-netbsd-sysv"
+# elif defined(__OpenBSD__)
+# define IR_TARGET_TRIPLET "x86-openbsd-sysv"
# else
# define IR_TARGET_TRIPLET "x86-unknown-sysv"
# endif
@@ -79,6 +81,8 @@ extern "C" {
# define IR_TARGET_TRIPLET "x86_64-freebsd-sysv"
# elif defined(__NetBSD__)
# define IR_TARGET_TRIPLET "x86_64-netbsd-sysv"
+# elif defined(__OpenBSD__)
+# define IR_TARGET_TRIPLET "x86_64-openbsd-sysv"
# else
# define IR_TARGET_TRIPLET "x86_64-unknown-sysv"
# endif
@@ -93,6 +97,8 @@ extern "C" {
# define IR_TARGET_TRIPLET "aarch64-freebsd-sysv"
# elif defined(__NetBSD__)
# define IR_TARGET_TRIPLET "aarch64-netbsd-sysv"
+# elif defined(__OpenBSD__)
+# define IR_TARGET_TRIPLET "aarch64-openbsd-sysv"
# else
# define IR_TARGET_TRIPLET "aarch64-unknown-sysv"
# endif
@@ -256,9 +262,9 @@ typedef enum _ir_type {
* E - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END
* T - control IR_OP_FLAG_CONTROL + IR_OP_FLAG_BB_END + IR_OP_FLAG_TERMINATOR
* l - load IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_LOAD
- * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_STORE
- * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_CALL
- * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_ALLOC
+ * s - store IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_STORE
+ * x - call IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_CALL
+ * a - alloc IR_OP_FLAG_MEM + IR_OP_FLAG_MEM_ALLOC
* 0-3 - number of input edges
* N - number of arguments is defined in the insn->inputs_count (MERGE, PHI, CALL)
* X1-X3 - number of extra data ops
@@ -279,6 +285,7 @@ typedef enum _ir_type {
* num - number: argument number (PARAM)
* prb - branch probability 1-99 (0 - unspecified): (IF_TRUE, IF_FALSE, CASE_VAL, CASE_DEFAULT)
* opt - optional number
+ * alg - optional alignment
* pro - function prototype
* lbl - label used as value (a reference to constant): (BEGIN)
*
@@ -314,10 +321,10 @@ typedef enum _ir_type {
_(NE, d2C, def, def, ___) /* not equal */ \
\
/* comparison ops (order matters, LT must be a modulo of 4 !!!) */ \
- _(LT, d2, def, def, ___) /* less */ \
- _(GE, d2, def, def, ___) /* greater or equal */ \
- _(LE, d2, def, def, ___) /* less or equal */ \
- _(GT, d2, def, def, ___) /* greater */ \
+ _(LT, d2, def, def, ___) /* less (signed) */ \
+ _(GE, d2, def, def, ___) /* greater or equal (signed) */ \
+ _(LE, d2, def, def, ___) /* less or equal (signed) */ \
+ _(GT, d2, def, def, ___) /* greater (signed) */ \
_(ULT, d2, def, def, ___) /* unsigned less */ \
_(UGE, d2, def, def, ___) /* unsigned greater or equal */ \
_(ULE, d2, def, def, ___) /* unsigned less or equal */ \
@@ -389,7 +396,7 @@ typedef enum _ir_type {
\
/* data ops */ \
_(PARAM, p1X2, reg, str, num) /* incoming parameter proj. */ \
- _(VAR, p1X1, reg, str, ___) /* local variable */ \
+ _(VAR, p1X2, reg, str, alg) /* local variable */ \
_(FUNC_ADDR, r0, ___, ___, ___) /* constant func ref */ \
_(FUNC, r0, ___, ___, ___) /* constant func ref */ \
_(SYM, r0, ___, ___, ___) /* constant symbol ref */ \
@@ -402,10 +409,10 @@ typedef enum _ir_type {
_(TAILCALL, xN, src, def, def) /* CALL+RETURN */ \
\
/* memory reference and load/store ops */ \
- _(ALLOCA, a2, src, def, ___) /* alloca(def) */ \
- _(AFREE, a2, src, def, ___) /* revert alloca(def) */ \
+ _(ALLOCA, a2X1, src, def, alg) /* alloca(def) */ \
+ _(AFREE, s2, src, def, ___) /* revert alloca(def) */ \
_(BLOCK_BEGIN, a1, src, ___, ___) /* stacksave */ \
- _(BLOCK_END, a2, src, def, ___) /* stackrestore */ \
+ _(BLOCK_END, s2, src, def, ___) /* stackrestore */ \
_(VLOAD, l2, src, var, ___) /* load value of local var */ \
_(VLOAD_v, l2, src, var, ___) /* volatile variant of VLOAD */ \
_(VSTORE, s3, src, var, def) /* store value to local var */ \
@@ -620,6 +627,7 @@ void ir_strtab_init(ir_strtab *strtab, uint32_t count, uint32_t buf_size);
ir_str ir_strtab_lookup(ir_strtab *strtab, const char *str, uint32_t len, ir_str val);
ir_str ir_strtab_find(const ir_strtab *strtab, const char *str, uint32_t len);
ir_str ir_strtab_update(ir_strtab *strtab, const char *str, uint32_t len, ir_str val);
+void ir_strtab_set(ir_strtab *strtab, const char *str, uint32_t len, ir_str val);
const char *ir_strtab_str(const ir_strtab *strtab, ir_str idx);
const char *ir_strtab_strl(const ir_strtab *strtab, ir_str idx, size_t *len);
void ir_strtab_apply(const ir_strtab *strtab, ir_strtab_apply_t func);
@@ -712,7 +720,6 @@ typedef struct _ir_code_buffer {
typedef struct {
int size;
int align;
- int offset;
} ir_value_param;
typedef struct _ir_bitqueue ir_bitqueue;
@@ -747,8 +754,8 @@ struct _ir_ctx {
uint32_t *cfg_map; /* map of instructions to basic block number */
uint32_t *cfg_schedule; /* BB order for code generation */
uint32_t *rules; /* array of target specific code-generation rules (for each instruction) */
- uint32_t *vregs;
- ir_ref vregs_count;
+ int32_t *vregs; /* map of insns to virtual regs (positive) or stack slots (negative) */
+ int32_t vregs_count;
ir_str func_name; /* Function name (should be set through ir_string()/ir_stringl()) */
int32_t spill_base; /* base register for special spill area (e.g. PHP VM frame pointer) */
uint64_t fixed_regset; /* fixed registers, excluded for regular register allocation */
@@ -962,6 +969,8 @@ int ir_compute_dessa_moves(ir_ctx *ctx);
int ir_reg_alloc(ir_ctx *ctx);
int ir_reg_alloc_simple(ir_ctx *ctx);
+#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx)
+
int ir_regs_number(void);
bool ir_reg_is_int(int32_t reg);
const char *ir_reg_name(int8_t reg, ir_type type);
@@ -1122,7 +1131,7 @@ IR_ALWAYS_INLINE void *ir_jit_compile(ir_ctx *ctx, int opt_level, size_t *size)
|| !ir_match(ctx)
|| !ir_assign_virtual_registers(ctx)
|| !ir_compute_dessa_moves(ctx)
- || !ir_reg_alloc_simple(ctx)) {
+ || !ir_reg_alloc_O0(ctx)) {
return NULL;
}
diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc
index 539108dec9a..a29e97c4d3e 100644
--- a/ext/opcache/jit/ir/ir_aarch64.dasc
+++ b/ext/opcache/jit/ir/ir_aarch64.dasc
@@ -390,6 +390,7 @@ const ir_call_conv_dsc ir_call_conv_aarch64_preserve_none = {
_(OP_INT) \
_(OP_FP) \
_(BINOP_INT) \
+ _(BIC) \
_(BINOP_FP) \
_(SHIFT) \
_(SHIFT_CONST) \
@@ -439,8 +440,6 @@ const ir_call_conv_dsc ir_call_conv_aarch64_preserve_none = {
#define IR_RULE_ENUM(name) IR_ ## name,
-#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA)
-
enum _ir_rule {
IR_FIRST_RULE = IR_LAST_OP,
IR_RULES(IR_RULE_ENUM)
@@ -476,6 +475,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
constraints->hints_count = 0;
switch (rule & IR_RULE_MASK) {
case IR_BINOP_INT:
+ case IR_BIC:
insn = &ctx->ir_base[ref];
n = 0;
if (IR_IS_CONST_REF(insn->op1)) {
@@ -523,7 +523,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
if (insn->op == IR_MOD) {
constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n++;
- } else if (insn->op == IR_MUL_OV && (ir_type_size[insn->type] == 8 || IR_IS_TYPE_SIGNED(insn->type))) {
+ } else if (insn->op == IR_MUL_OV && ir_type_size[insn->type] != 4) {
constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
n++;
}
@@ -601,6 +601,10 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
insn = &ctx->ir_base[ref];
constraints->tmp_regs[0] = IR_TMP_REG(2, IR_DOUBLE, IR_USE_SUB_REF, IR_SAVE_SUB_REF);
n = 1;
+ if (IR_IS_CONST_REF(insn->op1)) {
+ constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ n++;
+ }
break;
case IR_BINOP_FP:
case IR_MIN_MAX_INT:
@@ -1016,7 +1020,7 @@ static void ir_match_fuse_addr(ir_ctx *ctx, ir_ref addr_ref, ir_type type)
&& !IR_IS_CONST_REF(addr_insn->op1)
&& IR_IS_CONST_REF(addr_insn->op2) // TODO: temporary workaround
&& !IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op)
- && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_type_size[type])) {
+ && aarch64_may_encode_addr_offset(ctx->ir_base[addr_insn->op2].val.i64, ir_get_type_size(type))) {
ir_use_list *use_list = &ctx->use_lists[addr_ref];
ir_ref j = use_list->count;
@@ -1043,6 +1047,18 @@ static uint32_t ir_match_builtin_call(ir_ctx *ctx, const ir_insn *func)
return 0;
}
+static bool ir_have_calls_between(ir_ctx *ctx, ir_ref from, ir_ref to)
+{
+ ir_ref ref;
+
+ for (ref = ctx->prev_ref[to]; ref > from; ref = ctx->prev_ref[ref]) {
+ if (ctx->ir_base[ref].op == IR_CALL) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
static bool all_usages_are_fusable(ir_ctx *ctx, ir_ref ref)
{
ir_insn *insn = &ctx->ir_base[ref];
@@ -1453,6 +1469,30 @@ binop_fp:
return IR_COPY_INT | IR_MAY_REUSE;
}
}
+ if (IR_IS_TYPE_INT(insn->type) && insn->type != IR_BOOL) {
+ if (ctx->ir_base[insn->op1].op == IR_NOT
+ && !IR_IS_CONST_REF(ctx->ir_base[insn->op1].op1)
+ && ir_in_same_block(ctx, insn->op1)
+ && ctx->use_lists[insn->op1].count == 1
+ && !ir_have_calls_between(ctx, insn->op1, ref)
+ && (!IR_IS_CONST_REF(insn->op2) ||
+ !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op2].val.u64, ir_type_size[insn->type]))) {
+ /* AND(NOT(X), Y) => BIC */
+ ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT;
+ return IR_BIC;
+ } else if (ctx->ir_base[insn->op2].op == IR_NOT
+ && !IR_IS_CONST_REF(ctx->ir_base[insn->op2].op1)
+ && ir_in_same_block(ctx, insn->op2)
+ && ctx->use_lists[insn->op2].count == 1
+ && !ir_have_calls_between(ctx, insn->op2, ref)
+ && (!IR_IS_CONST_REF(insn->op1) ||
+ !aarch64_may_encode_logical_imm(ctx->ir_base[insn->op1].val.u64, ir_type_size[insn->type]))) {
+ /* AND(X, NOT(Y)) => AND(NOT(Y), X) => BIC */
+ SWAP_REFS(insn->op1, insn->op2);
+ ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_NOT;
+ return IR_BIC;
+ }
+ }
goto binop_int;
case IR_XOR:
#if IR_SIMD
@@ -1835,32 +1875,34 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem)
return IR_MEM_VAL(ir_ref_spill_slot(ctx, ref)) == IR_MEM_VAL(mem);
}
-static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg)
+static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref)
{
- ir_insn *var_insn = &ctx->ir_base[ref];
-
- IR_ASSERT(var_insn->op == IR_VAR);
- *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- return IR_SPILL_POS_TO_OFFSET(var_insn->op3);
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR ||
+ ctx->ir_base[ref].op == IR_ALLOCA ||
+ ctx->ir_base[ref].op == IR_VADDR ||
+ ctx->ir_base[ref].op == IR_PARAM);
+ IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref]));
+ return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]);
}
-static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use)
+static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref)
{
- ir_live_interval *ival;
+ int32_t offset;
- IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]);
- ival = ctx->live_intervals[ctx->vregs[ref]];
- while (ival) {
- ir_use_pos *use_pos = ival->use_pos;
- while (use_pos) {
- if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) {
- return !use_pos->next || use_pos->next->op_num == 0;
- }
- use_pos = use_pos->next;
- }
- ival = ival->next;
- }
- return 0;
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR ||
+ ctx->ir_base[ref].op == IR_ALLOCA ||
+ ctx->ir_base[ref].op == IR_VADDR ||
+ ctx->ir_base[ref].op == IR_PARAM);
+ IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref]));
+ offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]);
+ return IR_SPILL_POS_TO_OFFSET(offset);
+}
+
+static int32_t ir_var_spill_slot(ir_ctx *ctx, ir_ref ref, ir_reg *reg)
+{
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR);
+ *reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
+ return ir_local_offset(ctx, ref);
}
static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t val)
@@ -1925,6 +1967,42 @@ static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t
}
}
+static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
+ int32_t offset;
+
+ IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA);
+ offset = ir_local_offset(ctx, src);
+ if (aarch64_may_encode_imm12(offset)) {
+ | add Rx(reg), Rx(base), #offset
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset);
+ | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP)
+ }
+}
+
+static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use)
+{
+ ir_live_interval *ival;
+
+ IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]);
+ ival = ctx->live_intervals[ctx->vregs[ref]];
+ while (ival) {
+ ir_use_pos *use_pos = ival->use_pos;
+ while (use_pos) {
+ if (IR_LIVE_POS_TO_REF(use_pos->pos) == use) {
+ return !use_pos->next || use_pos->next->op_num == 0;
+ }
+ use_pos = use_pos->next;
+ }
+ ival = ival->next;
+ }
+ return 0;
+}
+
static void ir_emit_load_mem_int(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem)
{
ir_backend_data *data = ctx->data;
@@ -2162,34 +2240,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem)
}
}
-static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn)
-{
- IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR);
- return IR_SPILL_POS_TO_OFFSET(insn->op3);
-}
-
-static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src)
-{
- ir_backend_data *data = ctx->data;
- dasm_State **Dst = &data->dasm_state;
- ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- ir_insn *var_insn;
- int32_t offset;
-
- IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA);
- var_insn = &ctx->ir_base[src];
- if (var_insn->op == IR_VADDR) {
- var_insn = &ctx->ir_base[var_insn->op1];
- }
- offset = ir_local_offset(ctx, var_insn);
- if (aarch64_may_encode_imm12(offset)) {
- | add Rx(reg), Rx(base), #offset
- } else {
- ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset);
- | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP)
- }
-}
-
static void ir_resolve_label_syms(ir_ctx *ctx)
{
uint32_t b;
@@ -2246,7 +2296,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src)
} else {
ir_emit_load_imm_fp(ctx, type, reg, src);
}
- } else if (ctx->vregs[src]) {
+ } else if (ctx->vregs[src] > 0) {
ir_mem mem = ir_ref_spill_slot(ctx, src);
ir_emit_load_mem(ctx, type, reg, mem);
} else {
@@ -2842,13 +2892,73 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| ASM_REG_REG_REG_OP add, type, def_reg, op1_reg, op2_reg
break;
case IR_ADD_OV:
- | ASM_REG_REG_REG_OP adds, type, def_reg, op1_reg, op2_reg
+ if (ir_type_size[type] >= 4) {
+ | ASM_REG_REG_REG_OP adds, type, def_reg, op1_reg, op2_reg
+ } else {
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
+ }
+ if (ir_type_size[type] == 2) {
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxth Rw(def_reg), Rw(op1_reg)
+ | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxth
+ | cmp Rw(def_reg), Rw(def_reg), sxth
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xffff
+ | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxth
+ | cmp Rw(def_reg), Rw(def_reg), uxth
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xffff
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxtb Rw(def_reg), Rw(op1_reg)
+ | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxtb
+ | cmp Rw(def_reg), Rw(def_reg), sxtb
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xff
+ | add Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxtb
+ | cmp Rw(def_reg), Rw(def_reg), uxtb
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xff
+ }
+ }
break;
case IR_SUB:
| ASM_REG_REG_REG_OP sub, type, def_reg, op1_reg, op2_reg
break;
case IR_SUB_OV:
- | ASM_REG_REG_REG_OP subs, type, def_reg, op1_reg, op2_reg
+ if (ir_type_size[type] >= 4) {
+ | ASM_REG_REG_REG_OP subs, type, def_reg, op1_reg, op2_reg
+ } else {
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
+ }
+ if (ir_type_size[type] == 2) {
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxth Rw(def_reg), Rw(op1_reg)
+ | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxth
+ | cmp Rw(def_reg), Rw(def_reg), sxth
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xffff
+ | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxth
+ | cmp Rw(def_reg), Rw(def_reg), uxth
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xffff
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxtb Rw(def_reg), Rw(op1_reg)
+ | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), sxtb
+ | cmp Rw(def_reg), Rw(def_reg), sxtb
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xff
+ | sub Rw(def_reg), Rw(def_reg), Rw(op2_reg), uxtb
+ | cmp Rw(def_reg), Rw(def_reg), uxtb
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xff
+ }
+ }
break;
case IR_MUL:
| ASM_REG_REG_REG_OP mul, type, def_reg, op1_reg, op2_reg
@@ -2868,17 +2978,51 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| mul Rx(def_reg), Rx(op1_reg), Rx(op2_reg)
| cmp Rx(tmp_reg), xzr
}
- } else {
+ } else if (ir_type_size[type] == 4) {
if (IR_IS_TYPE_SIGNED(type)) {
- tmp_reg = ctx->regs[def][3];
- IR_ASSERT(tmp_reg != IR_REG_NONE);
| smull Rx(def_reg), Rw(op1_reg), Rw(op2_reg)
- | asr Rx(tmp_reg), Rx(def_reg), #32
- | cmp Rx(tmp_reg), Rx(def_reg), asr #31
+ | cmp Rx(def_reg), Rw(def_reg), sxtw
} else {
| umull Rx(def_reg), Rw(op1_reg), Rw(op2_reg)
- | cmp xzr, Rx(def_reg), lsr #32
+ | tst Rx(def_reg), #0xffffffff00000000
+ }
+ } else if (ir_type_size[type] == 2) {
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
+ }
+ tmp_reg = ctx->regs[def][3];
+ IR_ASSERT(tmp_reg != IR_REG_NONE && tmp_reg != def_reg);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxth Rw(def_reg), Rw(op1_reg)
+ | sxth Rw(tmp_reg), Rw(op2_reg)
+ | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg)
+ | cmp Rw(def_reg), Rw(def_reg), sxth
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xffff
+ | and Rw(tmp_reg), Rw(op2_reg), #0xffff
+ | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg)
+ | cmp Rw(def_reg), Rw(def_reg), uxth
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xffff
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
}
+ tmp_reg = ctx->regs[def][3];
+ IR_ASSERT(tmp_reg != IR_REG_NONE && tmp_reg != def_reg);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxtb Rw(def_reg), Rw(op1_reg)
+ | sxtb Rw(tmp_reg), Rw(op2_reg)
+ | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg)
+ | cmp Rw(def_reg), Rw(def_reg), sxtb
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xff
+ | and Rw(tmp_reg), Rw(op2_reg), #0xff
+ | mul Rw(def_reg), Rw(def_reg), Rw(tmp_reg)
+ | cmp Rw(def_reg), Rw(def_reg), uxtb
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xff
}
break;
case IR_DIV:
@@ -2920,13 +3064,73 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| ASM_REG_REG_IMM_OP add, type, def_reg, op1_reg, val
break;
case IR_ADD_OV:
- | ASM_REG_REG_IMM_OP adds, type, def_reg, op1_reg, val
+ if (ir_type_size[type] >= 4) {
+ | ASM_REG_REG_IMM_OP adds, type, def_reg, op1_reg, val
+ } else {
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
+ }
+ if (ir_type_size[type] == 2) {
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxth Rw(def_reg), Rw(op1_reg)
+ | add Rw(def_reg), Rw(def_reg), #((int32_t)(int16_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), sxth
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xffff
+ | add Rw(def_reg), Rw(def_reg), #((uint32_t)(uint16_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), uxth
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xffff
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxtb Rw(def_reg), Rw(op1_reg)
+ | add Rw(def_reg), Rw(def_reg), #((int32_t)(int8_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), sxtb
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xff
+ | add Rw(def_reg), Rw(def_reg), #((uint32_t)(uint8_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), uxtb
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xff
+ }
+ }
break;
case IR_SUB:
| ASM_REG_REG_IMM_OP sub, type, def_reg, op1_reg, val
break;
case IR_SUB_OV:
- | ASM_REG_REG_IMM_OP subs, type, def_reg, op1_reg, val
+ if (ir_type_size[type] >= 4) {
+ | ASM_REG_REG_IMM_OP subs, type, def_reg, op1_reg, val
+ } else {
+ if (def_reg == op2_reg) {
+ SWAP_REGS(op1_reg, op2_reg);
+ }
+ if (ir_type_size[type] == 2) {
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxth Rw(def_reg), Rw(op1_reg)
+ | sub Rw(def_reg), Rw(def_reg), #((int32_t)(int16_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), sxth
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xffff
+ | sub Rw(def_reg), Rw(def_reg), #((uint32_t)(uint16_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), uxth
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xffff
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ if (IR_IS_TYPE_SIGNED(type)) {
+ | sxtb Rw(def_reg), Rw(op1_reg)
+ | sub Rw(def_reg), Rw(def_reg), #((int32_t)(int8_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), sxtb
+ } else {
+ | and Rw(def_reg), Rw(op1_reg), #0xff
+ | sub Rw(def_reg), Rw(def_reg), #((uint32_t)(uint8_t)val)
+ | cmp Rw(def_reg), Rw(def_reg), uxtb
+ }
+ | and Rw(def_reg), Rw(def_reg), #0xff
+ }
+ }
break;
case IR_OR:
if (ir_type_size[type] == 8) {
@@ -2959,6 +3163,52 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
}
+static void ir_emit_bic(ir_ctx *ctx, ir_ref def, ir_insn *insn)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_type type = insn->type;
+ ir_ref not_ref = insn->op1;
+ ir_ref op1 = ctx->ir_base[not_ref].op1;
+ ir_ref op2 = insn->op2;
+ ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
+ ir_reg op1_reg;
+ ir_reg op2_reg = ctx->regs[def][2];
+
+ IR_ASSERT(ctx->ir_base[not_ref].op == IR_NOT);
+ if (UNEXPECTED(ctx->rules[not_ref] & IR_FUSED_REG)) {
+ op1_reg = ir_get_fused_reg(ctx, def, not_ref, 1);
+ } else {
+ op1_reg = ctx->regs[not_ref][1];
+ }
+
+ IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE);
+
+ if (IR_REG_SPILLED(op1_reg)) {
+ op1_reg = IR_REG_NUM(op1_reg);
+ ir_emit_load(ctx, type, op1_reg, op1);
+ }
+
+ if (op1 == op2) {
+ op2_reg = IR_REG_NUM(op1_reg);
+ } else if (op2_reg != IR_REG_NONE) {
+ if (IR_REG_SPILLED(op2_reg)) {
+ op2_reg = IR_REG_NUM(op2_reg);
+ ir_emit_load(ctx, type, op2_reg, op2);
+ }
+ } else {
+ IR_ASSERT(IR_IS_CONST_REF(insn->op2));
+ op2_reg = IR_REG_INT_TMP;
+ ir_emit_load(ctx, type, op2_reg, op2);
+ }
+
+ | ASM_REG_REG_REG_OP bic, type, def_reg, op2_reg, op1_reg
+
+ if (IR_REG_SPILLED(ctx->regs[def][0])) {
+ ir_emit_store(ctx, type, def, def_reg);
+ }
+}
+
static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
{
ir_backend_data *data = ctx->data;
@@ -3040,12 +3290,15 @@ static void ir_emit_overflow(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(def_reg != IR_REG_NONE);
IR_ASSERT(IR_IS_TYPE_INT(type));
- if (math_insn->op == IR_MUL_OV) {
+ if (math_insn->op == IR_MUL_OV || ir_type_size[math_insn->type] < 4) {
| cset Rw(def_reg), ne
} else if (IR_IS_TYPE_SIGNED(type)) {
| cset Rw(def_reg), vs
- } else {
+ } else if (math_insn->op == IR_ADD_OV) {
| cset Rw(def_reg), cs
+ } else {
+ IR_ASSERT(math_insn->op == IR_SUB_OV);
+ | cset Rw(def_reg), cc
}
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
@@ -3071,7 +3324,7 @@ static void ir_emit_overflow_and_branch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_
false_block = 0;
}
- if (math_insn->op == IR_MUL_OV) {
+ if (math_insn->op == IR_MUL_OV || ir_type_size[math_insn->type] < 4) {
if (reverse) {
| beq =>true_block
} else {
@@ -3083,12 +3336,19 @@ static void ir_emit_overflow_and_branch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_
} else {
| bvs =>true_block
}
- } else {
+ } else if (math_insn->op == IR_ADD_OV) {
if (reverse) {
| bcc =>true_block
} else {
| bcs =>true_block
}
+ } else {
+ IR_ASSERT(math_insn->op == IR_SUB_OV);
+ if (reverse) {
+ | bcs =>true_block
+ } else {
+ | bcc =>true_block
+ }
}
if (false_block) {
| b =>false_block
@@ -3563,7 +3823,11 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| ASM_REG_REG_OP clz, src_type, def_reg, def_reg
} else {
IR_ASSERT(insn->op == IR_BSWAP);
- | ASM_REG_REG_OP rev, src_type, def_reg, op1_reg
+ if (ir_type_size[src_type] == 2) {
+ | rev16 Rw(def_reg), Rw(op1_reg)
+ } else {
+ | ASM_REG_REG_OP rev, src_type, def_reg, op1_reg
+ }
}
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
@@ -3733,7 +3997,7 @@ static void ir_emit_fix_type(ir_ctx *ctx, ir_type type, ir_reg op1_reg)
}
}
-static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_reg op1_reg, ir_ref op1, ir_reg op2_reg, ir_ref op2)
+static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_op op, ir_type type, ir_reg op1_reg, ir_ref op1, ir_reg op2_reg, ir_ref op2)
{
ir_backend_data *data = ctx->data;
dasm_State **Dst = &data->dasm_state;
@@ -3744,7 +4008,24 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_reg op1_reg, ir
}
if (ir_type_size[type] < 4) {
- ir_emit_fix_type(ctx, type, op1_reg);
+ /* Short signed comparison */
+ if (ir_type_size[type] == 2) {
+ if (op >= IR_LT && op <= IR_GT) {
+ | sxth Rw(IR_REG_INT_TMP), Rw(op1_reg)
+ } else {
+ | uxth Rw(IR_REG_INT_TMP), Rw(op1_reg)
+ }
+ op1_reg = IR_REG_INT_TMP;
+ } else if (ir_type_size[type] == 1) {
+ if (op >= IR_LT && op <= IR_GT) {
+ | sxtb Rw(IR_REG_INT_TMP), Rw(op1_reg)
+ } else {
+ | uxtb Rw(IR_REG_INT_TMP), Rw(op1_reg)
+ }
+ op1_reg = IR_REG_INT_TMP;
+ } else {
+ IR_ASSERT(0);
+ }
}
if (op2_reg != IR_REG_NONE) {
if (ir_type_size[type] == 8) {
@@ -3752,13 +4033,13 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_reg op1_reg, ir
} else if (ir_type_size[type] == 4) {
| cmp Rw(op1_reg), Rw(op2_reg)
} else if (ir_type_size[type] == 2) {
- if (IR_IS_TYPE_SIGNED(type)) {
+ if (op >= IR_LT && op <= IR_GT) {
| cmp Rw(op1_reg), Rw(op2_reg), sxth
} else {
| cmp Rw(op1_reg), Rw(op2_reg), uxth
}
} else if (ir_type_size[type] == 1) {
- if (IR_IS_TYPE_SIGNED(type)) {
+ if (op >= IR_LT && op <= IR_GT) {
| cmp Rw(op1_reg), Rw(op2_reg), sxtb
} else {
| cmp Rw(op1_reg), Rw(op2_reg), uxtb
@@ -3773,8 +4054,34 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_reg op1_reg, ir
if (ir_type_size[type] == 8) {
| cmp Rx(op1_reg), #val
- } else {
+ } else if (ir_type_size[type] == 4) {
| cmp Rw(op1_reg), #val
+ } else if (ir_type_size[type] == 2) {
+ if (op >= IR_LT && op <= IR_GT) {
+ val = (int32_t)(int16_t)(val & 0xffff);
+ if (val < 0) {
+ val = -val;
+ | cmn Rw(op1_reg), #val
+ } else {
+ | cmp Rw(op1_reg), #val
+ }
+ } else {
+ val = val & 0xffff;
+ | cmp Rw(op1_reg), #val
+ }
+ } else if (ir_type_size[type] == 1) {
+ if (op >= IR_LT && op <= IR_GT) {
+ val = (int32_t)(int8_t)(val & 0xffff);
+ if (val < 0) {
+ val = -val;
+ | cmn Rw(op1_reg), #val
+ } else {
+ | cmp Rw(op1_reg), #val
+ }
+ } else {
+ val = val & 0xff;
+ | cmp Rw(op1_reg), #val
+ }
}
}
}
@@ -3827,7 +4134,7 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
op = IR_NE;
}
}
- ir_emit_cmp_int_common(ctx, type, op1_reg, op1, op2_reg, op2);
+ ir_emit_cmp_int_common(ctx, op, type, op1_reg, op1, op2_reg, op2);
switch (op) {
default:
IR_ASSERT(0 && "NIY binary op");
@@ -3879,8 +4186,8 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_
op1 = cmp_insn->op1;
op2 = cmp_insn->op2;
if (UNEXPECTED(ctx->rules[cmp_ref] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, root, cmp_ref, 1);
+ op2_reg = ir_get_fused_reg(ctx, root, cmp_ref, 2);
} else {
op1_reg = ctx->regs[cmp_ref][1];
op2_reg = ctx->regs[cmp_ref][2];
@@ -4133,8 +4440,8 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -4173,7 +4480,7 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
return;
}
}
- ir_emit_cmp_int_common(ctx, type, op1_reg, op1, op2_reg, op2);
+ ir_emit_cmp_int_common(ctx, op, type, op1_reg, op1, op2_reg, op2);
ir_emit_jcc(ctx, b, def, insn, next_block, op, 1);
}
@@ -4600,10 +4907,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| fmov Rw(def_reg), Rs(op1_reg-IR_REG_FP_FIRST)
}
} else if (IR_IS_CONST_REF(insn->op1)) {
- if (src_type == IR_DOUBLE) {
+ if (size == 8) {
ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i64);
} else {
- IR_ASSERT(src_type == IR_FLOAT);
+ IR_ASSERT(size <= 4);
ir_emit_load_imm_int(ctx, dst_type, def_reg, ctx->ir_base[insn->op1].val.i32);
}
} else {
@@ -4642,10 +4949,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
} else if (IR_IS_CONST_REF(insn->op1)) {
ir_emit_load_imm_int(ctx, src_type, IR_REG_INT_TMP, ctx->ir_base[insn->op1].val.i64);
- if (dst_type == IR_DOUBLE) {
+ if (size == 8) {
| fmov Rd(def_reg-IR_REG_FP_FIRST), Rx(IR_REG_INT_TMP)
} else {
- IR_ASSERT(dst_type == IR_FLOAT);
+ IR_ASSERT(size <= 4);
| fmov Rs(def_reg-IR_REG_FP_FIRST), Rw(IR_REG_INT_TMP)
}
} else {
@@ -4903,22 +5210,22 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn)
static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
{
- ir_insn *var_insn = &ctx->ir_base[insn->op2];
ir_ref type = insn->type;
ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
ir_reg fp;
- int32_t offset;
+ int32_t slot, offset;
ir_mem mem;
if (ctx->use_lists[def].count == 1) {
/* dead load */
return;
}
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR);
fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3);
+ slot = ir_local_slot(ctx, insn->op2);
+ offset = IR_SPILL_POS_TO_OFFSET(slot);
mem = IR_MEM_BO(fp, offset);
- if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) {
+ if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) {
return; // fake load
}
IR_ASSERT(def_reg != IR_REG_NONE);
@@ -4930,24 +5237,24 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
{
- ir_insn *var_insn = &ctx->ir_base[insn->op2];
ir_insn *val_insn = &ctx->ir_base[insn->op3];
ir_ref type = val_insn->type;
ir_reg op3_reg = ctx->regs[ref][3];
ir_reg fp;
- int32_t offset;
+ int32_t slot, offset;
ir_mem mem;
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR);
fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = IR_SPILL_POS_TO_OFFSET(var_insn->op3);
- IR_ASSERT(op3_reg != IR_REG_NONE);
- if (IR_REG_SPILLED(op3_reg)
+ slot = ir_local_slot(ctx, insn->op2);
+ offset = IR_SPILL_POS_TO_OFFSET(slot);
+ if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg))
&& !IR_IS_CONST_REF(insn->op3)
&& ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
- && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) {
+ && ir_is_same_mem_var(ctx, insn->op3, slot)) {
return; // fake store
}
+ IR_ASSERT(op3_reg != IR_REG_NONE);
if (IR_REG_SPILLED(op3_reg)) {
op3_reg = IR_REG_NUM(op3_reg);
ir_emit_load(ctx, type, op3_reg, insn->op3);
@@ -4967,12 +5274,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op));
if (ir_rule(ctx, addr_insn->op1) == IR_STATIC_ALLOCA) {
reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[addr_insn->op1]);
+ offset = ir_local_offset(ctx, addr_insn->op1);
offset += ctx->ir_base[addr_insn->op2].val.i32;
return IR_MEM_BO(reg, offset);
} else {
if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) {
- reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1);
+ reg = ir_get_fused_reg(ctx, root, ref, 1);
} else {
reg = ctx->regs[ref][1];
}
@@ -4985,7 +5292,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
} else {
IR_ASSERT(addr_insn->op == IR_ALLOCA || addr_insn->op == IR_VADDR);
reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[ref]);
+ offset = ir_local_offset(ctx, ref);
return IR_MEM_BO(reg, offset);
}
}
@@ -5157,7 +5464,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_type type = insn->type;
if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) {
- if (ctx->vregs[def]
+ if (ctx->vregs[def] > 0
&& ctx->live_intervals[ctx->vregs[def]]
&& ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) {
ir_emit_store(ctx, type, def, src_reg);
@@ -5384,7 +5691,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (ctx->flags & IR_USE_FRAME_POINTER) {
@@ -5394,8 +5701,18 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
fp = IR_REG_STACK_POINTER;
arg_area_offset = ctx->call_stack_size + ctx->stack_frame_size + ctx->param_stack_size;
}
- | add Rx(tmp_reg), Rx(fp), #arg_area_offset
- | str Rx(tmp_reg), [Rx(op2_reg), #offset]
+ if (aarch64_may_encode_imm12(arg_area_offset)) {
+ | add Rx(tmp_reg), Rx(fp), #arg_area_offset
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, arg_area_offset);
+ | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP)
+ }
+ if (aarch64_may_encode_addr_offset(offset, sizeof(void*))) {
+ | str Rx(tmp_reg), [Rx(op2_reg), #offset]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset);
+ | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
} else {
ir_reg fp;
int reg_save_area_offset;
@@ -5414,7 +5731,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (ctx->flags & IR_USE_FRAME_POINTER) {
@@ -5428,35 +5745,85 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
/* Set va_list.stack */
- | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset
- | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))]
+ if (aarch64_may_encode_imm12(overflow_arg_area_offset)) {
+ | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, overflow_arg_area_offset);
+ | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP)
+ }
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, stack), sizeof(void*))) {
+ | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, stack))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, stack));
+ | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
if ((ctx->flags2 & (IR_HAS_VA_ARG_GP|IR_HAS_VA_COPY)) && ctx->gp_reg_params < cc->int_param_regs_count) {
reg_save_area_offset += sizeof(void*) * cc->int_param_regs_count;
/* Set va_list.gr_top */
if (overflow_arg_area_offset != reg_save_area_offset) {
- | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset
+ if (aarch64_may_encode_imm12(reg_save_area_offset)) {
+ | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset);
+ | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP)
+ }
+ }
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_top), sizeof(void*))) {
+ | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_top));
+ | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
}
- | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_top))]
/* Set va_list.gr_offset */
| movn Rw(tmp_reg), #~(0 - (sizeof(void*) * (cc->int_param_regs_count - ctx->gp_reg_params)))
- | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))]
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) {
+ | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset));
+ | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
} else {
/* Set va_list.gr_offset */
- | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))]
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset), sizeof(void*))) {
+ | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, gr_offset));
+ | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
}
if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < cc->fp_param_regs_count) {
reg_save_area_offset += 16 * cc->fp_param_regs_count;
/* Set va_list.vr_top */
if (overflow_arg_area_offset != reg_save_area_offset || ctx->gp_reg_params < cc->int_param_regs_count) {
- | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset
+ if (aarch64_may_encode_imm12(reg_save_area_offset)) {
+ | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, reg_save_area_offset);
+ | add Rx(tmp_reg), Rx(fp), Rx(IR_REG_INT_TMP)
+ }
+ }
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_top), sizeof(void*))) {
+ | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_top));
+ | str Rx(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
}
- | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_top))]
/* Set va_list.vr_offset */
| movn Rw(tmp_reg), #~(0 - (16 * (cc->fp_param_regs_count - ctx->fp_reg_params)))
- | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))]
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) {
+ | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset));
+ | str Rw(tmp_reg), [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
} else {
/* Set va_list.vr_offset */
- | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))]
+ if (aarch64_may_encode_addr_offset(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset), sizeof(void*))) {
+ | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, vr_offset))]
+ } else {
+ ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset+offsetof(ir_aarch64_sysv_va_list, vr_offset));
+ | str wzr, [Rx(op2_reg), Rx(IR_REG_INT_TMP)]
+ }
}
}
}
@@ -5483,7 +5850,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ op2_offset = ir_local_offset(ctx, insn->op2);
}
if (op3_reg != IR_REG_NONE) {
if (IR_REG_SPILLED(op3_reg)) {
@@ -5494,7 +5861,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]);
+ op3_offset = ir_local_offset(ctx, insn->op3);
}
| ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset]
| str Rx(tmp_reg), [Rx(op2_reg), #op2_offset]
@@ -5514,7 +5881,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ op2_offset = ir_local_offset(ctx, insn->op2);
}
if (op3_reg != IR_REG_NONE) {
if (IR_REG_SPILLED(op3_reg)) {
@@ -5525,7 +5892,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]);
+ op3_offset = ir_local_offset(ctx, insn->op3);
}
| ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset]
| str Rx(tmp_reg), [Rx(op2_reg), #op2_offset]
@@ -5561,7 +5928,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
| ldr Rx(tmp_reg), [Rx(op2_reg), #offset]
if (def_reg != IR_REG_NONE) {
@@ -5589,7 +5956,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (IR_IS_TYPE_INT(type)) {
| ldr Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_aarch64_sysv_va_list, gr_offset))]
@@ -5690,10 +6057,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
IR_ASSERT(!IR_IS_SYM_CONST(val2->op));
if (IR_IS_TYPE_SIGNED(type)) {
IR_ASSERT(IR_IS_TYPE_SIGNED(val->type));
+ IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type));
min.i64 = IR_MIN(min.i64, val->val.i64);
max.i64 = IR_MAX(max.i64, val2->val.i64);
} else {
IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type));
+ IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type));
min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64);
max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64);
}
@@ -5715,10 +6084,10 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
}
/* Generate a table jmp or a sequence of calls */
- if (!has_case_range && count > 2 && (max.i64-min.i64) < count * 8) {
- int *labels = ir_mem_malloc(sizeof(int) * (max.i64 - min.i64 + 1));
+ if (!has_case_range && count > 2 && (max.u64 - min.u64) < (uint64_t)count * 8) {
+ int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.u64 - min.u64 + 1));
- for (i = 0; i <= (max.i64 - min.i64); i++) {
+ for (i = 0; i <= (int)(max.u64 - min.u64); i++) {
labels[i] = default_label;
}
p = &ctx->cfg_edges[bb->successors];
@@ -5729,7 +6098,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
val = &ctx->ir_base[use_insn->op2];
IR_ASSERT(!IR_IS_SYM_CONST(val->op));
label = ir_skip_empty_target_blocks(ctx, use_block);
- labels[val->val.i64 - min.i64] = label;
+ labels[val->val.u64 - min.u64] = label;
}
}
@@ -5775,7 +6144,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
}
|.align 8
|1:
- for (i = 0; i <= (max.i64 - min.i64); i++) {
+ for (i = 0; i <= (int)(max.u64 - min.u64); i++) {
int b = labels[i];
if (b) {
ir_block *bb = &ctx->cfg_blocks[b];
@@ -6219,7 +6588,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
- } else if (ctx->use_lists[def].count > 1) {
+ } else if (ctx->vregs[def] > 0) {
ir_emit_store(ctx, insn->type, def, cc->int_ret_reg);
}
} else {
@@ -6232,7 +6601,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_cal
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
- } else if (ctx->use_lists[def].count > 1) {
+ } else if (ctx->vregs[def] > 0) {
ir_emit_store(ctx, insn->type, def, cc->fp_ret_reg);
}
}
@@ -6533,8 +6902,8 @@ static void ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -6581,7 +6950,7 @@ static void ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *
return;
}
}
- ir_emit_cmp_int_common(ctx, type, op1_reg, op1, op2_reg, op2);
+ ir_emit_cmp_int_common(ctx, op, type, op1_reg, op1, op2_reg, op2);
if (insn->op == IR_GUARD) {
op ^= 1; // reverse
@@ -6615,7 +6984,7 @@ static void ir_emit_guard_overflow(ir_ctx *ctx, ir_ref def, ir_insn *insn)
void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
IR_ASSERT(IR_IS_TYPE_INT(type));
- if (math_insn->op == IR_MUL_OV) {
+ if (math_insn->op == IR_MUL_OV || ir_type_size[math_insn->type] < 4) {
if (insn->op == IR_GUARD) {
| beq &addr
} else {
@@ -6627,12 +6996,19 @@ static void ir_emit_guard_overflow(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
| bvs &addr
}
- } else {
+ } else if (math_insn->op == IR_ADD_OV) {
if (insn->op == IR_GUARD) {
| bcc &addr
} else {
| bcs &addr
}
+ } else {
+ IR_ASSERT(math_insn->op == IR_SUB_OV);
+ if (insn->op == IR_GUARD) {
+ | bcs &addr
+ } else {
+ | bcc &addr
+ }
}
}
@@ -8835,7 +9211,7 @@ static void ir_emit_load_params(ir_ctx *ctx)
}
fp_param_num++;
}
- if (ctx->vregs[use]) {
+ if (ctx->vregs[use] > 0) {
dst_reg = IR_REG_NUM(ctx->regs[use][0]);
IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE ||
stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos);
@@ -8857,36 +9233,6 @@ static void ir_emit_load_params(ir_ctx *ctx)
}
}
-static int ir_fix_dessa_tmps(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to, void *dessa_from_block)
-{
- ir_ref ref = ctx->cfg_blocks[(intptr_t)dessa_from_block].end;
-
- if (to == 0) {
- if (IR_IS_TYPE_INT(type)) {
- if (ctx->regs[ref][0] == IR_REG_NONE) {
- ctx->regs[ref][0] = IR_REG_X0;
- }
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type));
- if (ctx->regs[ref][1] == IR_REG_NONE) {
- ctx->regs[ref][1] = IR_REG_V0;
- }
- }
- } else if (from != 0) {
- if (IR_IS_TYPE_INT(type)) {
- if (ctx->regs[ref][0] == IR_REG_NONE) {
- ctx->regs[ref][0] = IR_REG_X0;
- }
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type));
- if (ctx->regs[ref][1] == IR_REG_NONE) {
- ctx->regs[ref][1] = IR_REG_V0;
- }
- }
- }
- return 1;
-}
-
static void ir_fix_param_spills(ir_ctx *ctx)
{
ir_use_list *use_list = &ctx->use_lists[1];
@@ -8921,12 +9267,15 @@ static void ir_fix_param_spills(ir_ctx *ctx)
fp_param_num++;
}
if (src_reg == IR_REG_NONE) {
- if (ctx->vregs[use]) {
+ if (ctx->vregs[use] > 0) {
ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]];
- if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)
- && ival->stack_spill_pos == -1
- && (ival->next || ival->reg == IR_REG_NONE)) {
+ if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) {
+ IR_ASSERT(ival->stack_spill_pos == -1);
ival->stack_spill_pos = stack_start + stack_offset;
+ if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
+ /* Remap VAR to PARAM stack slot */
+ ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
+ }
}
}
if (sizeof(void*) == 8) {
@@ -9187,6 +9536,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr)
case IR_BINOP_INT:
ir_emit_binop_int(ctx, i, insn);
break;
+ case IR_BIC:
+ ir_emit_bic(ctx, i, insn);
+ break;
case IR_BINOP_FP:
ir_emit_binop_fp(ctx, i, insn);
break;
@@ -9527,7 +9879,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr)
if (!c) {
break;
}
- w |= c << (8 * j);
+ w |= (uint32_t)(uint8_t)c << (8 * j);
i++;
}
| .long w
diff --git a/ext/opcache/jit/ir/ir_disasm.c b/ext/opcache/jit/ir/ir_disasm.c
index 0fe8aaea6e9..a4c847cb1e2 100644
--- a/ext/opcache/jit/ir/ir_disasm.c
+++ b/ext/opcache/jit/ir/ir_disasm.c
@@ -17,6 +17,9 @@
# if defined(__FreeBSD__) || defined(__DragonFly__)
# include <sys/sysctl.h>
# endif
+# if defined(__OpenBSD__)
+# include <sys/param.h>
+# endif
#endif
#include "ir.h"
@@ -599,7 +602,7 @@ int ir_disasm(const char *name,
}
if (p != q && *(q-1) == '-') {
q--;
- addr = (uint32_t)(-(int64_t)addr);
+ addr = (uint32_t)(0 - addr);
}
if (addr >= (uint64_t)(uintptr_t)start && addr < (uint64_t)(uintptr_t)orig_end) {
entry = ir_hashtab_find(&labels, (uint32_t)((uintptr_t)addr - (uintptr_t)start));
@@ -789,6 +792,12 @@ static void ir_elf_load_symbols(void)
return;
}
int fd = open(path, O_RDONLY);
+#elif defined(__OpenBSD__) && OpenBSD >= 202610
+ char path[PATH_MAX];
+ if (getexecpath(path, sizeof(path)) == -1) {
+ return;
+ }
+ int fd = open(path, O_RDONLY);
#elif defined(__sun)
int fd = open("/proc/self/path/a.out", O_RDONLY);
#elif defined(__HAIKU__)
diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c
index 8d15e12f842..ac751328411 100644
--- a/ext/opcache/jit/ir/ir_dump.c
+++ b/ext/opcache/jit/ir/ir_dump.c
@@ -400,13 +400,13 @@ void ir_dump_live_ranges(const ir_ctx *ctx, FILE *f)
fprintf(f, "TMP");
} else {
for (j = 1; j < ctx->insns_count; j++) {
- if (ctx->vregs[j] == (uint32_t)i) {
+ if (ctx->vregs[j] == i) {
break;
}
}
fprintf(f, "R%d (d_%d", i, j);
for (j++; j < ctx->insns_count; j++) {
- if (ctx->vregs[j] == (uint32_t)i) {
+ if (ctx->vregs[j] == i) {
fprintf(f, ", d_%d", j);
}
}
@@ -667,7 +667,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f)
fprintf(f, "\t");
ir_print_type_cname(insn->type, f);
fprintf(f, " d_%d", i);
- if (ctx->vregs && ctx->vregs[i]) {
+ if (ctx->vregs && ctx->vregs[i] > 0) {
fprintf(f, " {R%d}", ctx->vregs[i]);
}
if (ctx->regs) {
@@ -683,7 +683,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f)
if (flags & IR_OP_FLAG_DATA) {
ir_print_type_cname(insn->type, f);
fprintf(f, " d_%d", i);
- if (ctx->vregs && ctx->vregs[i]) {
+ if (ctx->vregs && ctx->vregs[i] > 0) {
fprintf(f, " {R%d}", ctx->vregs[i]);
}
if (ctx->regs) {
@@ -719,7 +719,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f)
} else {
fprintf(f, "%sd_%d", first ? "(" : ", ", ref);
}
- if (ctx->vregs && ref > 0 && ctx->vregs[ref]) {
+ if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) {
fprintf(f, " {R%d}", ctx->vregs[ref]);
}
if (ctx->regs) {
@@ -746,7 +746,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f)
fprintf(f, "%sfunc ", first ? "(" : ", ");
ir_print_proto(ctx, ref, f);
break;
- case IR_OPND_PROB:
+ case IR_OPND_OPT:
if (ref == 0) {
break;
}
diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c
index 8c01f56c9e3..bc3ae5859be 100644
--- a/ext/opcache/jit/ir/ir_emit.c
+++ b/ext/opcache/jit/ir/ir_emit.c
@@ -176,6 +176,62 @@ static ir_reg ir_get_param_reg(const ir_ctx *ctx, ir_ref ref)
return IR_REG_NONE;
}
+/* This remaps spill slot of the VAR, where PARAM is stored, to the PARAMs slot.
+ * This eliminates copying, but more important, this fixes va_start() compilation
+ * on systems, where it's defined through address calculation macro (e.g. MSVC x86).
+ */
+static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_slot)
+{
+ ir_use_list *param_use_list = &ctx->use_lists[param];
+
+ if (param_use_list->count >= 1) {
+ ir_ref param_store = ctx->use_edges[param_use_list->refs];
+ ir_insn *param_store_insn = &ctx->ir_base[param_store];
+
+ if (param_store_insn->op == IR_VSTORE && (ctx->rules[param_store] & IR_SKIPPED)) {
+ ir_ref var = param_store_insn->op2;
+
+ IR_ASSERT(param_store_insn->op3 == param);
+ IR_ASSERT(ctx->ir_base[var].op == IR_VAR);
+
+ /* Remap all related VADDR and VLOAD */
+ ir_use_list *var_use_list = &ctx->use_lists[var];
+ ir_ref n = var_use_list->count;
+ ir_ref *p = &ctx->use_edges[var_use_list->refs];
+ int32_t old_slot = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]);
+
+ for (; n > 0; p++, n--) {
+ ir_ref use = *p;
+ ir_insn *insn = &ctx->ir_base[use];
+
+ if (insn->op == IR_VADDR) {
+ if (ctx->rules[use] == IR_STATIC_ALLOCA) {
+ IR_ASSERT(ctx->vregs[use] == ctx->vregs[var]);
+ ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(spill_slot);
+ }
+ } else if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) {
+ if (ctx->vregs[use] > 0) {
+ ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]];
+ if (ival && ival->stack_spill_pos == old_slot) {
+ ival->stack_spill_pos = spill_slot;
+ }
+ }
+ } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) {
+ if (insn->op3 > 0 && ctx->vregs[insn->op3] > 0) {
+ ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op3]];
+ if (ival && ival->stack_spill_pos == old_slot) {
+ ival->stack_spill_pos = spill_slot;
+ }
+ }
+ }
+ }
+
+ /* Remap VAR itself */
+ ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(spill_slot);
+ }
+ }
+}
+
static int ir_get_args_regs(const ir_ctx *ctx, const ir_insn *insn, const ir_call_conv_dsc *cc, int8_t *regs)
{
int j, n;
@@ -250,7 +306,7 @@ static bool ir_is_same_mem_var(const ir_ctx *ctx, ir_ref r1, int32_t offset)
return 0;
}
- IR_ASSERT(ctx->vregs[r1]);
+ IR_ASSERT(ctx->vregs[r1] > 0);
ival1 = ctx->live_intervals[ctx->vregs[r1]];
IR_ASSERT(ival1);
o1 = ival1->stack_spill_pos;
@@ -347,7 +403,21 @@ static void *ir_jmp_addr(ir_ctx *ctx, ir_insn *insn, ir_insn *addr_insn)
return addr;
}
-static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op)
+void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op, int8_t reg)
+{
+ char key[10];
+ ir_ref ref_and_op = ref * sizeof(ir_ref) + op;
+
+ if (!ctx->fused_regs) {
+ ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab));
+ ir_strtab_init(ctx->fused_regs, 8, 128);
+ }
+ memcpy(key, &root, sizeof(ir_ref));
+ memcpy(key + 4, &ref_and_op, sizeof(ir_ref));
+ ir_strtab_set(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg);
+}
+
+static int8_t ir_get_fused_reg_ex(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op)
{
if (ctx->fused_regs) {
char key[10];
@@ -364,6 +434,11 @@ static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op)
return ((int8_t*)ctx->regs)[ref_and_op];
}
+static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op)
+{
+ return ir_get_fused_reg_ex(ctx, root, ref * sizeof(ir_ref) + op);
+}
+
#if defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Warray-bounds"
@@ -439,7 +514,7 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block
for (i = 0; i < count; i++, pos++) {
ref = ir_list_at(list, pos);
- IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]);
+ IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]);
if (!(ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILLED)) {
/* not spilled */
ir_reg reg = ctx->live_intervals[ctx->vregs[ref]]->reg;
@@ -933,6 +1008,27 @@ static uint32_t _find_mem_slot(ir_mem *mem_slots, uint32_t *mem_slots_count, ir_
return n;
}
+static bool ir_dessa_needs_spill_load(ir_ctx *ctx, ir_use_list *use_list, ir_ref skip, int op_num, ir_reg reg)
+{
+ ir_ref *p, n, use;
+ ir_insn *insn;
+
+ for (n = use_list->count, p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) {
+ use = *p;
+ if (use == skip) continue;
+ insn = &ctx->ir_base[use];
+ if (insn->op == IR_PHI) {
+ ir_reg src = ir_get_alocated_reg(ctx, use, op_num);
+
+ if (IR_REG_NUM(src) == reg) {
+ /* the same input register used twice */
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb)
{
uint32_t succ, k, n = 0;
@@ -964,7 +1060,7 @@ static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb)
ir_ref ref = *p;
ir_insn *insn = &ctx->ir_base[ref];
- if (insn->op == IR_PHI) {
+ if (insn->op == IR_PHI && ctx->vregs[ref] > 0) {
ir_ref input = ir_insn_op(insn, k);
ir_reg src = ir_get_alocated_reg(ctx, ref, k);
ir_reg dst = ctx->regs[ref][0];
@@ -979,7 +1075,21 @@ static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb)
} else if (ir_rule(ctx, input) == IR_STATIC_ALLOCA) {
/* encode local variable address */
from = -(ctx->consts_count + input);
- } else if (src != IR_REG_NONE && !IR_REG_SPILLED(src)) {
+ } else if (src != IR_REG_NONE
+ && (!IR_REG_SPILLED(src)
+ || ir_dessa_needs_spill_load(ctx, use_list, ref, k, IR_REG_NUM(src)))) {
+ if (IR_REG_SPILLED(src)) {
+ src = IR_REG_NUM(src);
+#if IR_X86_I64
+ if (insn->type == IR_I64 || insn->type == IR_U64) {
+ ir_reg reg_hi = IR_REG_I64_HI(src);
+ ir_reg reg_lo = IR_REG_I64_LO(src);
+ ir_emit_load_i64_lo(ctx, reg_lo, input);
+ ir_emit_load_i64_hi(ctx, reg_hi, input);
+ } else
+#endif
+ ir_emit_load(ctx, insn->type, src, input);
+ }
#if IR_X86_I64
if (insn->type == IR_I64 || insn->type == IR_U64) {
src_hi = IR_REG_I64_HI(src);
@@ -1393,7 +1503,7 @@ int32_t ir_get_spill_slot_offset(const ir_ctx *ctx, ir_ref ref)
{
int32_t offset;
- IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]);
+ IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]);
offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos;
IR_ASSERT(offset != -1);
return IR_SPILL_POS_TO_OFFSET(offset);
@@ -1427,551 +1537,3 @@ const ir_call_conv_dsc *ir_get_call_conv_dsc(uint32_t flags)
IR_ASSERT((flags & IR_CALL_CONV_MASK) == IR_CC_DEFAULT || (flags & IR_CALL_CONV_MASK) == IR_CC_BUILTIN);
return &ir_call_conv_default;
}
-
-/* Simple Register Allocator */
-typedef struct {
- int32_t num;
- ir_regset preserved_regs;
- ir_regset clobbered[IR_SUB_REFS_COUNT];
- struct {
- uint8_t type;
- int8_t start;
- int8_t end;
- int8_t hint;
- int8_t flags;
- ir_ref root;
- ir_ref ref;
- ir_ref op;
- } regs[32];
-} ir_reg_alloc_simple_data;
-
-static void _add_scratch(ir_reg_alloc_simple_data *x, ir_reg reg, int8_t start, int8_t end)
-{
- int8_t j;
-
- if (start < 0) start = 0; // TODO: ARGVAL support ???
- IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT);
- if (reg >= IR_REG_NUM) {
- for (j = start; j < end; j++) {
- x->clobbered[j] = IR_REGSET_UNION(x->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]);
- }
- } else {
- for (j = start; j < end; j++) {
- IR_REGSET_INCL(x->clobbered[j], reg);
- }
- }
-}
-
-static void _add_reg(ir_reg_alloc_simple_data *x, ir_type type,
- int8_t start, int8_t end, ir_reg hint, int8_t flags,
- ir_ref root, ir_ref ref, ir_ref op)
-{
- IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && x->num < 32);
- x->regs[x->num].type = type;
- x->regs[x->num].start = start;
- x->regs[x->num].end = end;
- x->regs[x->num].hint = hint;
- x->regs[x->num].flags = flags;
- x->regs[x->num].root = root;
- x->regs[x->num].ref = ref;
- x->regs[x->num].op = op;
- x->num++;
-}
-
-static ir_reg _get_free_reg(ir_type type, ir_regset available)
-{
- if (IR_IS_TYPE_INT(type)) {
- available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP);
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
- available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP);
- }
- if (!IR_REGSET_IS_EMPTY(available)) {
- return IR_REGSET_FIRST(available);
- } else {
- return IR_REG_NONE;
- }
-}
-
-static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *x, int j)
-{
- int n;
- ir_regset available;
- ir_reg reg;
-
- if (IR_IS_TYPE_INT(type)) {
- available = IR_REGSET_GP;
- if (ctx->flags & IR_USE_FRAME_POINTER) {
- IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER);
- }
-
-#if defined(IR_TARGET_X86)
- if (ir_type_size[type] == 1) {
- /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */
- IR_REGSET_EXCL(available, IR_REG_RBP);
- IR_REGSET_EXCL(available, IR_REG_RSI);
- IR_REGSET_EXCL(available, IR_REG_RDI);
- }
-#endif
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
- available = IR_REGSET_FP;
- }
- for (n = x->regs[j].start; n < x->regs[j].end; n++) {
- available = IR_REGSET_DIFFERENCE(available, x->clobbered[n]);
- }
- if (IR_REGSET_IS_EMPTY(available)) {
- fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n");
- IR_ASSERT(0);
- exit(-1);
- }
-
- reg = IR_REGSET_FIRST(available);
- if (IR_REGSET_IN(x->preserved_regs, reg)) {
- ir_regset tmp = ctx->used_preserved_regs;
- IR_REGSET_INCL(tmp, reg);
- ctx->used_preserved_regs = tmp;
- }
- return reg;
-}
-
-static void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op, int8_t reg)
-{
- char key[10];
-
- if (!ctx->fused_regs) {
- ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab));
- ir_strtab_init(ctx->fused_regs, 8, 128);
- }
- memcpy(key, &root, sizeof(ir_ref));
- memcpy(key + 4, &ref_and_op, sizeof(ir_ref));
- ir_strtab_lookup(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg);
-}
-
-static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load)
-{
- ir_use_list *use_list = &ctx->use_lists[load];
- ir_ref *p, use, i, n = use_list->count;
- ir_ref last_use = IR_UNUSED;
- ir_insn *insn;
-
- if (n) {
- for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
- use = *p;
- if (use < load || use > bb->end) return 0;
- if (use > last_use) last_use = use;
- }
- for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) {
- if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) {
- return 0;
- }
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- }
- }
- return 1;
-}
-
-static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val)
-{
- ir_ref i, n;
- ir_insn *insn;
-
- if (val < bb->start && val > store) return 0;
-
- for (i = val, insn = &ctx->ir_base[i]; i < store;) {
- if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v)
- && insn->op2 == var) {
- return 0;
- }
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- }
- return 1;
-}
-
-static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *x)
-{
- ir_ref stack[4];
- int stack_pos = 0;
- ir_target_constraints constraints;
- ir_insn *insn;
- uint32_t j, n, flags, def_flags;
- ir_ref *p, child;
-
- while (1) {
- IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED);
-
- if (!(ctx->rules[input] & IR_SIMPLE)) {
- def_flags = ir_get_target_constraints(ctx, input, &constraints);
- n = constraints.tmps_count;
- while (n > 0) {
- n--;
- if (constraints.tmp_regs[n].type) {
- ir_reg flags = 0;
- ir_ref op = constraints.tmp_regs[n].num;
-
- if (op > 0 && op <= ctx->ir_base[input].inputs_count) {
- ir_ref *ops = ctx->ir_base[input].ops;
-
- if (IR_IS_CONST_REF(ops[op])) {
- /* rematerialization */
- flags = IR_REG_SPILL_LOAD;
- } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
- /* local address rematerialization */
- flags = IR_REG_SPILL_LOAD;
- }
- }
- _add_reg(x, constraints.tmp_regs[n].type,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
- IR_UNUSED, input, op);
- } else {
- _add_scratch(x, constraints.tmp_regs[n].reg,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
- }
- }
- } else {
- def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG;
- constraints.hints_count = 0;
- }
-
- insn = &ctx->ir_base[input];
- flags = ir_op_flags[insn->op];
- n = IR_INPUT_EDGES_COUNT(flags);
- j = 1;
- p = insn->ops + j;
- if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) {
- j++;
- p++;
- }
- for (; j <= n; j++, p++) {
- IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA);
- child = *p;
- if (child > 0) {
- if (ctx->vregs[child]) {
- if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) {
- ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
- int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF;
-
- _add_reg(x, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
- ref, input, j);
- }
- } else if (ctx->rules[child] & IR_FUSED) {
- IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos)));
- stack[stack_pos++] = child;
- } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) {
- ctx->regs[input][j] = ctx->ir_base[child].op2;
- }
- }
- }
- if (!stack_pos) {
- break;
- }
- input = stack[--stack_pos];
- }
-}
-
-int ir_reg_alloc_simple(ir_ctx *ctx)
-{
- ir_reg_alloc_data data;
- uint32_t b;
- ir_block *bb;
- ir_insn *insn;
- ir_ref i, n, j, *p;
- uint32_t *rule, insn_flags;
- ir_target_constraints constraints;
- uint32_t def_flags;
- ir_reg reg;
- ir_regset scratch;
- ir_reg_alloc_simple_data x;
-
- memset(&data, 0, sizeof(data));
- data.cc = ir_get_call_conv_dsc(ctx->flags);
- ctx->data = &data;
-
- ctx->stack_frame_size = 0;
- ctx->call_stack_size = 0;
- ctx->used_preserved_regs = 0;
- ctx->used_preserved_regs = ctx->fixed_save_regset;
-
- scratch = ir_scratch_regset[data.cc->scratch_reg - IR_REG_NUM];
- x.preserved_regs = IR_REGSET_DIFFERENCE(data.cc->preserved_regs, ctx->fixed_save_regset);
-
- ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count);
- memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count);
-
- /* vregs + tmp + fixed + SRATCH + ALL */
- ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*));
-
- if (!ctx->arena) {
- ctx->arena = ir_arena_create(16 * 1024);
- }
-
- for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) {
- IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE));
- for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
- if (*rule & (IR_FUSED|IR_SKIPPED)) {
- if ((*rule & IR_RULE_MASK) == IR_ALLOCA) {
- if (insn->op == IR_VAR) {
- if (ctx->use_lists[i].count > 0) {
- insn->op3 = ir_allocate_spill_slot(ctx, insn->type);
- }
- } else if (insn->op == IR_ALLOCA) {
- if (ctx->use_lists[i].count > 0) {
- ir_insn *val = &ctx->ir_base[insn->op2];
-
- IR_ASSERT(IR_IS_CONST_REF(insn->op2));
- IR_ASSERT(IR_IS_TYPE_INT(val->type));
- IR_ASSERT(!IR_IS_SYM_CONST(val->op));
- IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0);
- IR_ASSERT(val->val.i64 < 0x7fffffff);
- insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32);
- }
- } else if (insn->op == IR_VADDR) {
- insn->op3 = ctx->ir_base[insn->op1].op3;
- }
- }
- } else {
- x.num = 0;
- for (j = 0; j < IR_SUB_REFS_COUNT; j++) {
- x.clobbered[j] = IR_REGSET_EMPTY;
- }
-
- def_flags = ir_get_target_constraints(ctx, i, &constraints);
- n = constraints.tmps_count;
- while (n) {
- n--;
-
- IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT);
- if (constraints.tmp_regs[n].type) {
- ir_reg flags = 0;
- ir_ref op = constraints.tmp_regs[n].num;
-
- if (op > 0 && op <= insn->inputs_count) {
- ir_ref *ops = insn->ops;
-
- if (IR_IS_CONST_REF(ops[op])) {
- /* rematerialization */
- flags = IR_REG_SPILL_LOAD;
- } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
- /* local address rematerialization */
- flags = IR_REG_SPILL_LOAD;
- }
- }
- _add_reg(&x, constraints.tmp_regs[n].type,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
- IR_UNUSED, i, op);
- } else {
- _add_scratch(&x, constraints.tmp_regs[n].reg,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
- }
- }
-
- if (ctx->vregs[i]) {
- reg = constraints.def_reg;
- if (!ctx->live_intervals[ctx->vregs[i]]) {
- ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
- memset(ival, 0, sizeof(ir_live_interval));
- ctx->live_intervals[ctx->vregs[i]] = ival;
- ival->type = insn->type;
- ival->reg = IR_REG_NONE;
- ival->vreg = ctx->vregs[i];
- ival->stack_spill_pos = -1;
- if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v)
- && ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) {
- ival->stack_spill_pos = ctx->ir_base[insn->op2].op3;
- reg = IR_REG_NONE;
- def_flags &= ~IR_USE_MUST_BE_IN_REG;
- } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) {
- ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
- } else if (ctx->use_lists[i].count == 1) {
- ir_ref use = ctx->use_edges[ctx->use_lists[i].refs];
- ir_insn *use_insn = &ctx->ir_base[use];
-
- if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v)
- && use_insn->op3 == i
- && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) {
- if (use_insn->op2 < i) {
- ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3;
- } else {
- ival->stack_spill_pos = ctx->ir_base[use_insn->op2].op3 =
- ir_allocate_spill_slot(ctx, ival->type);
- }
- } else {
- ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type);
- }
- } else {
- ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type);
- }
- } else if (insn->op == IR_PARAM) {
- IR_ASSERT(0 && "unexpected PARAM");
- return 0;
- }
-
- if (def_flags & IR_USE_MUST_BE_IN_REG) {
- ir_live_pos def_pos;
-
- if (reg != IR_REG_NONE) {
- def_pos = IR_SAVE_SUB_REF;
- } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- def_pos = IR_USE_SUB_REF;
- } else {
- def_pos = IR_LOAD_SUB_REF;
- }
- } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- def_pos = IR_LOAD_SUB_REF;
- } else {
- if (insn->op == IR_PARAM) {
- /* We may reuse parameter stack slot for spilling */
- ctx->live_intervals[ctx->vregs[i]]->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
- }
- def_pos = IR_DEF_SUB_REF;
- }
-
- _add_reg(&x, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE,
- IR_UNUSED, i, 0);
- }
- }
-
- n = insn->inputs_count;
- insn_flags = ir_op_flags[insn->op];
- j = 1;
- p = insn->ops + 1;
- if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) {
- j++;
- p++;
- }
- for (; j <= n; j++, p++) {
- ir_ref input = *p;
- ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
- ir_live_pos use_pos;
- uint32_t use_flags = IR_USE_FLAGS(def_flags, j);
-
- if (input > 0) {
- if (ctx->vregs[input]) {
- use_pos = IR_USE_SUB_REF;
- if (reg != IR_REG_NONE) {
- use_pos = IR_LOAD_SUB_REF;
-#if IR_X86_I64
- if (use_flags & IR_HINT_TWO_REGS) {
- IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg));
- IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg));
- } else
-#endif
- IR_REGSET_INCL(x.clobbered[IR_LOAD_SUB_REF], reg);
- } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (j == 1) {
- if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- use_pos = IR_USE_SUB_REF;
- } else {
- use_pos = IR_LOAD_SUB_REF;
- }
- } else if (input == insn->op1) {
- /* Input is the same as "op1" */
- use_pos = IR_LOAD_SUB_REF;
- }
- }
- if (use_flags & IR_USE_MUST_BE_IN_REG) {
- _add_reg(&x, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
- IR_UNUSED, i, j);
- }
- } else {
- if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) {
- ir_add_fusion_data(ctx, i, input, &x);
- } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) {
- ctx->regs[i][j] = ctx->ir_base[input].op2;
- }
- }
- }
- }
-
- for (j = 0; j < x.num; j++) {
- ir_regset available = scratch;
-#if IR_X86_I64
- ir_reg reg2;
-#endif
-
- for (n = x.regs[j].start; n < x.regs[j].end; n++) {
- available = IR_REGSET_DIFFERENCE(available, x.clobbered[n]);
- }
- reg = x.regs[j].hint;
-#if IR_X86_I64
- reg2 = IR_REG_NONE;
- if (reg != IR_REG_NONE && (x.regs[j].type == IR_I64 || x.regs[j].type == IR_U64)) {
- reg2 = IR_REG_I64_HI(reg);
- reg = IR_REG_I64_LO(reg);
- }
-#endif
- if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) {
- reg = _get_free_reg(x.regs[j].type, available);
- if (UNEXPECTED(reg == IR_REG_NONE)) {
- reg = _get_free_reg2(ctx, x.regs[j].type, &x, j);
- }
- }
- for (n = x.regs[j].start; n < x.regs[j].end; n++) {
- IR_REGSET_INCL(x.clobbered[n], reg);
- }
-#if IR_X86_I64
- if (x.regs[j].type == IR_I64 || x.regs[j].type == IR_U64) {
- IR_REGSET_EXCL(available, reg);
- if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) {
- reg2 = _get_free_reg(x.regs[j].type, available);
- if (UNEXPECTED(reg2 == IR_REG_NONE)) {
- reg2 = _get_free_reg2(ctx, x.regs[j].type, &x, j);
- }
- }
- for (n = x.regs[j].start; n < x.regs[j].end; n++) {
- IR_REGSET_INCL(x.clobbered[n], reg2);
- }
- if (reg > reg2) {
- SWAP_REGS(reg, reg2);
- }
- reg = IR_REG_I64_PAIR(reg, reg2);
- }
-#endif
- reg = reg | x.regs[j].flags;
- if (x.regs[j].op == 4 && insn->inputs_count < 4) {
- if (!ctx->tmp_regs) {
- ctx->tmp_regs = ir_mem_malloc(ctx->insns_count);
- memset(ctx->tmp_regs, -1, ctx->insns_count);
- }
- ctx->tmp_regs[x.regs[j].ref] = reg;
- } else if (!x.regs[j].root || ctx->regs[x.regs[j].ref][x.regs[j].op] == IR_REG_NONE) {
- ctx->regs[x.regs[j].ref][x.regs[j].op] = reg;
- } else if (ctx->regs[x.regs[j].ref][x.regs[j].op] != reg) {
- ctx->rules[x.regs[j].ref] |= IR_FUSED_REG;
- ir_set_fused_reg(ctx, x.regs[j].root, x.regs[j].ref * sizeof(ir_ref) + x.regs[j].op, reg);
- }
- }
- }
-
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- rule += n;
- }
- if (bb->flags & IR_BB_DESSA_MOVES) {
- ir_gen_dessa_moves(ctx, b, ir_fix_dessa_tmps, (void*)(intptr_t)b);
- }
- }
-
-#ifdef IR_TARGET_X86
- if (ctx->flags2 & IR_HAS_FP_RET_SLOT) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE);
- } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE)
- && data.cc->fp_ret_reg == IR_REG_NONE) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type);
- } else {
- ctx->ret_slot = -1;
- }
-#endif
-
- ctx->flags |= IR_NO_STACK_COMBINE;
- ir_fix_stack_frame(ctx);
- ctx->data = NULL;
-
- return 1;
-}
diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h
index abab986e0a3..bd73a3cbf41 100644
--- a/ext/opcache/jit/ir/ir_fold.h
+++ b/ext/opcache/jit/ir/ir_fold.h
@@ -81,7 +81,7 @@ IR_FOLD(LT(C_U64, C_U64))
IR_FOLD(LT(C_ADDR, C_ADDR))
{
- IR_FOLD_BOOL(op1_insn->val.addr < op2_insn->val.addr);
+ IR_FOLD_BOOL((intptr_t)op1_insn->val.addr < (intptr_t)op2_insn->val.addr);
}
IR_FOLD(LT(C_CHAR, C_CHAR))
@@ -126,7 +126,7 @@ IR_FOLD(GE(C_U64, C_U64))
IR_FOLD(GE(C_ADDR, C_ADDR))
{
- IR_FOLD_BOOL(op1_insn->val.addr >= op2_insn->val.addr);
+ IR_FOLD_BOOL((intptr_t)op1_insn->val.addr >= (intptr_t)op2_insn->val.addr);
}
IR_FOLD(GE(C_CHAR, C_CHAR))
@@ -171,7 +171,7 @@ IR_FOLD(LE(C_U64, C_U64))
IR_FOLD(LE(C_ADDR, C_ADDR))
{
- IR_FOLD_BOOL(op1_insn->val.addr <= op2_insn->val.addr);
+ IR_FOLD_BOOL((intptr_t)op1_insn->val.addr <= (intptr_t)op2_insn->val.addr);
}
IR_FOLD(LE(C_CHAR, C_CHAR))
@@ -217,7 +217,7 @@ IR_FOLD(GT(C_U64, C_U64))
IR_FOLD(GT(C_ADDR, C_ADDR))
{
- IR_FOLD_BOOL(op1_insn->val.addr > op2_insn->val.addr);
+ IR_FOLD_BOOL((intptr_t)op1_insn->val.addr > (intptr_t)op2_insn->val.addr);
}
@@ -890,7 +890,7 @@ IR_FOLD(ABS(C_I32))
if (op1_insn->val.i64 >= 0) {
IR_FOLD_COPY(op1);
} else {
- IR_FOLD_CONST_I((int32_t)-op1_insn->val.u32);
+ IR_FOLD_CONST_I((int32_t)(0 - op1_insn->val.u32));
}
}
@@ -900,7 +900,7 @@ IR_FOLD(ABS(C_I64))
if (op1_insn->val.i64 >= 0) {
IR_FOLD_COPY(op1);
} else {
- IR_FOLD_CONST_I(-op1_insn->val.u64);
+ IR_FOLD_CONST_I(0 - op1_insn->val.u64);
}
}
@@ -2675,7 +2675,7 @@ IR_FOLD(DIV(NEG, C_I32))
IR_FOLD(DIV(NEG, C_I64))
{
op1 = op1_insn->op1;
- val.i64 = -(uint64_t)op2_insn->val.i64;
+ val.u64 = (0 - op2_insn->val.u64);
op2 = ir_const(ctx, val, op2_insn->type);
IR_FOLD_RESTART;
}
@@ -3462,6 +3462,66 @@ IR_FOLD(AND(SHR, C_U64))
IR_FOLD_NEXT;
}
+IR_FOLD(AND(SAR, C_I8))
+IR_FOLD(AND(SAR, C_U8))
+{
+ if (IR_IS_CONST_REF(op1_insn->op2)) {
+ if (((uint8_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u8 & 0x7) == op2_insn->val.u8) {
+ /* (x >>s N) & (~0 >> N) => x >>u N */
+ opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt));
+ op1 = op1_insn->op1;
+ op2 = op1_insn->op2;
+ IR_FOLD_RESTART;
+ }
+ }
+ IR_FOLD_NEXT;
+}
+
+IR_FOLD(AND(SAR, C_I16))
+IR_FOLD(AND(SAR, C_U16))
+{
+ if (IR_IS_CONST_REF(op1_insn->op2)) {
+ if (((uint16_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u16 & 0xf) == op2_insn->val.u16) {
+ /* (x >>s N) & (~0 >> N) => x >>u N */
+ opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt));
+ op1 = op1_insn->op1;
+ op2 = op1_insn->op2;
+ IR_FOLD_RESTART;
+ }
+ }
+ IR_FOLD_NEXT;
+}
+
+IR_FOLD(AND(SAR, C_I32))
+IR_FOLD(AND(SAR, C_U32))
+{
+ if (IR_IS_CONST_REF(op1_insn->op2)) {
+ if (((uint32_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u32 & 0x1f) == op2_insn->val.u32) {
+ /* (x >>s N) & (~0 >> N) => x >>u N */
+ opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt));
+ op1 = op1_insn->op1;
+ op2 = op1_insn->op2;
+ IR_FOLD_RESTART;
+ }
+ }
+ IR_FOLD_NEXT;
+}
+
+IR_FOLD(AND(SAR, C_I64))
+IR_FOLD(AND(SAR, C_U64))
+{
+ if (IR_IS_CONST_REF(op1_insn->op2)) {
+ if (((uint64_t)-1) >> (ctx->ir_base[op1_insn->op2].val.u64 & 0x3f) == op2_insn->val.u64) {
+ /* (x >>s N) & (~0 >> N) => x >>u N */
+ opt = IR_OPT(IR_SHR, IR_OPT_TYPE(opt));
+ op1 = op1_insn->op1;
+ op2 = op1_insn->op2;
+ IR_FOLD_RESTART;
+ }
+ }
+ IR_FOLD_NEXT;
+}
+
IR_FOLD(AND(SHL, C_I8))
IR_FOLD(AND(SHL, C_U8))
{
@@ -3885,7 +3945,9 @@ IR_FOLD(MUL(MUL, C_ADDR))
{
if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) {
/* (x * c1) * c2 => x * (c1 * c2) */
+ uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8;
val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64;
+ val.u64 = (val.u64 << shift) >> shift;
op1 = op1_insn->op1;
op2 = ir_const(ctx, val, op2_insn->type);
IR_FOLD_RESTART;
@@ -3900,7 +3962,9 @@ IR_FOLD(MUL(MUL, C_I64))
{
if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) {
/* (x * c1) * c2 => x * (c1 * c2) */
- val.i64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64;
+ uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8;
+ val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64;
+ val.i64 = (int64_t)(val.u64 << shift) >> shift;
op1 = op1_insn->op1;
op2 = ir_const(ctx, val, op2_insn->type);
IR_FOLD_RESTART;
@@ -3974,7 +4038,7 @@ IR_FOLD(MIN(MIN, _))
IR_FOLD(MAX(MAX, _))
{
if (op1_insn->op1 == op2 || op1_insn->op2 == op2) {
- IR_FOLD_COPY(op2);
+ IR_FOLD_COPY(op1);
}
IR_FOLD_NEXT;
}
@@ -4029,6 +4093,51 @@ IR_FOLD(ADD(SHR, SHL))
}
+/* ROL/ROR by 1: SHL(x, 1) is folded to ADD(x, x) and then to MUL(x, 2),
+ * so the SHL/SHR rule above doesn't see it. */
+IR_FOLD(OR(MUL, SHR))
+IR_FOLD(ADD(MUL, SHR))
+{
+ const ir_insn *mul_insn = op1_insn;
+ const ir_insn *shr_insn = op2_insn;
+
+ if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt))
+ && mul_insn->op1 == shr_insn->op1
+ && IR_IS_CONST_REF(mul_insn->op2)
+ && IR_IS_CONST_REF(shr_insn->op2)
+ && ctx->ir_base[mul_insn->op2].val.i64 == 2
+ && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) {
+ /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */
+ op1 = shr_insn->op1;
+ op2 = shr_insn->op2;
+ opt = shr_insn->opt + 3; /* SHR -> ROR */
+ IR_FOLD_RESTART;
+ }
+ IR_FOLD_NEXT;
+}
+
+IR_FOLD(OR(SHR, MUL))
+IR_FOLD(ADD(SHR, MUL))
+{
+ const ir_insn *mul_insn = op2_insn;
+ const ir_insn *shr_insn = op1_insn;
+
+ if (IR_IS_TYPE_INT(IR_OPT_TYPE(opt))
+ && mul_insn->op1 == shr_insn->op1
+ && IR_IS_CONST_REF(mul_insn->op2)
+ && IR_IS_CONST_REF(shr_insn->op2)
+ && ctx->ir_base[mul_insn->op2].val.i64 == 2
+ && ctx->ir_base[shr_insn->op2].val.i64 == ir_type_size[IR_OPT_TYPE(opt)] * 8 - 1) {
+ /* (x * 2) | (x >> (N - 1)) -> ROR(x, N - 1) == ROL(x, 1) */
+ op1 = shr_insn->op1;
+ op2 = shr_insn->op2;
+ opt = shr_insn->opt + 3; /* SHR -> ROR */
+ IR_FOLD_RESTART;
+ }
+ IR_FOLD_NEXT;
+}
+
+
/* Swap operands (move lower ref to op2) for better CSE */
IR_FOLD(MUL(_, _))
IR_FOLD(ORDERED(_, _))
diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c
index 60422f12f6b..712411774a4 100644
--- a/ext/opcache/jit/ir/ir_gcm.c
+++ b/ext/opcache/jit/ir/ir_gcm.c
@@ -322,9 +322,13 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b)
ir_hashtab_init(&hash, use_list->count);
n = use_list->count;
for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) {
+ ir_ref src;
+
use = *p;
+ src = ctx->cfg_map[use];
insn = &ctx->ir_base[use];
- if (insn->op == IR_PHI) {
+ /* dead PHIs (src == NULL) are not scheduled, so they go together with other unscheduled uses */
+ if (src && insn->op == IR_PHI) {
ir_ref *p = insn->ops + 2; /* PHI data inputs */
ir_ref *q = ctx->ir_base[insn->op1].ops + 1; /* MERGE inputs */
ir_ref n = insn->inputs_count - 1;
@@ -359,9 +363,9 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b)
}
}
} else {
- j = i = ctx->cfg_map[use];
- if (i) {
- IR_ASSERT(i > 0);
+ j = src;
+ if (j) {
+ IR_ASSERT(j > 0);
while (ir_sparse_set_in(&data->totally_useful, ctx->cfg_blocks[j].idom)) {
j = ctx->cfg_blocks[j].idom;
}
@@ -371,14 +375,14 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b)
clone = clones_count++;
ir_hashtab_add(&hash, j, clone);
clones[clone].block = j;
- clones[clone].lca = i;
+ clones[clone].lca = src;
clones[clone].use_count = 0;
clones[clone].use = -1;
} else {
- clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, i);
+ clones[clone].lca = ir_gcm_find_lca(ctx, clones[clone].lca, src);
}
uses[uses_count].ref = use;
- uses[uses_count].block = i;
+ uses[uses_count].block = src;
uses[uses_count].next = clones[clone].use;
clones[clone].use_count++;
clones[clone].use = uses_count++;
@@ -478,7 +482,7 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b)
ir_insn *insn = &ctx->ir_base[use];
ir_ref k, l = insn->inputs_count;
- if (insn->op == IR_PHI) {
+ if (src && insn->op == IR_PHI) {
ir_insn *merge = &ctx->ir_base[insn->op1];
for (k = 2; k <= l; k++) {
j = ctx->cfg_map[ir_insn_op(merge, k - 1)];
@@ -1198,13 +1202,16 @@ int ir_schedule(ir_ctx *ctx)
consts_count += ir_count_constant(ctx, _xlat, insn->op2);
}
} else if (insn->op == IR_CASE_VAL) {
- IR_ASSERT(insn->op2 < IR_TRUE);
- consts_count += ir_count_constant(ctx, _xlat, insn->op2);
+ if (insn->op2 < IR_TRUE) {
+ consts_count += ir_count_constant(ctx, _xlat, insn->op2);
+ }
} else if (insn->op == IR_CASE_RANGE) {
- IR_ASSERT(insn->op2 < IR_TRUE);
- consts_count += ir_count_constant(ctx, _xlat, insn->op2);
- IR_ASSERT(insn->op3 < IR_TRUE);
- consts_count += ir_count_constant(ctx, _xlat, insn->op3);
+ if (insn->op2 < IR_TRUE) {
+ consts_count += ir_count_constant(ctx, _xlat, insn->op2);
+ }
+ if (insn->op3 < IR_TRUE) {
+ consts_count += ir_count_constant(ctx, _xlat, insn->op3);
+ }
}
n = insn->inputs_count;
insns_count += ir_insn_inputs_to_len(n);
@@ -1359,11 +1366,11 @@ int ir_schedule(ir_ctx *ctx)
*dst = *src;
_xlat[i] = j;
if (dst->op == IR_LONG_CONST) {
- uintptr_t n;
+ ir_ref n;
memset(dst + 1, 0, dst->long_const_size);
memcpy(dst + 1, src + 1, dst->long_const_size);
- n = IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn);
+ n = (ir_ref)IR_ALIGNED_SIZE(dst->long_const_size, sizeof(ir_insn)) / sizeof(ir_insn);
dst += n + 1;
src += n + 1;
i += n + 1;
diff --git a/ext/opcache/jit/ir/ir_perf.c b/ext/opcache/jit/ir/ir_perf.c
index 52be820088a..dca2b0eef08 100644
--- a/ext/opcache/jit/ir/ir_perf.c
+++ b/ext/opcache/jit/ir/ir_perf.c
@@ -167,6 +167,7 @@ int ir_perf_jitdump_open(void)
if (jitdump_mem == MAP_FAILED) {
close(jitdump_fd);
jitdump_fd = -1;
+ unlink(filename);
return 0;
}
@@ -179,6 +180,11 @@ int ir_perf_jitdump_open(void)
jit_hdr.time_stamp = ir_perf_timestamp();
jit_hdr.flags = 0;
if (write(jitdump_fd, &jit_hdr, sizeof(jit_hdr)) != sizeof(jit_hdr)) {
+ munmap(jitdump_mem, sysconf(_SC_PAGESIZE));
+ jitdump_mem = MAP_FAILED;
+ close(jitdump_fd);
+ jitdump_fd = -1;
+ unlink(filename);
return 0;
}
return 1;
diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h
index b4409f97df7..028acd66cb7 100644
--- a/ext/opcache/jit/ir/ir_private.h
+++ b/ext/opcache/jit/ir/ir_private.h
@@ -975,12 +975,15 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref)
#define IR_OP_FLAG_TERMINATOR (1<<14)
#define IR_OP_FLAG_PINNED (1<<15)
-#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(0<<7))
-#define IR_OP_FLAG_MEM_STORE ((0<<6)|(1<<7))
-#define IR_OP_FLAG_MEM_CALL ((1<<6)|(0<<7))
-#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7))
+#define IR_OP_FLAG_MEM_CALL ((0<<6)|(0<<7))
+#define IR_OP_FLAG_MEM_LOAD ((0<<6)|(1<<7)) /* no side effect (may be removed if not used) */
+#define IR_OP_FLAG_MEM_STORE ((1<<6)|(0<<7))
+#define IR_OP_FLAG_MEM_ALLOC ((1<<6)|(1<<7)) /* no side effect (may be removed if not used) */
#define IR_OP_FLAG_MEM_MASK ((1<<6)|(1<<7))
+#define IR_OP_FLAG_NO_SIDE_EFFECT ((0<<6)|(1<<7))
+
+
#define IR_OPND_UNUSED 0x0U
#define IR_OPND_DATA 0x1U
#define IR_OPND_CONTROL 0x2U
@@ -990,7 +993,7 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref)
#define IR_OPND_CONTROL_GUARD 0x6U
#define IR_OPND_STR 0x7U
#define IR_OPND_NUM 0x8U
-#define IR_OPND_PROB 0x9U
+#define IR_OPND_OPT 0x9U /* optional number (flags, branch propability, align, size+align) */
#define IR_OPND_PROTO 0xaU
#define IR_OP_FLAGS(op_flags, op1_flags, op2_flags, op3_flags) \
@@ -1317,12 +1320,18 @@ struct _ir_live_range {
#define IR_LIVE_INTERVAL_TEMP (1<<1)
#define IR_LIVE_INTERVAL_HAS_HINT_REGS (1<<2)
#define IR_LIVE_INTERVAL_HAS_HINT_REFS (1<<3)
-#define IR_LIVE_INTERVAL_MEM_PARAM (1<<4)
+#define IR_LIVE_INTERVAL_MEM_PARAM (1<<4) /* PARAM passed on stack */
#define IR_LIVE_INTERVAL_COALESCED (1<<5)
#define IR_LIVE_INTERVAL_SPILL_SPECIAL (1<<6) /* spill slot is pre-allocated in a special area (see ir_ctx.spill_reserved_base) */
#define IR_LIVE_INTERVAL_SPILLED (1<<7)
#define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<8)
#define IR_LIVE_INTERVAL_TWO_REGS (1<<9)
+#define IR_LIVE_INTERVAL_VAR_MEM_PARAM (1<<10) /* PARAM stack slot reused by VAR used in VSTORE(_, VAR, PARAM) */
+
+/* macros to encode and decode stack offset stored in ctx->vregs[] */
+#define IR_VREG_IS_STACK_SLOT(vreg) ((vreg) < 0)
+#define IR_VREG_TO_STACK_SLOT(vreg) ((vreg) + 0x40000000)
+#define IR_STACK_SLOT_TO_VREG(offset) ((int32_t)0xc0000000 + (offset))
struct _ir_live_interval {
uint8_t type;
@@ -1478,8 +1487,7 @@ typedef struct _ir_reg_alloc_data {
ir_live_interval **handled;
} ir_reg_alloc_data;
-int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type);
-int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size);
+int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align);
void ir_dump_reg(const ir_ctx *ctx, int8_t reg, ir_ref ref, bool store, FILE *f);
IR_ALWAYS_INLINE void ir_set_alocated_reg(ir_ctx *ctx, ir_ref ref, int op_num, int8_t reg)
@@ -1529,6 +1537,8 @@ IR_ALWAYS_INLINE int8_t ir_get_alocated_reg(const ir_ctx *ctx, ir_ref ref, int o
#define IR_MAY_REUSE (1U<<25) /* Result may reuse register of the source */
#define IR_TWO_REGS (1U<<24) /* Result needs two registers (used for 64-bit integers on x86) */
+#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA)
+
#define IR_RULE_MASK 0xffff
#define IR_MAX_REG_ARGS 64
@@ -1558,8 +1568,9 @@ typedef struct {
#define IR_SCRATCH_REG(_reg, _start, _end) \
(ir_tmp_reg){.reg=(_reg), .type=IR_VOID, .start=(_start), .end=(_end)}
-int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *constraints);
+int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *constraints);
+void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ref op, int8_t reg);
void ir_fix_stack_frame(ir_ctx *ctx);
/* Utility */
diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c
index fe052b2a097..53c3cbd8a04 100644
--- a/ext/opcache/jit/ir/ir_ra.c
+++ b/ext/opcache/jit/ir/ir_ra.c
@@ -40,8 +40,8 @@ bool ir_reg_is_int(int32_t reg)
static int ir_assign_virtual_registers_slow(ir_ctx *ctx)
{
- uint32_t *vregs;
- uint32_t vregs_count = 0;
+ int32_t *vregs;
+ int32_t vregs_count = 0;
uint32_t b;
ir_ref i, n;
ir_block *bb;
@@ -81,8 +81,8 @@ static int ir_assign_virtual_registers_slow(ir_ctx *ctx)
int ir_assign_virtual_registers(ir_ctx *ctx)
{
- uint32_t *vregs;
- uint32_t vregs_count = 0;
+ int32_t *vregs;
+ int32_t vregs_count = 0;
ir_ref i;
ir_insn *insn;
@@ -94,7 +94,7 @@ int ir_assign_virtual_registers(ir_ctx *ctx)
vregs = ir_mem_malloc(ctx->insns_count * sizeof(ir_ref));
for (i = 1, insn = &ctx->ir_base[1]; i < ctx->insns_count; i++, insn++) {
- uint32_t v = 0;
+ int32_t v = 0;
if (ctx->rules[i] && !(ctx->rules[i] & (IR_FUSED|IR_SKIPPED|IR_NO_REG))) {
uint32_t flags = ir_op_flags[insn->op];
@@ -548,9 +548,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block
IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA);
child = *p;
if (child > 0) {
- uint32_t v = ctx->vregs[child];
+ int32_t v = ctx->vregs[child];
- if (v) {
+ if (v > 0) {
use_flags = IR_FUSED_USE | IR_USE_FLAGS(def_flags, j);
reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
@@ -694,9 +694,9 @@ int ir_compute_live_ranges(ir_ctx *ctx)
if (insn->op == IR_PHI) {
ir_ref input = ir_insn_op(insn, k);
if (input > 0) {
- uint32_t v = ctx->vregs[input];
+ int32_t v = ctx->vregs[input];
- if (v) {
+ if (v > 0) {
/* live.add(phi.inputOf(b)) */
ir_bitset_incl(live, v);
/* intervals[phi.inputOf(b)].addRange(b.from, b.to) */
@@ -722,7 +722,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
uint32_t flags;
ir_ref *p;
ir_target_constraints constraints;
- uint32_t v;
+ int32_t v;
if (ctx->rules) {
int n;
@@ -730,7 +730,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
#if IR_X86_I64
if (ctx->rules[ref] & IR_TWO_REGS) {
v = ctx->vregs[ref];
- if (v) {
+ if (v > 0) {
IR_ASSERT(ctx->live_intervals[v]);
ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS;
}
@@ -742,7 +742,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
&& ctx->use_lists[ref].count > 0) {
insn = &ctx->ir_base[ref];
if (insn->op == IR_VAR || insn->op == IR_ALLOCA) {
- insn->op3 = ctx->vars;
+ ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars);
ctx->vars = ref;
}
}
@@ -770,7 +770,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
insn = &ctx->ir_base[ref];
v = ctx->vregs[ref];
- if (v) {
+ if (v > 0) {
if (insn->op != IR_PHI) {
ir_live_pos def_pos;
ir_ref hint_ref = 0;
@@ -789,7 +789,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos);
}
} else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) {
+ if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) {
hint_ref = insn->op1;
}
def_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
@@ -798,7 +798,25 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else {
if (insn->op == IR_PARAM) {
/* We may reuse parameter stack slot for spilling */
- ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
+ ir_live_interval *param_ival = ctx->live_intervals[v];
+
+ param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
+ if (ctx->rules && ctx->use_lists[ref].count > 0) {
+ ir_ref use = ctx->use_edges[ctx->use_lists[ref].refs];
+ if (use < bb->end) {
+ ir_insn *use_insn = &ctx->ir_base[use];
+ if (use_insn->op == IR_VSTORE
+ && !(ctx->rules[use] & IR_SKIPPED)
+ /* first use of VAR is VSTORE(_, VAR, PARAM) */
+ && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) {
+ IR_ASSERT(use_insn->op3 == ref);
+ /* skip VSTORE (VAR is going to be remapped to PARAM) */
+ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
+ param_ival->use_pos = param_ival->use_pos->next;
+ ctx->rules[use] = IR_SKIPPED | IR_NOP;
+ }
+ }
+ }
}
def_pos = IR_DEF_LIVE_POS_FROM_REF(ref);
}
@@ -831,11 +849,11 @@ int ir_compute_live_ranges(ir_ctx *ctx)
p = insn->ops + 2;
for (; j <= insn->inputs_count; j++, p++) {
ir_ref input = *p;
- uint32_t v;
+ int32_t v;
if (input > 0) {
v = ctx->vregs[input];
- IR_ASSERT(v);
+ IR_ASSERT(v > 0);
use_pos = IR_USE_LIVE_POS_FROM_REF(next);
if (!ir_bitset_in(live, v)) {
/* live.add(opd) */
@@ -865,12 +883,12 @@ int ir_compute_live_ranges(ir_ctx *ctx)
ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
ir_live_pos use_pos;
ir_ref hint_ref = 0;
- uint32_t v;
+ int32_t v;
uint32_t use_flags = IR_USE_FLAGS(def_flags, j);
if (input > 0) {
v = ctx->vregs[input];
- if (v) {
+ if (v > 0) {
use_pos = IR_USE_LIVE_POS_FROM_REF(ref);
if (reg != IR_REG_NONE) {
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
@@ -895,7 +913,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else if (def_flags & IR_DEF_REUSES_OP1_REG) {
if (j == 1) {
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
- IR_ASSERT(ctx->vregs[ref]);
+ IR_ASSERT(ctx->vregs[ref] > 0);
hint_ref = ref;
} else if (input == insn->op1) {
/* Input is the same as "op1" */
@@ -1022,7 +1040,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} while (0)
/* Returns the last virtual register alive at the end of the block (it is used as an already-visited marker) */
-IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b)
+IR_ALWAYS_INLINE int32_t ir_live_out_top(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b)
{
#if 0
return live_outs[b];
@@ -1035,7 +1053,7 @@ IR_ALWAYS_INLINE uint32_t ir_live_out_top(ir_ctx *ctx, uint32_t *live_outs, ir_l
}
/* Remember a virtual register alive at the end of the block */
-IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists, uint32_t b, uint32_t v)
+IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists, uint32_t b, int32_t v)
{
#if 0
ir_block *bb = &ctx->cfg_blocks[b];
@@ -1061,7 +1079,7 @@ IR_ALWAYS_INLINE void ir_live_out_push(ir_ctx *ctx, uint32_t *live_outs, ir_list
* "Computing Liveness Sets for SSA-Form Programs", Florian Brandner, Benoit Boissinot.
* Alain Darte, Benoit Dupont de Dinechin, Fabrice Rastello. TR Inria RR-7503, 2011
*/
-static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live_lists)
+static void ir_compute_live_sets(ir_ctx *ctx, int32_t *live_outs, ir_list *live_lists)
{
ir_list block_queue, fuse_queue;
ir_ref i;
@@ -1071,9 +1089,9 @@ static void ir_compute_live_sets(ir_ctx *ctx, uint32_t *live_outs, ir_list *live
/* For each virtual register explore paths from all uses to definition */
for (i = ctx->insns_count - 1; i > 0; i--) {
- uint32_t v = ctx->vregs[i];
+ int32_t v = ctx->vregs[i];
- if (v) {
+ if (v > 0) {
uint32_t def_block = ctx->cfg_map[i];
ir_use_list *use_list = &ctx->use_lists[i];
ir_ref *p, n = use_list->count;
@@ -1285,9 +1303,9 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block
IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA);
child = *p;
if (child > 0) {
- uint32_t v = ctx->vregs[child];
+ int32_t v = ctx->vregs[child];
- if (v) {
+ if (v > 0) {
reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
use_pos = pos;
if (EXPECTED(reg == IR_REG_NONE)) {
@@ -1330,7 +1348,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
ir_ref ref;
ir_insn *insn;
ir_block *bb, *succ_bb;
- uint32_t *live_outs;
+ int32_t *live_outs;
uint32_t *live_in_block;
ir_list live_lists;
ir_live_interval *ival;
@@ -1395,9 +1413,9 @@ int ir_compute_live_ranges(ir_ctx *ctx)
if (insn->op == IR_PHI) {
ir_ref input = ir_insn_op(insn, k);
if (input > 0) {
- uint32_t v = ctx->vregs[input];
+ int32_t v = ctx->vregs[input];
- if (v) {
+ if (v > 0) {
ival = ctx->live_intervals[v];
ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use);
}
@@ -1418,7 +1436,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
uint32_t flags;
ir_ref *p;
ir_target_constraints constraints;
- uint32_t v;
+ int32_t v;
if (ctx->rules) {
int n;
@@ -1426,7 +1444,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
#if IR_X86_I64
if (ctx->rules[ref] & IR_TWO_REGS) {
v = ctx->vregs[ref];
- if (v) {
+ if (v > 0) {
IR_ASSERT(ctx->live_intervals[v]);
ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_TWO_REGS;
}
@@ -1438,7 +1456,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
&& ctx->use_lists[ref].count > 0) {
insn = &ctx->ir_base[ref];
if (insn->op == IR_VAR || insn->op == IR_ALLOCA) {
- insn->op3 = ctx->vars;
+ ctx->vregs[ref] = IR_STACK_SLOT_TO_VREG(ctx->vars);
ctx->vars = ref;
}
}
@@ -1466,7 +1484,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
insn = &ctx->ir_base[ref];
v = ctx->vregs[ref];
- if (v) {
+ if (v > 0) {
if (insn->op != IR_PHI) {
ir_live_pos def_pos;
ir_ref hint_ref = 0;
@@ -1485,7 +1503,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
ir_add_fixed_live_range(ctx, reg, IR_START_LIVE_POS_FROM_REF(bb->start), def_pos);
}
} else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1]) {
+ if (!IR_IS_CONST_REF(insn->op1) && ctx->vregs[insn->op1] > 0) {
hint_ref = insn->op1;
}
if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
@@ -1498,7 +1516,25 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else {
if (insn->op == IR_PARAM) {
/* We may reuse parameter stack slot for spilling */
- ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
+ ir_live_interval *param_ival = ctx->live_intervals[v];
+
+ param_ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
+ if (ctx->rules && ctx->use_lists[ref].count > 0) {
+ ir_ref use = ctx->use_edges[ctx->use_lists[ref].refs];
+ if (use < bb->end) {
+ ir_insn *use_insn = &ctx->ir_base[use];
+ if (use_insn->op == IR_VSTORE
+ && !(ctx->rules[use] & IR_SKIPPED)
+ /* first use of VAR is VSTORE(_, VAR, PARAM) */
+ && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) {
+ IR_ASSERT(use_insn->op3 == ref);
+ /* skip VSTORE (VAR is going to be remapped to PARAM) */
+ param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
+ param_ival->use_pos = param_ival->use_pos->next;
+ ctx->rules[use] = IR_SKIPPED | IR_NOP;
+ }
+ }
+ }
}
def_pos = IR_DEF_LIVE_POS_FROM_REF(ref);
}
@@ -1527,11 +1563,11 @@ int ir_compute_live_ranges(ir_ctx *ctx)
p = insn->ops + 2;
for (; j <= insn->inputs_count; j++, p++) {
ir_ref input = *p;
- uint32_t v;
+ int32_t v;
if (input > 0) {
v = ctx->vregs[input];
- IR_ASSERT(v);
+ IR_ASSERT(v > 0);
use_pos = IR_USE_LIVE_POS_FROM_REF(next);
if (!IS_LIVE_IN_BLOCK(v, b)) {
/* live.add(opd) */
@@ -1561,12 +1597,12 @@ int ir_compute_live_ranges(ir_ctx *ctx)
ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
ir_live_pos use_pos;
ir_ref hint_ref = 0;
- uint32_t v;
+ int32_t v;
uint32_t use_flags = IR_USE_FLAGS(def_flags, j);
if (input > 0) {
v = ctx->vregs[input];
- if (v) {
+ if (v > 0) {
use_pos = IR_USE_LIVE_POS_FROM_REF(ref);
if (reg != IR_REG_NONE) {
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
@@ -1595,7 +1631,7 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else {
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
}
- IR_ASSERT(ctx->vregs[ref]);
+ IR_ASSERT(ctx->vregs[ref] > 0);
hint_ref = ref;
} else if (input == insn->op1) {
/* Input is the same as "op1" */
@@ -1688,7 +1724,7 @@ static ir_live_pos ir_ivals_overlap(ir_live_range *lrg1, ir_live_range *lrg2)
}
}
-static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, uint32_t r1, uint32_t r2)
+static ir_live_pos ir_vregs_overlap(ir_ctx *ctx, int32_t r1, int32_t r2)
{
ir_live_interval *ival1 = ctx->live_intervals[r1];
ir_live_interval *ival2 = ctx->live_intervals[r2];
@@ -1716,7 +1752,7 @@ static bool ir_ivals_inside(ir_live_range *parent, ir_live_range *child)
return 1;
}
-static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child)
+static bool ir_vregs_inside(ir_ctx *ctx, int32_t parent, int32_t child)
{
ir_live_interval *child_ival = ctx->live_intervals[child];
ir_live_interval *parent_ival = ctx->live_intervals[parent];
@@ -1733,7 +1769,7 @@ static bool ir_vregs_inside(ir_ctx *ctx, uint32_t parent, uint32_t child)
return ir_ivals_inside(&parent_ival->range, &child_ival->range);
}
-static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2)
+static void ir_vregs_join(ir_ctx *ctx, int32_t r1, int32_t r2)
{
ir_live_interval *ival = ctx->live_intervals[r2];
ir_live_range *live_range = &ival->range;
@@ -1795,7 +1831,7 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2)
//ir_mem_free(ival);
}
-static void ir_vregs_coalesce(ir_ctx *ctx, uint32_t v1, uint32_t v2, ir_ref from, ir_ref to)
+static void ir_vregs_coalesce(ir_ctx *ctx, int32_t v1, int32_t v2, ir_ref from, ir_ref to)
{
ir_ref i;
uint16_t f1 = ctx->live_intervals[v1]->flags;
@@ -1914,7 +1950,7 @@ static void ir_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn)
p = p->next;
}
- if (insn->op2 > 0 && ctx->vregs[insn->op2]) {
+ if (insn->op2 > 0 && ctx->vregs[insn->op2] > 0) {
ival = ctx->live_intervals[ctx->vregs[insn->op2]];
r = &ival->range;
while (r) {
@@ -1977,13 +2013,13 @@ static int ir_hint_conflict(ir_ctx *ctx, ir_ref ref, int use, int def)
static int ir_try_swap_operands(ir_ctx *ctx, ir_ref i, ir_insn *insn)
{
- if (ctx->vregs[insn->op1]
+ if (ctx->vregs[insn->op1] > 0
&& ctx->vregs[insn->op1] != ctx->vregs[i]
&& !ir_vregs_overlap(ctx, ctx->vregs[insn->op1], ctx->vregs[i])
&& !ir_hint_conflict(ctx, i, ctx->vregs[insn->op1], ctx->vregs[i])) {
/* pass */
} else {
- if (ctx->vregs[insn->op2] && ctx->vregs[insn->op2] != ctx->vregs[i]) {
+ if (ctx->vregs[insn->op2] > 0 && ctx->vregs[insn->op2] != ctx->vregs[i]) {
ir_live_pos pos = IR_USE_LIVE_POS_FROM_REF(i);
ir_live_pos load_pos = IR_LOAD_LIVE_POS_FROM_REF(i);
ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op2]];
@@ -2081,7 +2117,7 @@ int ir_coalesce(ir_ctx *ctx)
insn = &ctx->ir_base[use];
if (insn->op == IR_PHI) {
input = ir_insn_op(insn, k);
- if (input > 0 && ctx->vregs[input]) {
+ if (input > 0 && ctx->vregs[input] > 0) {
uint32_t v1 = ctx->vregs[input];
uint32_t v2 = ctx->vregs[use];
@@ -2151,7 +2187,7 @@ int ir_coalesce(ir_ctx *ctx)
for (i = 1; i < ctx->insns_count; rule++, i++) {
if ((*rule) & (IR_MAY_SWAP|IR_MAY_REUSE)) {
insn = &ctx->ir_base[i];
- IR_ASSERT(ctx->vregs[i]);
+ IR_ASSERT(ctx->vregs[i] > 0);
if ((*rule) & IR_MAY_SWAP) {
IR_ASSERT(ir_op_flags[insn->op] & IR_OP_FLAG_COMMUTATIVE);
if (ctx->live_intervals[ctx->vregs[i]]->use_pos
@@ -2164,7 +2200,7 @@ int ir_coalesce(ir_ctx *ctx)
} else {
IR_ASSERT((*rule) & IR_MAY_REUSE);
if (insn->op1 > 0
- && ctx->vregs[insn->op1]
+ && ctx->vregs[insn->op1] > 0
&& ctx->vregs[i] != ctx->vregs[insn->op1]) {
if (ir_vregs_inside(ctx, ctx->vregs[insn->op1], ctx->vregs[i])) {
if (ctx->binding) {
@@ -2208,7 +2244,7 @@ int ir_coalesce(ir_ctx *ctx)
}
}
for (j = 1; j < ctx->insns_count; j++) {
- if (ctx->vregs[j]) {
+ if (ctx->vregs[j] > 0) {
ctx->vregs[j] = xlat[ctx->vregs[j]];
}
}
@@ -2243,6 +2279,12 @@ int ir_compute_dessa_moves(ir_ctx *ctx)
use = *p;
insn = &ctx->ir_base[use];
if (insn->op == IR_PHI) {
+ if (ctx->use_lists[use].count == 0) {
+ ctx->rules[use] = IR_SKIPPED | IR_NOP;
+ ctx->vregs[use] = 0;
+ continue;
+ }
+ IR_ASSERT(ctx->vregs[use] > 0);
for (j = 2; j <= k; j++) {
ir_ref input = ir_insn_op(insn, j);
@@ -2325,7 +2367,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat
insn = &ctx->ir_base[ref];
if (insn->op == IR_PHI) {
input = ir_insn_op(insn, k);
- if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) {
+ if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) {
have_constants_or_addresses = 1;
} else if (ctx->vregs[input] != ctx->vregs[ref]) {
s = ctx->vregs[input];
@@ -2394,7 +2436,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy, void *dat
insn = &ctx->ir_base[ref];
if (insn->op == IR_PHI) {
input = ir_insn_op(insn, k);
- if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) {
+ if (IR_IS_CONST_REF(input) || !(ctx->vregs[input] > 0)) {
emit_copy(ctx, insn->type, input, ref, data);
}
}
@@ -2712,7 +2754,7 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size)
}
IR_ASSERT(size == 1 || size == 2 || size == 4 || size == 8);
- n = ir_ntz(size);
+ n = IR_LOG2(size);
if (data->handled && data->handled[n]) {
ret = data->handled[n]->stack_spill_pos;
data->handled[n] = data->handled[n]->list_next;
@@ -2808,21 +2850,13 @@ static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size)
return ret;
}
-int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size)
+static int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, size_t size, size_t align)
{
int32_t ret;
- if (size <= 8) {
- if (size == 3) {
- size = 4;
- } else if (size > 4 && size < 8) {
- size = 8;
- }
- return ir_allocate_small_spill_slot(ctx, size);
- }
-
+ IR_ASSERT(size < 0xfffffff);
if (size <= 64 && (size & (size - 1)) == 0) {
- uint32_t n = ir_ntz(size);
+ uint32_t n = IR_LOG2(size);
ir_reg_alloc_data *data = ctx->data;
if (data->handled && data->handled[n]) {
@@ -2833,25 +2867,27 @@ int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size)
}
/* Align stack allocated data to 16 byte */
+ // TODO: support for alignment larger than 16 ???
ctx->flags2 |= IR_16B_FRAME_ALIGNMENT;
ret = IR_ALIGNED_SIZE(ctx->stack_frame_size, 16);
size = IR_ALIGNED_SIZE(size, 8);
- ctx->stack_frame_size = ret + size;
+ ctx->stack_frame_size = ret + (int32_t)size;
return ret;
}
-int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type)
+int32_t ir_allocate_spill_slot(ir_ctx *ctx, size_t size, size_t align)
{
- if (IR_IS_TYPE_SCALAR(type)) {
- return ir_allocate_small_spill_slot(ctx, ir_type_size[type]);
+ IR_ASSERT(align == 0 || (align & (align -1)) == 0); /* align must be a power of 2 */
+ if (size <= 8 && align < 16) {
+ if (align > size) {
+ size = align;
+ } else if ((size & (size - 1)) != 0) {
+ size = (size == 3) ? 4 : 8;
+ }
+ return ir_allocate_small_spill_slot(ctx, size);
} else {
- int32_t size;
-
- IR_ASSERT(IR_IS_TYPE_VECTOR(type));
- size = IR_VECTOR_SIZE(type);
- size = IR_MAX(size, 4);
- return ir_allocate_big_spill_slot(ctx, size);
+ return ir_allocate_big_spill_slot(ctx, size, align);
}
}
@@ -3917,12 +3953,12 @@ static void ir_assign_bound_spill_slots(ir_ctx *ctx)
{
ir_hashtab_bucket *b = ctx->binding->data;
uint32_t n = ctx->binding->count;
- uint32_t v;
+ int32_t v;
ir_live_interval *ival;
while (n > 0) {
v = ctx->vregs[b->key];
- if (v) {
+ if (v > 0) {
ival = ctx->live_intervals[v];
if (ival
&& ival->stack_spill_pos == -1
@@ -3984,25 +4020,35 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars)
ir_insn *insn = &ctx->ir_base[var];
IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA);
- vars = insn->op3; /* list next */
+ vars = IR_VREG_TO_STACK_SLOT(ctx->vregs[var]); /* list next */
if (insn->op == IR_VAR) {
- ir_ref slot = ir_allocate_spill_slot(ctx, insn->type);
+ ir_ref slot;
ir_use_list *use_list;
ir_ref n, *p;
+ size_t size = ir_get_type_size(insn->type);
+ size_t align = insn->op3;
- insn->op3 = slot;
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(insn->type) && size < 4) {
+ size = 4;
+ align = IR_MAX(align, 4);
+ }
+#endif
+ slot = ir_allocate_spill_slot(ctx, size, align);
+ ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(slot);
use_list = &ctx->use_lists[var];
n = use_list->count;
p = &ctx->use_edges[use_list->refs];
for (; n > 0; p++, n--) {
insn = &ctx->ir_base[*p];
if (insn->op == IR_VADDR) {
- insn->op3 = slot;
+ ctx->vregs[*p] = IR_STACK_SLOT_TO_VREG(slot);
}
}
} else {
ir_insn *val = &ctx->ir_base[insn->op2];
+ int32_t offset;
IR_ASSERT(IR_IS_CONST_REF(insn->op2));
IR_ASSERT(IR_IS_TYPE_INT(val->type));
@@ -4010,7 +4056,8 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars)
IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0);
IR_ASSERT(val->val.i64 < 0x7fffffff);
- insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32);
+ offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3);
+ ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset);
}
}
@@ -4228,7 +4275,13 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars)
other = prev ? prev->list_next : active;
}
- ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type);
+ size_t size = ir_get_type_size(ival->type);
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) {
+ size = 4;
+ }
+#endif
+ ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0);
if (unhandled && ival->end > unhandled->range.start) {
ival->list_next = active;
active = ival;
@@ -4256,10 +4309,10 @@ static int ir_linear_scan(ir_ctx *ctx, ir_ref vars)
#ifdef IR_TARGET_X86
if (ctx->flags2 & IR_HAS_FP_RET_SLOT) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, IR_DOUBLE);
+ ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0);
} else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE)
&& ((ir_reg_alloc_data*)(ctx->data))->cc->fp_ret_reg == IR_REG_NONE) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, ctx->ret_type);
+ ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0);
} else {
ctx->ret_slot = -1;
}
@@ -4320,19 +4373,6 @@ static bool needs_spill_load(ir_ctx *ctx, ir_live_interval *ival, ir_use_pos *us
return use_pos->next && use_pos->next->op_num != 0;
}
-static void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op, int8_t reg)
-{
- char key[10];
-
- if (!ctx->fused_regs) {
- ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab));
- ir_strtab_init(ctx->fused_regs, 8, 128);
- }
- memcpy(key, &root, sizeof(ir_ref));
- memcpy(key + 4, &ref_and_op, sizeof(ir_ref));
- ir_strtab_lookup(ctx->fused_regs, key, 8, 0x10000000 | (uint8_t)reg);
-}
-
static void assign_regs(ir_ctx *ctx)
{
ir_ref i;
@@ -4430,7 +4470,7 @@ static void assign_regs(ir_ctx *ctx)
|| ctx->ir_base[ref].op == IR_BITCAST
|| ctx->ir_base[ref].op == IR_TRUNC)
&& !IR_IS_CONST_REF(ctx->ir_base[ref].op1)
- && ctx->vregs[ctx->ir_base[ref].op1] == (uint32_t)i) {
+ && ctx->vregs[ctx->ir_base[ref].op1] == i) {
/* register reuse */
ir_set_alocated_reg(ctx, ref, use_pos->op_num, reg);
prev_use_ref = ref;
@@ -4446,13 +4486,6 @@ static void assign_regs(ir_ctx *ctx)
&& (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) {
/* Stack PARAM var is passed through memory */
reg = IR_REG_NONE;
-#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
- if (use_pos->next
- && ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->next->pos)].op == IR_VSTORE) {
- /* skip VSTORE (VAR is going to be remapped to PARAM on x86) */
- use_pos = use_pos->next;
- }
-#endif
} else {
uint32_t use_b = ctx->cfg_map[ref];
@@ -4509,7 +4542,7 @@ static void assign_regs(ir_ctx *ctx)
if (use_pos->hint_ref < 0) {
if (use_pos->flags & IR_PHI_USE) {
IR_ASSERT(use_pos->hint_ref < 0);
- IR_ASSERT(ctx->vregs[-use_pos->hint_ref]);
+ IR_ASSERT(ctx->vregs[-use_pos->hint_ref] > 0);
IR_ASSERT(ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]);
if (ctx->live_intervals[ctx->vregs[-use_pos->hint_ref]]->flags & IR_LIVE_INTERVAL_SPILLED) {
/* Spilled PHI var is passed through memory */
@@ -4520,7 +4553,7 @@ static void assign_regs(ir_ctx *ctx)
old_reg = ir_get_alocated_reg(ctx, -use_pos->hint_ref, use_pos->op_num);
if ((old_reg != IR_REG_NONE && reg != old_reg) || reg == IR_REG_NONE) {
ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG;
- ir_set_fused_reg(ctx, ref, -use_pos->hint_ref * sizeof(ir_ref) + use_pos->op_num, reg);
+ ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, reg);
use_pos = use_pos->next;
continue;
}
@@ -4546,7 +4579,7 @@ static void assign_regs(ir_ctx *ctx)
} else if (use_pos->hint_ref < 0 && !(use_pos->flags & IR_PHI_USE)) {
IR_ASSERT(ctx->rules[-use_pos->hint_ref] & IR_FUSED);
ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG;
- ir_set_fused_reg(ctx, ref, -use_pos->hint_ref * sizeof(ir_ref) + use_pos->op_num, IR_REG_NONE);
+ ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, IR_REG_NONE);
}
use_pos = use_pos->next;
}
@@ -4571,10 +4604,12 @@ static void assign_regs(ir_ctx *ctx)
if (ival->tmp_op_num <= insn->inputs_count) {
ir_ref *ops = insn->ops;
+
if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) {
/* constant rematerialization */
reg |= IR_REG_SPILL_LOAD;
- } else if (ctx->rules[ops[ival->tmp_op_num]] == (IR_SKIPPED|IR_FUSED|IR_SIMPLE|IR_ALLOCA)) {
+ } else if (ctx->rules[ops[ival->tmp_op_num]] == IR_STATIC_ALLOCA
+ && IR_OPND_KIND(ir_op_flags[insn->op], ival->tmp_op_num) == IR_OPND_DATA) {
/* local address rematerialization */
reg |= IR_REG_SPILL_LOAD;
}
diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c
new file mode 100644
index 00000000000..4ecdd79986b
--- /dev/null
+++ b/ext/opcache/jit/ir/ir_ra_simple.c
@@ -0,0 +1,662 @@
+/*
+ * IR - Lightweight JIT Compilation Framework
+ * Simple Local Register Allocation (used for -O0)
+ * This file is part of the IR Project distributed under the MIT-style LICENSE.
+ * Authors: Dmitry Stogov <dmitry@php.net>
+ *
+ * Simple RA assign registers for each IR instruction separately and always pass
+ * values between instrutios in stack spill slots.
+ */
+
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+
+#include "ir.h"
+
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+# include "ir_x86.h"
+#elif defined(IR_TARGET_AARCH64)
+# include "ir_aarch64.h"
+#else
+# error "Unknown IR target"
+#endif
+
+#include "ir_private.h"
+
+typedef struct {
+ ir_reg_alloc_data ra_data;
+ int32_t num;
+ ir_regset preserved_regs;
+ ir_regset clobbered[IR_SUB_REFS_COUNT];
+ struct {
+ uint8_t type;
+ int8_t start;
+ int8_t end;
+ int8_t hint;
+ int8_t flags;
+ ir_ref root;
+ ir_ref ref;
+ ir_ref op;
+ } regs[32];
+} ir_reg_alloc_simple_data;
+
+static void _add_scratch(ir_reg_alloc_simple_data *data, ir_reg reg, int8_t start, int8_t end)
+{
+ int8_t j;
+
+ if (start < 0) start = 0; // TODO: ARGVAL support ???
+ IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT);
+ if (reg >= IR_REG_NUM) {
+ for (j = start; j < end; j++) {
+ data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]);
+ }
+ } else {
+ for (j = start; j < end; j++) {
+ IR_REGSET_INCL(data->clobbered[j], reg);
+ }
+ }
+}
+
+static void _add_reg(ir_reg_alloc_simple_data *data, ir_type type,
+ int8_t start, int8_t end, ir_reg hint, int8_t flags,
+ ir_ref root, ir_ref ref, ir_ref op)
+{
+ IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32);
+ data->regs[data->num].type = type;
+ data->regs[data->num].start = start;
+ data->regs[data->num].end = end;
+ data->regs[data->num].hint = hint;
+ data->regs[data->num].flags = flags;
+ data->regs[data->num].root = root;
+ data->regs[data->num].ref = ref;
+ data->regs[data->num].op = op;
+ data->num++;
+}
+
+static ir_reg _get_free_reg(ir_type type, ir_regset available)
+{
+ if (IR_IS_TYPE_INT(type)) {
+ available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP);
+ } else {
+ IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
+ available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP);
+ }
+ if (!IR_REGSET_IS_EMPTY(available)) {
+ return IR_REGSET_FIRST(available);
+ } else {
+ return IR_REG_NONE;
+ }
+}
+
+static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *data, int j)
+{
+ int n;
+ ir_regset available;
+ ir_reg reg;
+
+ if (IR_IS_TYPE_INT(type)) {
+ available = IR_REGSET_GP;
+ if (ctx->flags & IR_USE_FRAME_POINTER) {
+ IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER);
+ }
+
+#if defined(IR_TARGET_X86)
+ if (ir_type_size[type] == 1) {
+ /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */
+ IR_REGSET_EXCL(available, IR_REG_RBP);
+ IR_REGSET_EXCL(available, IR_REG_RSI);
+ IR_REGSET_EXCL(available, IR_REG_RDI);
+ }
+#endif
+ } else {
+ IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
+ available = IR_REGSET_FP;
+ }
+ for (n = data->regs[j].start; n < data->regs[j].end; n++) {
+ available = IR_REGSET_DIFFERENCE(available, data->clobbered[n]);
+ }
+ if (IR_REGSET_IS_EMPTY(available)) {
+ fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n");
+ IR_ASSERT(0);
+ exit(-1);
+ }
+
+ reg = IR_REGSET_FIRST(available);
+ if (IR_REGSET_IN(data->preserved_regs, reg)) {
+ ir_regset tmp = (ir_regset)ctx->used_preserved_regs;
+ IR_REGSET_INCL(tmp, reg);
+ ctx->used_preserved_regs = tmp;
+ }
+ return reg;
+}
+
+static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load)
+{
+ ir_use_list *use_list = &ctx->use_lists[load];
+ ir_ref *p, use, i, n = use_list->count;
+ ir_ref last_use = IR_UNUSED;
+ ir_insn *insn;
+ int address_taken = -1; /* unknown */
+
+ if (n) {
+ /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */
+ for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
+ use = *p;
+ if (use < load || use > bb->end) return 0;
+ if (use > last_use) last_use = use;
+ }
+ /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */
+ for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) {
+ if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) {
+ return 0;
+ } else if (address_taken
+ && (insn->op == IR_STORE
+ || insn->op == IR_STORE_v
+ || insn->op == IR_CALL)) {
+ if (address_taken == -1) {
+ ir_use_list *use_list = &ctx->use_lists[var];
+ ir_ref *p, n = use_list->count;
+
+ for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
+ if (ctx->ir_base[*p].op == IR_VADDR) {
+ return 0;
+ }
+ }
+ address_taken = 0;
+ }
+ }
+ n = ir_insn_len(insn);
+ i += n;
+ insn += n;
+ }
+ }
+ return 1;
+}
+
+static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val)
+{
+ ir_ref i, n;
+ ir_insn *insn;
+ int address_taken = -1; /* unknown */
+
+ /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */
+ if (val < bb->start || val > store || store > bb->end) return 0;
+
+ /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */
+ for (i = val, insn = &ctx->ir_base[i]; i < store;) {
+ if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v)
+ && insn->op2 == var) {
+ return 0;
+ } else if (address_taken
+ && (insn->op == IR_LOAD
+ || insn->op == IR_LOAD_v
+ || insn->op == IR_STORE
+ || insn->op == IR_STORE_v
+ || insn->op == IR_CALL)) {
+ if (address_taken == -1) {
+ ir_use_list *use_list = &ctx->use_lists[var];
+ ir_ref *p, n = use_list->count;
+
+ for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
+ if (ctx->ir_base[*p].op == IR_VADDR) {
+ return 0;
+ }
+ }
+ address_taken = 0;
+ }
+ }
+ n = ir_insn_len(insn);
+ i += n;
+ insn += n;
+ }
+ return 1;
+}
+
+static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *data)
+{
+ ir_ref stack[4];
+ int stack_pos = 0;
+ ir_target_constraints constraints;
+ ir_insn *insn;
+ uint32_t j, n, flags, def_flags;
+ ir_ref *p, child;
+
+ while (1) {
+ IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED);
+
+ if (!(ctx->rules[input] & IR_SIMPLE)) {
+ def_flags = ir_get_target_constraints(ctx, input, &constraints);
+ n = constraints.tmps_count;
+ while (n > 0) {
+ n--;
+ if (constraints.tmp_regs[n].type) {
+ ir_reg flags = 0;
+ ir_ref op = constraints.tmp_regs[n].num;
+
+ if (op > 0 && op <= ctx->ir_base[input].inputs_count) {
+ ir_ref *ops = ctx->ir_base[input].ops;
+
+ if (IR_IS_CONST_REF(ops[op])) {
+ /* rematerialization */
+ flags = IR_REG_SPILL_LOAD;
+ } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
+ /* local address rematerialization */
+ flags = IR_REG_SPILL_LOAD;
+ }
+ }
+ _add_reg(data, constraints.tmp_regs[n].type,
+ constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
+ IR_UNUSED, input, op);
+ } else {
+ _add_scratch(data, constraints.tmp_regs[n].reg,
+ constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
+ }
+ }
+ } else {
+ def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG;
+ constraints.hints_count = 0;
+ }
+
+ insn = &ctx->ir_base[input];
+ flags = ir_op_flags[insn->op];
+ n = IR_INPUT_EDGES_COUNT(flags);
+ j = 1;
+ p = insn->ops + j;
+ if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) {
+ j++;
+ p++;
+ }
+ for (; j <= n; j++, p++) {
+ IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA);
+ child = *p;
+ if (child > 0) {
+ if (ctx->vregs[child] > 0) {
+ if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) {
+ ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
+ int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF;
+
+ _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
+ ref, input, j);
+ }
+ } else if (ctx->rules[child] & IR_FUSED) {
+ IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos)));
+ stack[stack_pos++] = child;
+ } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) {
+ ctx->regs[input][j] = ctx->ir_base[child].op2;
+ }
+ }
+ }
+ if (!stack_pos) {
+ break;
+ }
+ input = stack[--stack_pos];
+ }
+}
+
+static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn)
+{
+ size_t size = ir_get_type_size(var_insn->type);
+ size_t align = var_insn->op3;
+ int32_t offset;
+
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) {
+ size = 4;
+ align = IR_MAX(align, 4);
+ }
+#endif
+
+ offset = ir_allocate_spill_slot(ctx, size, align);
+ ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset);
+}
+
+int ir_reg_alloc_simple(ir_ctx *ctx)
+{
+ uint32_t b;
+ ir_block *bb;
+ ir_insn *insn;
+ ir_ref i, n, j, *p;
+ uint32_t *rule, insn_flags;
+ ir_target_constraints constraints;
+ uint32_t def_flags;
+ ir_reg reg;
+ ir_regset scratch;
+ ir_reg_alloc_simple_data data;
+ int32_t offset;
+
+ ir_compute_dessa_moves(ctx);
+
+ memset(&data.ra_data, 0, sizeof(data.ra_data));
+ data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags);
+ ctx->data = &data.ra_data;
+
+ ctx->stack_frame_size = 0;
+ ctx->call_stack_size = 0;
+ ctx->used_preserved_regs = 0;
+ ctx->used_preserved_regs = ctx->fixed_save_regset;
+
+ scratch = ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM];
+ data.preserved_regs = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, ctx->fixed_save_regset);
+
+ ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count);
+ memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count);
+
+ /* vregs + tmp + fixed + SRATCH + ALL */
+ ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*));
+
+ if (!ctx->arena) {
+ ctx->arena = ir_arena_create(16 * 1024);
+ }
+
+ for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) {
+ IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE));
+ for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
+ if (*rule & (IR_FUSED|IR_SKIPPED)) {
+ if ((*rule & IR_RULE_MASK) == IR_ALLOCA) {
+ if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) {
+ /* spill slot already allocated */
+ } else if (insn->op == IR_VAR) {
+ if (ctx->use_lists[i].count > 0) {
+ ir_allocate_var_spill_slot(ctx, i, insn);
+ }
+ } else if (insn->op == IR_ALLOCA) {
+ if (ctx->use_lists[i].count > 0) {
+ ir_insn *val = &ctx->ir_base[insn->op2];
+
+ IR_ASSERT(IR_IS_CONST_REF(insn->op2));
+ IR_ASSERT(IR_IS_TYPE_INT(val->type));
+ IR_ASSERT(!IR_IS_SYM_CONST(val->op));
+ IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0);
+ IR_ASSERT(val->val.i64 < 0x7fffffff);
+ offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3);
+ ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset);
+ }
+ } else if (insn->op == IR_VADDR) {
+ if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) {
+ ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]);
+ }
+ ctx->vregs[i] = ctx->vregs[insn->op1];
+ }
+ }
+ } else {
+ data.num = 0;
+ for (j = 0; j < IR_SUB_REFS_COUNT; j++) {
+ data.clobbered[j] = IR_REGSET_EMPTY;
+ }
+
+ def_flags = ir_get_target_constraints(ctx, i, &constraints);
+ n = constraints.tmps_count;
+ while (n) {
+ n--;
+
+ IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT);
+ if (constraints.tmp_regs[n].type) {
+ ir_reg flags = 0;
+ ir_ref op = constraints.tmp_regs[n].num;
+
+ if (op > 0 && op <= insn->inputs_count) {
+ ir_ref *ops = insn->ops;
+
+ if (IR_IS_CONST_REF(ops[op])) {
+ /* rematerialization */
+ flags = IR_REG_SPILL_LOAD;
+ } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
+ /* local address rematerialization */
+ flags = IR_REG_SPILL_LOAD;
+ }
+ }
+ _add_reg(&data, constraints.tmp_regs[n].type,
+ constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
+ IR_UNUSED, i, op);
+ } else {
+ _add_scratch(&data, constraints.tmp_regs[n].reg,
+ constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
+ }
+ }
+
+ if (ctx->vregs[i] > 0) {
+ reg = constraints.def_reg;
+ if (!ctx->live_intervals[ctx->vregs[i]]) {
+ do {
+ ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+ memset(ival, 0, sizeof(ir_live_interval));
+ ctx->live_intervals[ctx->vregs[i]] = ival;
+ ival->type = insn->type;
+ ival->reg = IR_REG_NONE;
+ ival->vreg = ctx->vregs[i];
+ ival->stack_spill_pos = -1;
+
+ if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) {
+ if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) {
+ ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]);
+ }
+ if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) {
+ ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]);
+ reg = IR_REG_NONE;
+ def_flags &= ~IR_USE_MUST_BE_IN_REG;
+ break;
+ }
+ } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) {
+ /* We may reuse parameter stack slot for spilling */
+ ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
+ if (ctx->use_lists[i].count > 0) {
+ ir_ref use = ctx->use_edges[ctx->use_lists[i].refs];
+ if (use < bb->end) {
+ ir_insn *use_insn = &ctx->ir_base[use];
+ if (use_insn->op == IR_VSTORE
+ && !(ctx->rules[use] & IR_SKIPPED)
+ /* first use of VAR is VSTORE(_, VAR, PARAM) */
+ && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) {
+ IR_ASSERT(use_insn->op3 == i);
+ /* skip VSTORE (VAR is going to be remapped to PARAM) */
+ ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
+ ival->use_pos = NULL;
+ ctx->rules[use] = IR_SKIPPED | IR_NOP;
+ }
+ }
+ }
+ break;
+ }
+
+ if (ctx->use_lists[i].count == 1) {
+ ir_ref use = ctx->use_edges[ctx->use_lists[i].refs];
+ ir_insn *use_insn = &ctx->ir_base[use];
+
+ if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v)
+ && use_insn->op3 == i
+ && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) {
+ if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) {
+ ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]);
+ }
+ ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]);
+ break;
+ }
+ }
+
+ size_t size = ir_get_type_size(ival->type);
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) {
+ size = 4;
+ }
+#endif
+ ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0);
+ } while (0);
+ }
+
+ if (def_flags & IR_USE_MUST_BE_IN_REG) {
+ ir_live_pos def_pos;
+
+ if (reg != IR_REG_NONE) {
+ def_pos = IR_SAVE_SUB_REF;
+ } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
+ if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
+ def_pos = IR_USE_SUB_REF;
+ } else {
+ def_pos = IR_LOAD_SUB_REF;
+ }
+ } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
+ def_pos = IR_LOAD_SUB_REF;
+ } else {
+ def_pos = IR_DEF_SUB_REF;
+ }
+
+ _add_reg(&data, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE,
+ IR_UNUSED, i, 0);
+ }
+ } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) {
+ if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) {
+ ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]);
+ }
+ }
+
+ n = insn->inputs_count;
+ insn_flags = ir_op_flags[insn->op];
+ j = 1;
+ p = insn->ops + 1;
+ if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) {
+ j++;
+ p++;
+ }
+ for (; j <= n; j++, p++) {
+ ir_ref input = *p;
+ ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
+ ir_live_pos use_pos;
+ uint32_t use_flags = IR_USE_FLAGS(def_flags, j);
+
+ if (reg != IR_REG_NONE) {
+#if IR_X86_I64
+ if (use_flags & IR_HINT_TWO_REGS) {
+ IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg));
+ IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg));
+ } else
+#endif
+ IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg);
+ }
+
+ if (input > 0) {
+ if (ctx->vregs[input] > 0) {
+ use_pos = IR_USE_SUB_REF;
+ if (reg != IR_REG_NONE) {
+ use_pos = IR_LOAD_SUB_REF;
+ } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
+ if (j == 1) {
+ if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
+ use_pos = IR_USE_SUB_REF;
+ } else {
+ use_pos = IR_LOAD_SUB_REF;
+ }
+ } else if (input == insn->op1) {
+ /* Input is the same as "op1" */
+ use_pos = IR_LOAD_SUB_REF;
+ }
+ }
+ if (use_flags & IR_USE_MUST_BE_IN_REG) {
+ _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
+ IR_UNUSED, i, j);
+ }
+ } else {
+ if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) {
+ ir_add_fusion_data(ctx, i, input, &data);
+ } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) {
+ ctx->regs[i][j] = ctx->ir_base[input].op2;
+ }
+ }
+ }
+ }
+
+ for (j = 0; j < data.num; j++) {
+ ir_regset available = scratch;
+#if IR_X86_I64
+ ir_reg reg2;
+#endif
+
+ for (n = data.regs[j].start; n < data.regs[j].end; n++) {
+ available = IR_REGSET_DIFFERENCE(available, data.clobbered[n]);
+ }
+ reg = data.regs[j].hint;
+#if IR_X86_I64
+ reg2 = IR_REG_NONE;
+ if (reg != IR_REG_NONE && (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64)) {
+ reg2 = IR_REG_I64_HI(reg);
+ reg = IR_REG_I64_LO(reg);
+ }
+#endif
+ if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) {
+ reg = _get_free_reg(data.regs[j].type, available);
+ if (UNEXPECTED(reg == IR_REG_NONE)) {
+ reg = _get_free_reg2(ctx, data.regs[j].type, &data, j);
+ }
+ }
+ for (n = data.regs[j].start; n < data.regs[j].end; n++) {
+ IR_REGSET_INCL(data.clobbered[n], reg);
+ }
+#if IR_X86_I64
+ if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) {
+ IR_REGSET_EXCL(available, reg);
+ if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) {
+ reg2 = _get_free_reg(data.regs[j].type, available);
+ if (UNEXPECTED(reg2 == IR_REG_NONE)) {
+ reg2 = _get_free_reg2(ctx, data.regs[j].type, &data, j);
+ }
+ }
+ for (n = data.regs[j].start; n < data.regs[j].end; n++) {
+ IR_REGSET_INCL(data.clobbered[n], reg2);
+ }
+ if (reg > reg2) {
+ SWAP_REGS(reg, reg2);
+ }
+ reg = IR_REG_I64_PAIR(reg, reg2);
+ }
+#endif
+ reg = reg | data.regs[j].flags;
+ if (data.regs[j].op == 4 && insn->inputs_count < 4) {
+ if (!ctx->tmp_regs) {
+ ctx->tmp_regs = ir_mem_malloc(ctx->insns_count);
+ memset(ctx->tmp_regs, -1, ctx->insns_count);
+ }
+ ctx->tmp_regs[data.regs[j].ref] = reg;
+ } else if (!data.regs[j].root || ctx->regs[data.regs[j].ref][data.regs[j].op] == IR_REG_NONE) {
+ ctx->regs[data.regs[j].ref][data.regs[j].op] = reg;
+ } else if (ctx->regs[data.regs[j].ref][data.regs[j].op] != reg) {
+ ctx->rules[data.regs[j].ref] |= IR_FUSED_REG;
+ ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg);
+ }
+ }
+ }
+
+ n = ir_insn_len(insn);
+ i += n;
+ insn += n;
+ rule += n;
+ }
+ if (bb->flags & IR_BB_DESSA_MOVES) {
+#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
+ ctx->regs[bb->end][0] = IR_REG_RAX;
+ ctx->regs[bb->end][1] = IR_REG_XMM0;
+#elif defined(IR_TARGET_AARCH64)
+ ctx->regs[bb->end][0] = IR_REG_X0;
+ ctx->regs[bb->end][1] = IR_REG_V0;
+#else
+# error "Unknown IR target"
+#endif
+ }
+ }
+
+#ifdef IR_TARGET_X86
+ if (ctx->flags2 & IR_HAS_FP_RET_SLOT) {
+ ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0);
+ } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE)
+ && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) {
+ ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0);
+ } else {
+ ctx->ret_slot = -1;
+ }
+#endif
+
+ ctx->flags |= IR_NO_STACK_COMBINE;
+ ir_fix_stack_frame(ctx);
+ ctx->data = NULL;
+
+ return 1;
+}
diff --git a/ext/opcache/jit/ir/ir_save.c b/ext/opcache/jit/ir/ir_save.c
index 1286f550385..f3b885d3bff 100644
--- a/ext/opcache/jit/ir/ir_save.c
+++ b/ext/opcache/jit/ir/ir_save.c
@@ -299,7 +299,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f)
ir_print_type_cname(insn->type, f);
fprintf(f, " d_%d", i);
if (save_flags & IR_SAVE_REGS) {
- if (ctx->vregs && ctx->vregs[i]) {
+ if (ctx->vregs && ctx->vregs[i] > 0) {
fprintf(f, " {R%d}", ctx->vregs[i]);
}
if (ctx->regs) {
@@ -317,7 +317,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f)
ir_print_type_cname(insn->type, f);
fprintf(f, " d_%d", i);
if (save_flags & IR_SAVE_REGS) {
- if (ctx->vregs && ctx->vregs[i]) {
+ if (ctx->vregs && ctx->vregs[i] > 0) {
fprintf(f, " {R%d}", ctx->vregs[i]);
}
if (ctx->regs) {
@@ -355,7 +355,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f)
fprintf(f, "%sd_%d", first ? "(" : ", ", ref);
}
if (save_flags & IR_SAVE_REGS) {
- if (ctx->vregs && ref > 0 && ctx->vregs[ref]) {
+ if (ctx->vregs && ref > 0 && ctx->vregs[ref] > 0) {
fprintf(f, " {R%d}", ctx->vregs[ref]);
}
if (ctx->regs) {
@@ -383,7 +383,7 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f)
fprintf(f, "%sfunc ", first ? "(" : ", ");
ir_print_proto(ctx, ref, f);
break;
- case IR_OPND_PROB:
+ case IR_OPND_OPT:
if (ref == 0) {
break;
}
diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c
index 4745c7000d7..27bb9163dcc 100644
--- a/ext/opcache/jit/ir/ir_sccp.c
+++ b/ext/opcache/jit/ir/ir_sccp.c
@@ -520,25 +520,13 @@ static bool ir_sccp_analyze_phi(const ir_ctx *ctx, ir_sccp_val *_values, ir_bitq
static bool ir_is_dead_load_ex(const ir_ctx *ctx, ir_ref ref, uint32_t flags, const ir_insn *insn)
{
- if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) {
- return ctx->use_lists[ref].count == 1;
- } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) {
- return ctx->use_lists[ref].count == 1;
- }
- return 0;
+ return (flags & IR_OP_FLAG_NO_SIDE_EFFECT) && ctx->use_lists[ref].count == 1;
}
static bool ir_is_dead_load(const ir_ctx *ctx, ir_ref ref)
{
if (ctx->use_lists[ref].count == 1) {
- const ir_insn *insn = &ctx->ir_base[ref];
- uint32_t flags = ir_op_flags[insn->op];
-
- if ((flags & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) {
- return 1;
- } else if (insn->op == IR_ALLOCA || insn->op == IR_BLOCK_BEGIN) {
- return 1;
- }
+ return (ir_op_flags[ctx->ir_base[ref].op] & IR_OP_FLAG_NO_SIDE_EFFECT) != 0;
}
return 0;
}
@@ -3040,26 +3028,62 @@ static bool ir_cmp_is_true(ir_op op, const ir_insn *op1, const ir_insn *op2)
} else if (op == IR_LT) {
if (IR_IS_TYPE_SIGNED(op1->type)) {
return op1->val.i64 < op2->val.i64;
+ } else if (ir_type_size[op1->type] == 8) {
+ return op1->val.i64 < op2->val.i64;
+ } else if (ir_type_size[op1->type] == 4) {
+ return op1->val.i32 < op2->val.i32;
+ } else if (ir_type_size[op1->type] == 2) {
+ return op1->val.i16 < op2->val.i16;
+ } else if (ir_type_size[op1->type] == 1) {
+ return op1->val.i8 < op2->val.i8;
} else {
- return op1->val.u64 < op2->val.u64;
+ IR_ASSERT(0);
+ return 0;
}
} else if (op == IR_GE) {
if (IR_IS_TYPE_SIGNED(op1->type)) {
return op1->val.i64 >= op2->val.i64;
+ } else if (ir_type_size[op1->type] == 8) {
+ return op1->val.i64 >= op2->val.i64;
+ } else if (ir_type_size[op1->type] == 4) {
+ return op1->val.i32 >= op2->val.i32;
+ } else if (ir_type_size[op1->type] == 2) {
+ return op1->val.i16 >= op2->val.i16;
+ } else if (ir_type_size[op1->type] == 1) {
+ return op1->val.i8 >= op2->val.i8;
} else {
- return op1->val.u64 >= op2->val.u64;
+ IR_ASSERT(0);
+ return 0;
}
} else if (op == IR_LE) {
if (IR_IS_TYPE_SIGNED(op1->type)) {
return op1->val.i64 <= op2->val.i64;
+ } else if (ir_type_size[op1->type] == 8) {
+ return op1->val.i64 <= op2->val.i64;
+ } else if (ir_type_size[op1->type] == 4) {
+ return op1->val.i32 <= op2->val.i32;
+ } else if (ir_type_size[op1->type] == 2) {
+ return op1->val.i16 <= op2->val.i16;
+ } else if (ir_type_size[op1->type] == 1) {
+ return op1->val.i8 <= op2->val.i8;
} else {
- return op1->val.u64 <= op2->val.u64;
+ IR_ASSERT(0);
+ return 0;
}
} else if (op == IR_GT) {
if (IR_IS_TYPE_SIGNED(op1->type)) {
return op1->val.i64 > op2->val.i64;
+ } else if (ir_type_size[op1->type] == 8) {
+ return op1->val.i64 > op2->val.i64;
+ } else if (ir_type_size[op1->type] == 4) {
+ return op1->val.i32 > op2->val.i32;
+ } else if (ir_type_size[op1->type] == 2) {
+ return op1->val.i16 > op2->val.i16;
+ } else if (ir_type_size[op1->type] == 1) {
+ return op1->val.i8 > op2->val.i8;
} else {
- return op1->val.u64 > op2->val.u64;
+ IR_ASSERT(0);
+ return 0;
}
} else if (op == IR_ULT) {
return op1->val.u64 < op2->val.u64;
@@ -3752,7 +3776,7 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con
&& ctx->use_lists[condition].count == 1
&& ctx->use_lists[condition_insn->op1].count == 1
&& (op1_insn->op == IR_ZEXT || op1_insn->op == IR_SEXT)
- && val_insn->val.u64 <= (((uint64_t)-1ULL) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) {
+ && val_insn->val.u64 <= (((uint64_t)(0-1ULL)) >> (64 - ir_type_size[ctx->ir_base[op1_insn->op1].type] * 8))) {
/* IF(AND(ZEXT(X), C)) => IF(AND(X, C)) */
if (op1_insn->op1 > 0) {
ir_use_list_replace_one(ctx, op1_insn->op1, condition_insn->op1, condition);
diff --git a/ext/opcache/jit/ir/ir_strtab.c b/ext/opcache/jit/ir/ir_strtab.c
index b93860d0ca1..306217a298f 100644
--- a/ext/opcache/jit/ir/ir_strtab.c
+++ b/ext/opcache/jit/ir/ir_strtab.c
@@ -180,6 +180,55 @@ ir_str ir_strtab_lookup(ir_strtab *strtab, const char *str, uint32_t len, ir_str
return val;
}
+void ir_strtab_set(ir_strtab *strtab, const char *str, uint32_t len, ir_str val)
+{
+ uint32_t h = ir_str_hash(str, len);
+ char *data = (char*)strtab->data;
+ uint32_t pos = ((uint32_t*)data)[(int32_t)(h | strtab->mask)];
+ ir_strtab_bucket *p;
+
+ while (pos != IR_INVALID_IDX) {
+ p = (ir_strtab_bucket*)(data + pos);
+ if (p->h == h
+ && p->len == len
+ && memcmp(p->str, str, len) == 0) {
+ p->val = val;
+ return;
+ }
+ pos = p->next;
+ }
+
+ IR_ASSERT(val != 0);
+
+ if (UNEXPECTED(strtab->count >= strtab->size)) {
+ ir_strtab_resize(strtab);
+ data = strtab->data;
+ }
+
+ if (strtab->buf) {
+ if (UNEXPECTED(strtab->buf_size - strtab->buf_top < len + 1)) {
+ ir_strtab_grow_buf(strtab, len + 1);
+ }
+
+ memcpy(strtab->buf + strtab->buf_top, str, len);
+ strtab->buf[strtab->buf_top + len] = 0;
+ str = (const char*)strtab->buf + strtab->buf_top;
+ strtab->buf_top += len + 1;
+ }
+
+ pos = strtab->pos;
+ strtab->pos += sizeof(ir_strtab_bucket);
+ strtab->count++;
+ p = (ir_strtab_bucket*)(data + pos);
+ p->h = h;
+ p->len = len;
+ p->str = str;
+ h |= strtab->mask;
+ p->next = ((uint32_t*)data)[(int32_t)h];
+ ((uint32_t*)data)[(int32_t)h] = pos;
+ p->val = val;
+}
+
ir_str ir_strtab_update(ir_strtab *strtab, const char *str, uint32_t len, ir_str val)
{
uint32_t h = ir_str_hash(str, len);
diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc
index 5ce20343bd0..312a72ff175 100644
--- a/ext/opcache/jit/ir/ir_x86.dasc
+++ b/ext/opcache/jit/ir/ir_x86.dasc
@@ -28,7 +28,7 @@ typedef uint64_t ir_mem;
# define IR_MEM_NONE 0
# define IR_MEM_VAL(loc) (loc)
-# define IR_MEM_ADD(mem, offset) ((mem) + (offset))
+# define IR_MEM_ADD(mem, offset) (((mem) & 0xffffffff00000000) | (((mem)+ (offset)) & 0xffffffff))
#endif
#define IR_MEM_OFFSET(loc) ((int32_t)(IR_MEM_VAL(loc) & 0xffffffff))
@@ -1711,7 +1711,8 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = {
_(CMP_I64) \
_(BINOP_I64) \
_(MUL_I64) \
- _(MUL_OV_I64) \
+ _(IMUL_OV_I64) \
+ _(UMUL_OV_I64) \
_(BINOP_HELPER_I64) \
_(OP_I64) \
_(SHIFT_I64) \
@@ -1743,8 +1744,6 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = {
#define IR_RULE_ENUM(name) IR_ ## name,
-#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA)
-
enum _ir_rule {
IR_FIRST_RULE = IR_LAST_OP,
IR_RULES(IR_RULE_ENUM)
@@ -2031,7 +2030,6 @@ op2_const:
n = 1;
}
break;
- case IR_LOAD_FP:
case IR_LOAD_INT:
case IR_MEM_OP_INT:
case IR_MEM_INC:
@@ -2049,6 +2047,19 @@ op2_const:
n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
}
break;
+ case IR_LOAD_FP:
+ flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG;
+ insn = &ctx->ir_base[ref];
+ if (IR_IS_CONST_REF(insn->op2)) {
+ n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
+ }
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(insn->type) && IR_VECTOR_SIZE(insn->type) < 4) {
+ constraints->tmp_regs[n] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ n++;
+ }
+#endif
+ break;
case IR_STORE_FP:
flags = IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG;
insn = &ctx->ir_base[ref];
@@ -2060,6 +2071,13 @@ op2_const:
constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n++;
}
+#if IR_SIMD
+ else if (IR_IS_TYPE_VECTOR(ctx->ir_base[insn->op3].type)
+ && IR_VECTOR_SIZE(ctx->ir_base[insn->op3].type) < 4) {
+ constraints->tmp_regs[n] = IR_TMP_REG(1, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ n++;
+ }
+#endif
break;
case IR_SWITCH:
flags = IR_OP2_MUST_BE_IN_REG;
@@ -2476,14 +2494,14 @@ get_arg_hints:
ctx->flags2 |= IR_HAS_FP_RET_SLOT;
break;
case IR_MUL_I64:
- case IR_MUL_OV_I64:
+ case IR_IMUL_OV_I64:
flags = IR_USE_SHOULD_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_HINT_TWO_REGS;
constraints->def_reg = IR_REG_I64_PAIR(IR_REG_RAX, IR_REG_RDX);
constraints->tmp_regs[0] = IR_SCRATCH_REG(IR_REG_RAX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
constraints->tmp_regs[1] = IR_SCRATCH_REG(IR_REG_RDX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
constraints->tmp_regs[2] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n = 3;
- if ((rule & IR_RULE_MASK) == IR_MUL_OV_I64) {
+ if ((rule & IR_RULE_MASK) == IR_IMUL_OV_I64) {
constraints->tmp_regs[3] = IR_TMP_REG(4, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n = 4;
}
@@ -2495,6 +2513,15 @@ get_arg_hints:
constraints->hints_count = 2;
}
break;
+ case IR_UMUL_OV_I64:
+ flags = IR_USE_SHOULD_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_HINT_TWO_REGS;
+ constraints->def_reg = IR_REG_I64_PAIR(IR_REG_RAX, IR_REG_RDX);
+ constraints->tmp_regs[0] = IR_SCRATCH_REG(IR_REG_RAX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ constraints->tmp_regs[1] = IR_SCRATCH_REG(IR_REG_RDX, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ constraints->tmp_regs[2] = IR_TMP_REG(3, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ constraints->tmp_regs[3] = IR_TMP_REG(4, IR_U32, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ n = 4;
+ break;
case IR_BIT_COUNT_I64:
flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG;
constraints->tmp_regs[0] = IR_TMP_REG(2, IR_U32, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
@@ -2723,6 +2750,11 @@ get_arg_hints:
constraints->tmp_regs[n] = IR_TMP_REG(2, ctx->ir_base[insn->op2].type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n++;
}
+#if IR_X86_I64
+ if (IR_VECTOR_BASE_TYPE(insn->type) == IR_I64 || IR_VECTOR_BASE_TYPE(insn->type) == IR_U64) {
+ constraints->tmp_regs[n] = IR_TMP_REG(4, IR_U32, IR_USE_SUB_REF, IR_DEF_SUB_REF);
+ } else
+#endif
constraints->tmp_regs[n] = IR_TMP_REG(4, IR_VECTOR_BASE_TYPE(insn->type), IR_USE_SUB_REF, IR_DEF_SUB_REF);
n++;
if (!IR_IS_CONST_REF(insn->op3)) {
@@ -2757,7 +2789,7 @@ get_arg_hints:
&& IR_VECTOR_SIZE(insn->type) == 32) {
constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
n++;
- if (ctx->vregs[insn->op1]) {
+ if (ctx->vregs[insn->op1] > 0) {
flags &= ~IR_DEF_CONFLICTS_WITH_INPUT_REGS;
flags |= IR_DEF_REUSES_OP1_REG;
constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
@@ -2907,7 +2939,7 @@ get_arg_hints:
if (!(ctx->mflags & IR_X86_AVX2) && IR_VECTOR_SIZE(insn->type) == 32) {
constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
n++;
- if (ctx->vregs[insn->op2] || insn->op == IR_NE) {
+ if (ctx->vregs[insn->op2] > 0 || insn->op == IR_NE) {
constraints->tmp_regs[n] = IR_TMP_REG(4, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
n++;
}
@@ -3288,6 +3320,11 @@ static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root)
if (ir_in_same_block(ctx, ref) &&
(ctx->ir_base[ref].op == IR_LOAD || ctx->ir_base[ref].op == IR_LOAD_v ||
ctx->ir_base[ref].op == IR_VLOAD || ctx->ir_base[ref].op == IR_VLOAD_v)) {
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) {
+ return;
+ }
+#endif
if (ctx->use_lists[ref].count == 2
&& !ir_match_has_mem_deps(ctx, ref, root)) {
ir_ref addr_ref = ctx->ir_base[ref].op2;
@@ -3314,6 +3351,11 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root)
if (ir_in_same_block(ctx, ref)
&& (insn->op == IR_LOAD || insn->op == IR_LOAD_v || insn->op == IR_VLOAD || insn->op == IR_VLOAD_v)) {
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(ctx->ir_base[ref].type) && IR_VECTOR_SIZE(ctx->ir_base[ref].type) < 2) {
+ return 0;
+ }
+#endif
if (ctx->use_lists[ref].count == 2
&& !ir_match_has_mem_deps(ctx, ref, root)) {
ir_ref addr_ref = ctx->ir_base[ref].op2;
@@ -4073,9 +4115,12 @@ binop_fp:
case IR_MUL_OV:
IR_ASSERT(IR_IS_TYPE_INT(insn->type));
#if IR_X86_I64
- if (insn->type == IR_I64 || insn->type == IR_U64) {
+ if (insn->type == IR_I64) {
+ ir_match_fuse_load(ctx, insn->op2, ref);
+ return IR_TWO_REGS | IR_IMUL_OV_I64;
+ } else if (insn->type == IR_U64) {
ir_match_fuse_load(ctx, insn->op2, ref);
- return IR_TWO_REGS | IR_MUL_OV_I64;
+ return IR_TWO_REGS | IR_UMUL_OV_I64;
}
#endif
if (IR_IS_TYPE_SIGNED(insn->type) && ir_type_size[insn->type] != 1) {
@@ -4334,6 +4379,13 @@ binop_fp:
element_type == IR_I8 || element_type == IR_U8) {
return IR_VECTOR_BINOP_EXPAND;
}
+ if (IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
+ if (ctx->mflags & IR_X86_AVX) {
+ return IR_VECTOR_BINOP_AVX;
+ } else {
+ return IR_VECTOR_BINOP_SSE2;
+ }
+ }
goto binop_vector;
}
#endif
@@ -4393,6 +4445,13 @@ binop_fp:
(element_type == IR_I64 && insn->op == IR_SAR)) {
return IR_VECTOR_BINOP_EXPAND;
}
+ if (IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
+ if (ctx->mflags & IR_X86_AVX) {
+ return IR_VECTOR_BINOP_AVX;
+ } else {
+ return IR_VECTOR_BINOP_SSE2;
+ }
+ }
goto binop_vector;
}
#endif
@@ -5373,7 +5432,7 @@ static int32_t ir_ref_spill_slot_offset(ir_ctx *ctx, ir_ref ref, ir_reg *reg)
{
int32_t offset;
- IR_ASSERT(ref >= 0 && ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]);
+ IR_ASSERT(ref >= 0 && ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]);
offset = ctx->live_intervals[ctx->vregs[ref]]->stack_spill_pos;
IR_ASSERT(offset != -1);
if (ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL) {
@@ -5414,21 +5473,59 @@ static bool ir_is_same_spill_slot(ir_ctx *ctx, ir_ref ref, ir_mem mem)
return IR_MEM_VAL(m) == IR_MEM_VAL(mem);
}
+static int32_t ir_local_slot(ir_ctx *ctx, ir_ref ref)
+{
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR ||
+ ctx->ir_base[ref].op == IR_ALLOCA ||
+ ctx->ir_base[ref].op == IR_VADDR ||
+ ctx->ir_base[ref].op == IR_PARAM);
+ IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref]));
+ return IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]);
+}
+
+static int32_t ir_local_offset(ir_ctx *ctx, ir_ref ref)
+{
+ int32_t offset;
+
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR ||
+ ctx->ir_base[ref].op == IR_ALLOCA ||
+ ctx->ir_base[ref].op == IR_VADDR ||
+ ctx->ir_base[ref].op == IR_PARAM);
+ IR_ASSERT(IR_VREG_IS_STACK_SLOT(ctx->vregs[ref]));
+ offset = IR_VREG_TO_STACK_SLOT(ctx->vregs[ref]);
+ return IR_SPILL_POS_TO_OFFSET(offset);
+}
+
static ir_mem ir_var_spill_slot(ir_ctx *ctx, ir_ref ref)
{
- ir_insn *var_insn = &ctx->ir_base[ref];
ir_reg reg;
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[ref].op == IR_VAR);
reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- return IR_MEM_BO(reg, IR_SPILL_POS_TO_OFFSET(var_insn->op3));
+ return IR_MEM_BO(reg, ir_local_offset(ctx, ref));
+}
+
+static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
+ int32_t offset;
+
+ IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA);
+ offset = ir_local_offset(ctx, src);
+ if (offset == 0) {
+ | mov Ra(reg), Ra(base)
+ } else {
+ | lea Ra(reg), [Ra(base)+offset]
+ }
}
static bool ir_may_avoid_spill_load(ir_ctx *ctx, ir_ref ref, ir_ref use)
{
ir_live_interval *ival;
- IR_ASSERT(ctx->vregs[ref] && ctx->live_intervals[ctx->vregs[ref]]);
+ IR_ASSERT(ctx->vregs[ref] > 0 && ctx->live_intervals[ctx->vregs[ref]]);
ival = ctx->live_intervals[ctx->vregs[ref]];
while (ival) {
ir_use_pos *use_pos = ival->use_pos;
@@ -5633,6 +5730,49 @@ static void ir_emit_load_imm_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref sr
}
}
+#if IR_SIMD
+/* Vectos with size less than 4 have to be loaded throug general purpose register */
+static void ir_emit_load_short_vector(ir_ctx *ctx, uint32_t width, ir_reg reg, ir_mem mem, ir_reg tmp_reg)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ bool do_pop = 0;
+
+ IR_ASSERT(width == 1 || width == 2);
+ if (tmp_reg == IR_REG_NONE) {
+ IR_ASSERT(0 && "this is not expected");
+ if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) {
+ tmp_reg = IR_REG_RAX;
+ } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) {
+ tmp_reg = IR_REG_RCX;
+ } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) {
+ tmp_reg = IR_REG_RDX;
+ }
+ if (IR_MEM_BASE(mem) == IR_REG_RSP) {
+ mem = IR_MEM_ADD(mem, sizeof(void*));
+ }
+
+ | push Ra(tmp_reg)
+ do_pop = 1;
+ }
+
+ if (width == 1) {
+ | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem
+ } else {
+ | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem
+ }
+ if (ctx->mflags & IR_X86_AVX) {
+ | vmovd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg)
+ } else {
+ | movd xmm(reg-IR_REG_FP_FIRST), Rd(tmp_reg)
+ }
+
+ if (do_pop) {
+ | pop Ra(tmp_reg)
+ }
+}
+#endif
+
static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem)
{
ir_backend_data *data = ctx->data;
@@ -5650,13 +5790,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me
| ASM_TXT_TMEM_OP vmovdqa, xmm(reg-IR_REG_FP_FIRST), oword, mem
} else if (width == 8) {
| ASM_TXT_TMEM_OP vmovq, xmm(reg-IR_REG_FP_FIRST), qword, mem
- } else if (width == 4) {
+ } else if (width == 4 ||
+ (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) ||
+ ((ctx->flags & IR_USE_FRAME_POINTER) &&
+ IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) {
| ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem
} else {
- // TODO: 4-byte read may be unsafe ???
- // IR_ASSERT(width >= 4 && width <= 16);
- IR_ASSERT(width <= 4);
- | ASM_TXT_TMEM_OP vmovd, xmm(reg-IR_REG_FP_FIRST), dword, mem
+ ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE);
}
} else if (element_type == IR_DOUBLE) {
if (width == 16) {
@@ -5699,13 +5839,13 @@ static void ir_emit_load_mem_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem me
|.else
| ASM_TXT_TMEM_OP movlpd, xmm(reg-IR_REG_FP_FIRST), qword, mem
|.endif
- } else if (width == 4) {
+ } else if (width == 4 ||
+ (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) ||
+ ((ctx->flags & IR_USE_FRAME_POINTER) &&
+ IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) {
| ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem
} else {
- // TODO: 4-byte read may be unsafe ???
- // IR_ASSERT(width >= 4 && width <= 16);
- IR_ASSERT(width <= 4);
- | ASM_TXT_TMEM_OP movd, xmm(reg-IR_REG_FP_FIRST), dword, mem
+ ir_emit_load_short_vector(ctx, width, reg, mem, IR_REG_NONE);
}
} else if (element_type == IR_DOUBLE) {
if (width == 16) {
@@ -5741,38 +5881,6 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem)
}
}
-static int32_t ir_local_offset(ir_ctx *ctx, ir_insn *insn)
-{
- if (insn->op != IR_PARAM) {
- IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA || insn->op == IR_VADDR);
- return IR_SPILL_POS_TO_OFFSET(insn->op3);
- } else {
- IR_ASSERT(ctx->value_params && ctx->value_params[insn->op3 - 1].align);
- return IR_SPILL_POS_TO_OFFSET(ctx->value_params[insn->op3 - 1].offset);
- }
-}
-
-static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src)
-{
- ir_backend_data *data = ctx->data;
- dasm_State **Dst = &data->dasm_state;
- ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- ir_insn *var_insn;
- int32_t offset;
-
- IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA);
- var_insn = &ctx->ir_base[src];
- if (var_insn->op == IR_VADDR) {
- var_insn = &ctx->ir_base[var_insn->op1];
- }
- offset = ir_local_offset(ctx, var_insn);
- if (offset == 0) {
- | mov Ra(reg), Ra(base)
- } else {
- | lea Ra(reg), [Ra(base)+offset]
- }
-}
-
static void ir_resolve_label_syms(ir_ctx *ctx)
{
uint32_t b;
@@ -5828,7 +5936,7 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src)
} else {
ir_emit_load_imm_fp(ctx, type, reg, src);
}
- } else if (ctx->vregs[src]) {
+ } else if (ctx->vregs[src] > 0) {
ir_emit_load_mem(ctx, type, reg, ir_ref_spill_slot(ctx, src));
} else {
ir_load_local_addr(ctx, reg, src);
@@ -5843,6 +5951,49 @@ static void ir_emit_store_mem_int(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg
| ASM_MEM_REG_OP mov, type, mem, reg
}
+#if IR_SIMD
+/* Vectos with size less than 4 have to be stored throug general purpose register */
+static void ir_emit_store_short_vector(ir_ctx *ctx, uint32_t width, ir_mem mem, ir_reg reg, ir_reg tmp_reg)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ bool do_pop = 0;
+
+ IR_ASSERT(width == 1 || width == 2);
+ if (tmp_reg == IR_REG_NONE) {
+ IR_ASSERT(0 && "this is not expected");
+ if (IR_MEM_BASE(mem) != IR_REG_RAX && IR_MEM_INDEX(mem) != IR_REG_RAX) {
+ tmp_reg = IR_REG_RAX;
+ } else if (IR_MEM_BASE(mem) != IR_REG_RCX && IR_MEM_INDEX(mem) != IR_REG_RCX) {
+ tmp_reg = IR_REG_RCX;
+ } else if (IR_MEM_BASE(mem) != IR_REG_RDX && IR_MEM_INDEX(mem) != IR_REG_RDX) {
+ tmp_reg = IR_REG_RDX;
+ }
+ if (IR_MEM_BASE(mem) == IR_REG_RSP) {
+ mem = IR_MEM_ADD(mem, sizeof(void*));
+ }
+
+ | push Ra(tmp_reg)
+ do_pop = 1;
+ }
+
+ if (ctx->mflags & IR_X86_AVX) {
+ | vmovd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST)
+ } else {
+ | movd Rd(tmp_reg), xmm(reg-IR_REG_FP_FIRST)
+ }
+ if (width == 1) {
+ | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg)
+ } else {
+ | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg)
+ }
+
+ if (do_pop) {
+ | pop Ra(tmp_reg)
+ }
+}
+#endif
+
static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg reg)
{
ir_backend_data *data = ctx->data;
@@ -5860,11 +6011,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r
| ASM_TMEM_TXT_OP vmovdqa, oword, mem, xmm(reg-IR_REG_FP_FIRST)
} else if (width == 8) {
| ASM_TMEM_TXT_OP vmovq, qword, mem, xmm(reg-IR_REG_FP_FIRST)
- } else {
- // TODO: 4-byte write may be unsafe ???
- // IR_ASSERT(width >= 4 && width <= 16);
- IR_ASSERT(width <= 4);
+ } else if (width == 4 ||
+ (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) ||
+ ((ctx->flags & IR_USE_FRAME_POINTER) &&
+ IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) {
| ASM_TMEM_TXT_OP vmovd, dword, mem, xmm(reg-IR_REG_FP_FIRST)
+ } else {
+ ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE);
}
} else if (element_type == IR_DOUBLE) {
if (width == 16) {
@@ -5907,11 +6060,13 @@ static void ir_emit_store_mem_fp(ir_ctx *ctx, ir_type type, ir_mem mem, ir_reg r
|.else
| ASM_TMEM_TXT_OP movlpd, qword, mem, xmm(reg-IR_REG_FP_FIRST)
|.endif
- } else {
- // TODO: 4-byte write may be unsafe ???
- // IR_ASSERT(width >= 4 && width <= 16);
- IR_ASSERT(width <= 4);
+ } else if (width == 4 ||
+ (IR_MEM_BASE(mem) == IR_REG_RSP && IR_MEM_INDEX(mem) == IR_REG_NONE) ||
+ ((ctx->flags & IR_USE_FRAME_POINTER) &&
+ IR_MEM_BASE(mem) == IR_REG_RBP && IR_MEM_INDEX(mem) == IR_REG_NONE)) {
| ASM_TMEM_TXT_OP movd, dword, mem, xmm(reg-IR_REG_FP_FIRST)
+ } else {
+ ir_emit_store_short_vector(ctx, width, mem, reg, IR_REG_NONE);
}
} else if (element_type == IR_DOUBLE) {
if (width == 16) {
@@ -5955,7 +6110,7 @@ static void ir_emit_load_i64_lo(ir_ctx *ctx, ir_reg reg, ir_ref src)
IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op));
ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32);
- } else if (ctx->vregs[src]) {
+ } else if (ctx->vregs[src] > 0) {
ir_emit_load_mem_int(ctx, IR_U32, reg, ir_ref_spill_slot(ctx, src));
} else {
IR_ASSERT(0);
@@ -5969,7 +6124,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src)
IR_ASSERT(IR_IS_TYPE_INT(insn->type) && !IR_IS_SYM_CONST(insn->op));
ir_emit_load_imm_int(ctx, IR_U32, reg, insn->val.u32_hi);
- } else if (ctx->vregs[src]) {
+ } else if (ctx->vregs[src] > 0) {
ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, src));
ir_emit_load_mem_int(ctx, IR_U32, reg, mem);
} else {
@@ -5979,7 +6134,7 @@ static void ir_emit_load_i64_hi(ir_ctx *ctx, ir_reg reg, ir_ref src)
static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg)
{
- if (ctx->vregs[dst]) {
+ if (ctx->vregs[dst] > 0) {
ir_emit_store_mem_int(ctx, IR_U32, ir_ref_spill_slot(ctx, dst), reg);
} else {
IR_ASSERT(0);
@@ -5988,7 +6143,7 @@ static void ir_emit_store_i64_lo(ir_ctx *ctx, ir_ref dst, ir_reg reg)
static void ir_emit_store_i64_hi(ir_ctx *ctx, ir_ref dst, ir_reg reg)
{
- if (ctx->vregs[dst]) {
+ if (ctx->vregs[dst] > 0) {
ir_mem mem = IR_MEM_I64_HI(ir_ref_spill_slot(ctx, dst));
ir_emit_store_mem_int(ctx, IR_U32, mem, reg);
} else {
@@ -6207,6 +6362,8 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
ir_ref base_reg_ref, index_reg_ref;
ir_reg base_reg = IR_REG_NONE, index_reg;
int32_t offset = 0, scale;
+ intptr_t long_offset;
+ bool base_loaded = 0;
IR_ASSERT(((rule & IR_RULE_MASK) >= IR_LEA_FIRST &&
(rule & IR_RULE_MASK) <= IR_LEA_LAST) ||
@@ -6217,7 +6374,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
case IR_LEA_OB:
offset_insn = insn;
if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]);
+ offset = ir_local_offset(ctx, insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6239,12 +6396,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
break;
case IR_LEA_IB:
if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]);
+ offset = ir_local_offset(ctx, insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = ref * sizeof(ir_ref) + 2;
} else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = ref * sizeof(ir_ref) + 1;
@@ -6260,12 +6417,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
offset_insn = op1_insn;
scale = 1;
if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
} else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]);
+ offset = ir_local_offset(ctx, op1_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = ref * sizeof(ir_ref) + 2;
@@ -6279,12 +6436,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
offset_insn = op2_insn;
scale = 1;
if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]);
+ offset = ir_local_offset(ctx, insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
} else if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]);
+ offset = ir_local_offset(ctx, op2_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = ref * sizeof(ir_ref) + 1;
@@ -6311,12 +6468,12 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
offset_insn = insn;
scale = 1;
if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]);
+ offset = ir_local_offset(ctx, op1_insn->op2);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
} else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]);
+ offset = ir_local_offset(ctx, op1_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
index_reg_ref = insn->op1 * sizeof(ir_ref) + 2;
@@ -6332,7 +6489,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
op2_insn = &ctx->ir_base[insn->op2];
scale = ctx->ir_base[op2_insn->op2].val.i32;
if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]);
+ offset = ir_local_offset(ctx, op1_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6346,7 +6503,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
op2_insn = &ctx->ir_base[insn->op2];
offset_insn = op2_insn;
if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op2_insn->op1]);
+ offset = ir_local_offset(ctx, op2_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6355,7 +6512,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
break;
case IR_LEA_B_SI:
if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op1]);
+ offset = ir_local_offset(ctx, insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6369,7 +6526,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
case IR_LEA_SI_B:
index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6383,7 +6540,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
offset_insn = insn;
op1_insn = &ctx->ir_base[insn->op1];
if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op1]);
+ offset = ir_local_offset(ctx, op1_insn->op1);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6398,7 +6555,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
op1_insn = &ctx->ir_base[insn->op1];
index_reg_ref = op1_insn->op1 * sizeof(ir_ref) + 1;
if (ir_rule(ctx, op1_insn->op2) == IR_STATIC_ALLOCA) {
- offset = ir_local_offset(ctx, &ctx->ir_base[op1_insn->op2]);
+ offset = ir_local_offset(ctx, op1_insn->op2);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = IR_UNUSED;
} else {
@@ -6410,7 +6567,9 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
case IR_LEA_SYM_O:
op1_insn = &ctx->ir_base[insn->op1];
op2_insn = &ctx->ir_base[insn->op2];
- offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64;
+ long_offset = (intptr_t)ir_sym_val(ctx, op1_insn) + (intptr_t)op2_insn->val.i64;
+ IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset));
+ offset = (int32_t)long_offset;
base_reg_ref = index_reg_ref = IR_UNUSED;
scale = 1;
offset_insn = NULL;
@@ -6418,13 +6577,15 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
case IR_LEA_O_SYM:
op1_insn = &ctx->ir_base[insn->op1];
op2_insn = &ctx->ir_base[insn->op2];
- offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64;
+ long_offset = (intptr_t)ir_sym_val(ctx, op2_insn) + (intptr_t)op1_insn->val.i64;
+ IR_ASSERT(IR_IS_SIGNED_32BIT(long_offset));
+ offset = (int32_t)long_offset;
base_reg_ref = index_reg_ref = IR_UNUSED;
scale = 1;
offset_insn = NULL;
break;
case IR_ALLOCA:
- offset = ir_local_offset(ctx, insn);
+ offset = ir_local_offset(ctx, ref);
base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
base_reg_ref = index_reg_ref = IR_UNUSED;
scale = 1;
@@ -6450,7 +6611,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
if (base_reg_ref) {
if (UNEXPECTED(ctx->rules[base_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) {
- base_reg = ir_get_fused_reg(ctx, root, base_reg_ref);
+ base_reg = ir_get_fused_reg_ex(ctx, root, base_reg_ref);
} else {
base_reg = ((int8_t*)ctx->regs)[base_reg_ref];
}
@@ -6458,24 +6619,21 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref)
if (IR_REG_SPILLED(base_reg)) {
base_reg = IR_REG_NUM(base_reg);
ir_emit_load(ctx, insn->type, base_reg, ((ir_ref*)ctx->ir_base)[base_reg_ref]);
+ base_loaded = 1;
}
}
index_reg = IR_REG_NONE;
if (index_reg_ref) {
- if (base_reg_ref
- && ((ir_ref*)ctx->ir_base)[index_reg_ref]
- == ((ir_ref*)ctx->ir_base)[base_reg_ref]) {
- index_reg = base_reg;
+ if (UNEXPECTED(ctx->rules[index_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) {
+ index_reg = ir_get_fused_reg_ex(ctx, root, index_reg_ref);
} else {
- if (UNEXPECTED(ctx->rules[index_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) {
- index_reg = ir_get_fused_reg(ctx, root, index_reg_ref);
- } else {
- index_reg = ((int8_t*)ctx->regs)[index_reg_ref];
- }
- IR_ASSERT(index_reg != IR_REG_NONE);
- if (IR_REG_SPILLED(index_reg)) {
- index_reg = IR_REG_NUM(index_reg);
+ index_reg = ((int8_t*)ctx->regs)[index_reg_ref];
+ }
+ IR_ASSERT(index_reg != IR_REG_NONE);
+ if (IR_REG_SPILLED(index_reg)) {
+ index_reg = IR_REG_NUM(index_reg);
+ if (index_reg != base_reg || !base_loaded) {
ir_emit_load(ctx, insn->type, index_reg, ((ir_ref*)ctx->ir_base)[index_reg_ref]);
}
}
@@ -6507,7 +6665,7 @@ static ir_mem ir_fuse_load(ir_ctx *ctx, ir_ref root, ir_ref ref)
IR_ASSERT(load_insn->op == IR_LOAD || load_insn->op == IR_LOAD_v ||
load_insn->op == IR_VLOAD || load_insn->op == IR_VLOAD_v);
if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) {
- reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2);
+ reg = ir_get_fused_reg(ctx, root, ref, 2);
} else {
reg = ctx->regs[ref][2];
}
@@ -7123,9 +7281,14 @@ static void ir_emit_mem_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_insn *op_insn = &ctx->ir_base[insn->op3];
ir_type type = op_insn->type;
ir_ref op2 = op_insn->op2;
- ir_reg op2_reg = ctx->regs[insn->op3][2];
+ ir_reg op2_reg;
ir_mem mem;
+ if (UNEXPECTED(ctx->rules[insn->op3] & IR_FUSED_REG)) {
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op3, 2);
+ } else {
+ op2_reg = ctx->regs[insn->op3][2];
+ }
if (insn->op == IR_STORE) {
mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
} else {
@@ -7738,6 +7901,9 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule
switch (ir_type_size[insn->type]) {
default:
IR_ASSERT(0);
+ case 2:
+ | rol Rw(def_reg), 8
+ break;
case 4:
| bswap Rd(def_reg)
break;
@@ -8678,8 +8844,8 @@ static void ir_emit_cmp_int_common2(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ins
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, root, ref, 1);
+ op2_reg = ir_get_fused_reg(ctx, root, ref, 2);
} else {
op1_reg = ctx->regs[ref][1];
op2_reg = ctx->regs[ref][2];
@@ -8852,8 +9018,8 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_op
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, root, ref, 1);
+ op2_reg = ir_get_fused_reg(ctx, root, ref, 2);
} else {
op1_reg = ctx->regs[ref][1];
op2_reg = ctx->regs[ref][2];
@@ -8972,7 +9138,7 @@ static void ir_emit_test_bit_common(ir_ctx *ctx, ir_ref root, ir_ref ref)
bit = IR_LOG2(ctx->ir_base[binop_insn->op2].val.u64);
if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1);
+ op1_reg = ir_get_fused_reg(ctx, root, ref, 1);
} else {
op1_reg = ctx->regs[ref][1];
}
@@ -9032,8 +9198,8 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_
op1 = cmp_insn->op1;
op2 = cmp_insn->op2;
if (UNEXPECTED(ctx->rules[cmp_ref] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, root, cmp_ref * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, root, cmp_ref, 1);
+ op2_reg = ir_get_fused_reg(ctx, root, cmp_ref, 2);
} else {
op1_reg = ctx->regs[cmp_ref][1];
op2_reg = ctx->regs[cmp_ref][2];
@@ -9313,8 +9479,8 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -9508,6 +9674,30 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_emit_load(ctx, op1_type, op1_reg, op1);
}
+ if (IR_IS_CONST_REF(op1)) {
+ if (!ir_const_is_true(&ctx->ir_base[insn->op1])) {
+ op2 = op3;
+ op2_reg = op3_reg;
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ if (def_reg != op2_reg) {
+ if (IR_IS_TYPE_INT(type)) {
+ ir_emit_mov(ctx, type, def_reg, op2_reg);
+ } else {
+ ir_emit_fp_mov(ctx, type, def_reg, op2_reg);
+ }
+ }
+ } else {
+ ir_emit_load_ex(ctx, type, def_reg, op2, def);
+ }
+
+ if (IR_REG_SPILLED(ctx->regs[def][0])) {
+ ir_emit_store(ctx, type, def, def_reg);
+ }
+ return;
+ }
+
if (IR_IS_TYPE_INT(op1_type)) {
if (op1_reg != IR_REG_NONE) {
| ASM_REG_REG_OP test, op1_type, op1_reg, op1_reg
@@ -11377,20 +11567,21 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn)
static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
{
- ir_insn *var_insn = &ctx->ir_base[insn->op2];
ir_ref type = insn->type;
ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
ir_reg fp;
ir_mem mem;
+ int32_t slot;
if (ctx->use_lists[def].count == 1) {
/* dead load */
return;
}
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR);
fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3));
- if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, var_insn->op3)) {
+ slot = ir_local_slot(ctx, insn->op2);
+ mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot));
+ if (def_reg == IR_REG_NONE && ir_is_same_mem_var(ctx, def, slot)) {
return; // fake load
}
IR_ASSERT(def_reg != IR_REG_NONE);
@@ -11420,20 +11611,21 @@ static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
{
- ir_insn *var_insn = &ctx->ir_base[insn->op2];
ir_insn *val_insn = &ctx->ir_base[insn->op3];
ir_ref type = val_insn->type;
ir_reg op3_reg = ctx->regs[ref][3];
ir_reg fp;
ir_mem mem;
+ int32_t slot;
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR);
fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3));
+ slot = ir_local_slot(ctx, insn->op2);
+ mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot));
if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg))
&& !IR_IS_CONST_REF(insn->op3)
&& ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
- && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) {
+ && ir_is_same_mem_var(ctx, insn->op3, slot)) {
return; // fake store
}
if (IR_IS_CONST_REF(insn->op3)) {
@@ -11470,19 +11662,20 @@ static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
static void ir_emit_vstore_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
{
- ir_insn *var_insn = &ctx->ir_base[insn->op2];
ir_ref type = ctx->ir_base[insn->op3].type;
ir_reg op3_reg = ctx->regs[ref][3];
ir_reg fp;
ir_mem mem;
+ int32_t slot;
- IR_ASSERT(var_insn->op == IR_VAR);
+ IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR);
fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(var_insn->op3));
+ slot = ir_local_slot(ctx, insn->op2);
+ mem = IR_MEM_BO(fp, IR_SPILL_POS_TO_OFFSET(slot));
if ((op3_reg == IR_REG_NONE || IR_REG_SPILLED(op3_reg))
&& !IR_IS_CONST_REF(insn->op3)
&& ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
- && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) {
+ && ir_is_same_mem_var(ctx, insn->op3, slot)) {
return; // fake store
}
if (IR_IS_CONST_REF(insn->op3)) {
@@ -11565,6 +11758,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_reg op2_reg = ctx->regs[def][2];
ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
ir_mem mem;
+ bool avoid_spill_store = 0;
if (ctx->use_lists[def].count == 1) {
/* dead load */
@@ -11584,16 +11778,38 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
mem = ir_fuse_addr(ctx, def, insn->op2);
if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, mem)) {
- if (!ir_may_avoid_spill_load(ctx, def, def)) {
- ir_emit_load_mem_fp(ctx, type, def_reg, mem);
+
+ if (ir_may_avoid_spill_load(ctx, def, def)) {
+ avoid_spill_store = 1;
}
- /* avoid load to the same location (valid only when register is not reused) */
- return;
}
}
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) {
+ uint32_t width = IR_VECTOR_SIZE(type);
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_reg tmp_reg = ctx->regs[def][3];
+
+ IR_ASSERT(tmp_reg != IR_REG_NONE);
+ IR_ASSERT(width == 1 || width == 2);
+ if (width == 1) {
+ | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), byte, mem
+ } else {
+ | ASM_TXT_TMEM_OP movzx, Rd(tmp_reg), word, mem
+ }
+ if (ctx->mflags & IR_X86_AVX) {
+ | vmovd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg)
+ } else {
+ | movd xmm(def_reg-IR_REG_FP_FIRST), Rd(tmp_reg)
+ }
+ } else
+#endif
+
ir_emit_load_mem_fp(ctx, type, def_reg, mem);
- if (IR_REG_SPILLED(ctx->regs[def][0])) {
+
+ if (IR_REG_SPILLED(ctx->regs[def][0]) && !avoid_spill_store) {
ir_emit_store(ctx, type, def, def_reg);
}
}
@@ -11738,6 +11954,26 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
}
if (IR_IS_CONST_REF(insn->op3)) {
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) {
+ uint32_t width = IR_VECTOR_SIZE(type);
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+
+ IR_ASSERT(width == 1 || width == 2);
+ if (width == 1) {
+ uint8_t *p = ir_long_const_ptr(ctx, insn->op3);
+ int32_t val = *p;
+
+ | ASM_TMEM_TXT_OP mov, byte, mem, val
+ } else {
+ uint16_t *p = ir_long_const_ptr(ctx, insn->op3);
+ int32_t val = *p;
+
+ | ASM_TMEM_TXT_OP mov, word, mem, val
+ }
+ } else
+#endif
ir_emit_store_mem_fp_const(ctx, type, mem, insn->op3, IR_REG_NONE, op3_reg);
} else {
IR_ASSERT(op3_reg != IR_REG_NONE);
@@ -11745,6 +11981,28 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
op3_reg = IR_REG_NUM(op3_reg);
ir_emit_load(ctx, type, op3_reg, insn->op3);
}
+
+#if IR_SIMD
+ if (IR_IS_TYPE_VECTOR(type) && IR_VECTOR_SIZE(type) < 4) {
+ uint32_t width = IR_VECTOR_SIZE(type);
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_reg tmp_reg = ctx->regs[ref][1];
+
+ IR_ASSERT(width == 1 || width == 2);
+ IR_ASSERT(tmp_reg != IR_REG_NONE);
+ if (ctx->mflags & IR_X86_AVX) {
+ | vmovd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST)
+ } else {
+ | movd Rd(tmp_reg), xmm(op3_reg-IR_REG_FP_FIRST)
+ }
+ if (width == 1) {
+ | ASM_TMEM_TXT_OP mov, byte, mem, Rb(tmp_reg)
+ } else {
+ | ASM_TMEM_TXT_OP mov, word, mem, Rw(tmp_reg)
+ }
+ } else
+#endif
ir_emit_store_mem_fp(ctx, type, mem, op3_reg);
}
}
@@ -11755,7 +12013,7 @@ static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_type type = insn->type;
if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), src_reg)) {
- if (ctx->vregs[def]
+ if (ctx->vregs[def] > 0
&& ctx->live_intervals[ctx->vregs[def]]
&& ctx->live_intervals[ctx->vregs[def]]->stack_spill_pos != -1) {
ir_emit_store(ctx, type, def, src_reg);
@@ -11978,7 +12236,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (ctx->flags & IR_USE_FRAME_POINTER) {
@@ -12012,7 +12270,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (ctx->flags & IR_USE_FRAME_POINTER) {
@@ -12080,7 +12338,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ op2_offset = ir_local_offset(ctx, insn->op2);
}
if (op3_reg != IR_REG_NONE) {
if (IR_REG_SPILLED(op3_reg)) {
@@ -12091,7 +12349,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]);
+ op3_offset = ir_local_offset(ctx, insn->op3);
}
| mov Ra(tmp_reg), aword [Ra(op3_reg)+op3_offset]
| mov aword [Ra(op2_reg)+op2_offset], Ra(tmp_reg)
@@ -12114,7 +12372,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op2_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ op2_offset = ir_local_offset(ctx, insn->op2);
}
if (op3_reg != IR_REG_NONE) {
if (IR_REG_SPILLED(op3_reg)) {
@@ -12125,7 +12383,7 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- op3_offset = ir_local_offset(ctx, &ctx->ir_base[insn->op3]);
+ op3_offset = ir_local_offset(ctx, insn->op3);
}
| mov Rd(tmp_reg), dword [Ra(op3_reg)+(op3_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))]
| mov dword [Ra(op2_reg)+(op2_offset+offsetof(ir_x86_64_sysv_va_list, gp_offset))], Rd(tmp_reg)
@@ -12163,7 +12421,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
| mov Ra(tmp_reg), aword [Ra(op2_reg)+offset]
#if IR_X86_I64
@@ -12233,7 +12491,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER;
- offset = ir_local_offset(ctx, &ctx->ir_base[insn->op2]);
+ offset = ir_local_offset(ctx, insn->op2);
}
if (insn->op3) {
/* long struct arguemnt */
@@ -12266,9 +12524,9 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
|2:
if (def_reg != IR_REG_NONE) {
if (ir_type_size[type] == 8) {
- | mov Rq(def_reg), qword [Ra(tmp_reg)-sizeof(void*)]
+ | mov Rq(def_reg), qword [Ra(tmp_reg)+(0-sizeof(void*))]
} else {
- | mov Rd(def_reg), dword [Ra(tmp_reg)-sizeof(void*)]
+ | mov Rd(def_reg), dword [Ra(tmp_reg)+(0-sizeof(void*))]
}
}
} else {
@@ -12309,7 +12567,7 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
uint32_t n, *p, use_block;
int i;
int label, default_label = 0;
- int count = 0;
+ uint32_t count = 0;
ir_val min, max;
ir_reg op2_reg = ctx->regs[def][2];
ir_reg tmp_reg = ctx->regs[def][3];
@@ -12342,8 +12600,8 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
max.i64 = IR_MAX(max.i64, val->val.i64);
} else {
IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type));
- min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64);
- max.u64 = (int64_t)IR_MAX(max.u64, val->val.u64);
+ min.u64 = IR_MIN(min.u64, val->val.u64);
+ max.u64 = IR_MAX(max.u64, val->val.u64);
}
count++;
} else if (use_insn->op == IR_CASE_RANGE) {
@@ -12354,12 +12612,14 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
IR_ASSERT(!IR_IS_SYM_CONST(val2->op));
if (IR_IS_TYPE_SIGNED(type)) {
IR_ASSERT(IR_IS_TYPE_SIGNED(val->type));
+ IR_ASSERT(IR_IS_TYPE_SIGNED(val2->type));
min.i64 = IR_MIN(min.i64, val->val.i64);
max.i64 = IR_MAX(max.i64, val2->val.i64);
} else {
IR_ASSERT(!IR_IS_TYPE_SIGNED(val->type));
- min.u64 = (int64_t)IR_MIN(min.u64, val->val.u64);
- max.u64 = (int64_t)IR_MAX(max.u64, val2->val.u64);
+ IR_ASSERT(!IR_IS_TYPE_SIGNED(val2->type));
+ min.u64 = IR_MIN(min.u64, val->val.u64);
+ max.u64 = IR_MAX(max.u64, val2->val.u64);
}
} else {
IR_ASSERT(use_insn->op == IR_CASE_DEFAULT);
@@ -12388,8 +12648,10 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
}
/* Generate a table jmp or a seqence of calls */
- if (!has_case_range && count > 2 && (max.i64-min.i64) < count * 8) {
- int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.i64 - min.i64 + 1));
+ if ((sizeof(void*) == 8 || ir_type_size[type] <= 4)
+ && !has_case_range && count > 2 && (max.u64 - min.u64) < count * 8) {
+ int *labels = ir_mem_malloc(sizeof(int) * (size_t)(max.u64 - min.u64 + 1));
+ ir_reg idx_reg;
for (i = 0; i <= (max.i64 - min.i64); i++) {
labels[i] = default_label;
@@ -12442,62 +12704,69 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
}
if (min.i64 != 0) {
- int64_t offset = -min.i64;
+ int64_t offset = 0 - min.u64;
if (IR_IS_SIGNED_32BIT(offset)) {
| lea Ra(tmp_reg), [Ra(op2_reg)+(int32_t)offset]
} else {
- IR_ASSERT(sizeof(void*) == 8);
|.if X64
| mov64 Rq(tmp_reg), offset
| add Ra(tmp_reg), Ra(op2_reg)
|.endif
}
- if (default_label) {
- offset = max.i64 - min.i64;
+ idx_reg = tmp_reg;
+ } else {
+ idx_reg = op2_reg;
+ }
- IR_ASSERT(IR_IS_SIGNED_32BIT(offset));
- | cmp Ra(tmp_reg), (int32_t)offset
- | ja =>default_label
- }
-|.if X64
- if (ctx->code_buffer
- && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start)
- && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) {
- | jmp aword [Ra(tmp_reg)*8+>1]
- } else {
- int64_t offset = -min.i64;
+ if (default_label) {
+ int64_t offset = max.u64 - min.u64;
- IR_ASSERT(IR_IS_SIGNED_32BIT(offset));
- offset *= 8;
- IR_ASSERT(IR_IS_SIGNED_32BIT(offset));
- | lea Ra(tmp_reg), aword [>1]
- | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+offset]
- }
-|.else
- | jmp aword [Ra(tmp_reg)*4+>1]
+ IR_ASSERT(IR_IS_SIGNED_32BIT(offset));
+ | cmp Ra(idx_reg), (int32_t)offset
+ | ja =>default_label
+ }
+
+#ifdef IR_TARGET_X86
+|.if not X64
+ | jmp aword [Ra(idx_reg)*4+>1]
|.endif
+#else
+|.if X64
+ bool use_table_address = 0;
+
+ if ((ctx->code_buffer
+ && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start)
+ && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end))) {
+ /* Use SIB with absolute jmp_table displacement */
+ | jmp aword [Ra(idx_reg)*8+>1]
+
} else {
- if (default_label) {
- int64_t offset = max.i64;
+ /* Use IP-relative addressing mode to get the jmp_table address */
- IR_ASSERT(IR_IS_SIGNED_32BIT(offset));
- | cmp Ra(op2_reg), (int32_t)offset
- | ja =>default_label
- }
-|.if X64
- if (ctx->code_buffer
- && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->start)
- && IR_IS_SIGNED_32BIT((char*)ctx->code_buffer->end)) {
- | jmp aword [Ra(op2_reg)*8+>1]
- } else {
+ // TODO: it's possible to use an extra temporary register (not conflicting with "killed" op2) ???
+// ir_reg op1_reg = ctx->regs[def][1];
+// if (op1_reg != IR_REG_NONE) {
+// | lea Ra(op1_reg), aword [>1]
+// | jmp aword [Ra(op1_reg)+Ra(idx_reg)*8]
+// } else
+ if ((min.i64 >= (INT32_MIN+1)/8) && (min.i64 <= INT32_MAX/8)) {
+ tmp_reg = ctx->regs[def][3];
| lea Ra(tmp_reg), aword [>1]
- | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8]
+ | jmp aword [Ra(tmp_reg)+Ra(op2_reg)*8+(-min.i32 * 8)]
+ } else {
+ use_table_address = 1;
+ if (idx_reg == tmp_reg) {
+ | shl Ra(tmp_reg), 3
+ } else {
+ | lea Ra(tmp_reg), aword [Ra(idx_reg)*8]
+ }
+ | add Ra(tmp_reg), aword [>2]
+ | jmp aword [Ra(tmp_reg)]
}
-|.else
- | jmp aword [Ra(op2_reg)*4+>1]
-|.endif
}
+|.endif
+#endif
|.jmp_table
if (!data->jmp_table_label) {
@@ -12505,6 +12774,12 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
|=>data->jmp_table_label:
}
|.align aword
+#ifndef IR_TARGET_X86
+ if (use_table_address) {
+ |2:
+ | .aword >1
+ }
+#endif
|1:
for (i = 0; i <= (max.i64 - min.i64); i++) {
int b = labels[i];
@@ -13073,7 +13348,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, const i
type = IR_U32;
}
ir_emit_load(ctx, type, dst_reg, arg);
- } else if (ctx->vregs[arg]) {
+ } else if (ctx->vregs[arg] > 0) {
ir_mem mem = ir_ref_spill_slot(ctx, arg);
uint32_t size = ir_type_size[type];
@@ -13294,7 +13569,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro
ir_emit_store(ctx, insn->type, def, def_reg);
}
}
- } else if (ctx->use_lists[def].count > 1) {
+ } else if (ctx->vregs[def] > 0) {
#if IR_X86_I64
if (insn->type == IR_I64 || insn->type == IR_U64) {
IR_ASSERT(cc->int_ret2_reg != IR_REG_NONE);
@@ -13322,14 +13597,14 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
- } else if (ctx->use_lists[def].count > 1) {
+ } else if (ctx->vregs[def] > 0) {
ir_emit_store(ctx, insn->type, def, ret_reg);
}
}
#ifdef IR_TARGET_X86
if (insn->op == IR_TAILCALL) {
/* pass */
- } else if (ctx->use_lists[def].count > 1 && ret_reg == IR_REG_NONE) {
+ } else if (ctx->vregs[def] > 0 && ret_reg == IR_REG_NONE) {
int32_t offset;
ir_reg fp;
@@ -13357,7 +13632,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, const ir_pro
ir_emit_store(ctx, insn->type, def, def_reg);
}
}
- } else {
+ } else if (ret_reg == IR_REG_NONE) {
/* discard return value on top of FPU stack */
| fstp st0
}
@@ -13407,7 +13682,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn)
bool is_spill_slot = op2_reg != IR_REG_NONE
&& IR_REG_SPILLED(op2_reg)
- && ctx->vregs[func];
+ && ctx->vregs[func] > 0;
if (op2_reg != IR_REG_NONE && !is_spill_slot) {
if (IR_REGSET_IN(preserved_regs, IR_REG_NUM(op2_reg))) {
@@ -13948,8 +14223,8 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *
ir_reg op1_reg, op2_reg;
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -14167,7 +14442,7 @@ static void ir_emit_tls_base_addr(ir_ctx *ctx, ir_reg reg, int mod)
||#if defined(_WIN64)
| gs
|| if (mod < 0) {
-| mov Ra(reg), [0]
+| mov Ra(reg), [0x30]
|| } else {
| mov Ra(reg), aword [0x58]
| mov Ra(reg), aword [Ra(reg)+mod]
@@ -14175,7 +14450,7 @@ static void ir_emit_tls_base_addr(ir_ctx *ctx, ir_reg reg, int mod)
||#elif defined(_WIN32)
| fs
|| if (mod < 0) {
-| mov Ra(reg), [0]
+| mov Ra(reg), [0x18]
|| } else {
| mov Ra(reg), aword [0x2c]
| mov Ra(reg), aword [Ra(reg)+mod]
@@ -14679,8 +14954,8 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
ir_reg op1_reg_hi, op2_reg_hi;
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -15082,11 +15357,10 @@ static void ir_emit_mul_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
}
-static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
+static void ir_emit_imul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
{
ir_backend_data *data = ctx->data;
dasm_State **Dst = &data->dasm_state;
- ir_type type = insn->type;
ir_ref op1 = insn->op1;
ir_ref op2 = insn->op2;
ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
@@ -15127,14 +15401,9 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(op2_reg != IR_REG_RDX && tmp_reg != op2_reg_hi && tmp_reg != op2_reg);
- if (type == IR_I64) {
- | mov Rd(tmp2_reg), Rd(op2_reg_hi) // expected sign = sign A ^ sign B
- | xor Rd(tmp2_reg), edx
- | and Rd(tmp2_reg), 0x80000000
- } else {
- | xor Rd(tmp2_reg), Rd(tmp2_reg)
- }
-
+ | mov Rd(tmp2_reg), Rd(op2_reg_hi) // expected sign = sign A ^ sign B
+ | xor Rd(tmp2_reg), edx
+ | and Rd(tmp2_reg), 0x80000000
| mov Rd(tmp_reg), edx
| imul Rd(tmp_reg), Rd(op2_reg_hi) // hi(Ah * Bh) == 0
| adc Rd(tmp2_reg), 0
@@ -15149,31 +15418,22 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| adc Rd(tmp2_reg), 0
| mul Rd(op2_reg) // Al * Bl + c
| add edx, Rd(tmp_reg)
-
- if (type == IR_I64) {
- | adc Rd(tmp2_reg), 0
- | mov Rd(tmp_reg), edx // check sign of result
- | xor Rd(tmp_reg), Rd(tmp2_reg)
- | and Rd(tmp2_reg), 0x7f
- | and Rd(tmp_reg), 0x80000000
- | add Rd(tmp_reg), -1
- | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
- } else {
- | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow
- }
+ | adc Rd(tmp2_reg), 0
+ | mov Rd(tmp_reg), edx // check sign of result
+ | xor Rd(tmp_reg), Rd(tmp2_reg)
+ | and Rd(tmp2_reg), 0x7f
+ | and Rd(tmp_reg), 0x80000000
+ | add Rd(tmp_reg), -1
+ | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
} else if (IR_IS_CONST_REF(op2)) {
ir_insn *val = &ctx->ir_base[op2];
- if (type == IR_I64) {
- | mov Rd(tmp2_reg), edx // expected sign = sign A ^ sign B
- if (val->val.i64 >= 0) {
- | and Rd(tmp2_reg), 0x80000000
- } else {
- | not Rd(tmp2_reg)
- | and Rd(tmp2_reg), 0x80000000
- }
- } else {
- | xor Rd(tmp2_reg), Rd(tmp2_reg)
+ | mov Rd(tmp2_reg), edx // expected sign = sign A ^ sign B
+ if (val->val.i64 >= 0) {
+ | and Rd(tmp2_reg), 0x80000000
+ } else {
+ | not Rd(tmp2_reg)
+ | and Rd(tmp2_reg), 0x80000000
}
if (val->val.u32_hi) {
@@ -15194,22 +15454,18 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| imul Rd(tmp_reg), val->val.u32 // hi(Ah * Bl) == 0, c = lo(Ah * Bl)
| adc Rd(tmp2_reg), 0
}
+
| mov edx, val->val.u32 // Al * Bl + c
| mul edx
| add edx, Rd(tmp_reg)
| adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow
-
- if (type == IR_I64) {
- | adc Rd(tmp2_reg), 0
- | mov Rd(tmp_reg), edx // check sign of result
- | xor Rd(tmp_reg), Rd(tmp2_reg)
- | and Rd(tmp2_reg), 0x7f
- | and Rd(tmp_reg), 0x80000000
- | add Rd(tmp_reg), -1
- | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
- } else {
- | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow
- }
+ | adc Rd(tmp2_reg), 0
+ | mov Rd(tmp_reg), edx // check sign of result
+ | xor Rd(tmp_reg), Rd(tmp2_reg)
+ | and Rd(tmp2_reg), 0x7f
+ | and Rd(tmp_reg), 0x80000000
+ | add Rd(tmp_reg), -1
+ | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
} else {
ir_mem mem, mem_hi;
@@ -15220,14 +15476,9 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
mem_hi = IR_MEM_I64_HI(mem);
- if (type == IR_I64) {
- | ASM_TXT_TMEM_OP mov, Rd(tmp2_reg), dword, mem_hi // expected sign = sign A ^ sign B
- | xor Rd(tmp2_reg), edx
- | and Rd(tmp2_reg), 0x80000000
- } else {
- | xor Rd(tmp2_reg), Rd(tmp2_reg)
- }
-
+ | ASM_TXT_TMEM_OP mov, Rd(tmp2_reg), dword, mem_hi // expected sign = sign A ^ sign B
+ | xor Rd(tmp2_reg), edx
+ | and Rd(tmp2_reg), 0x80000000
| mov Rd(tmp_reg), edx
| ASM_TXT_TMEM_OP imul, Rd(tmp_reg), dword, mem_hi // hi(Ah * Bh) == 0
| adc Rd(tmp2_reg), 0
@@ -15242,18 +15493,204 @@ static void ir_emit_mul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| adc Rd(tmp2_reg), 0
| ASM_TMEM_OP mul, dword, mem // Al * Bl + c
| add edx, Rd(tmp_reg)
+ | adc Rd(tmp2_reg), 0
+ | mov Rd(tmp_reg), edx // check sign of result
+ | xor Rd(tmp_reg), Rd(tmp2_reg)
+ | and Rd(tmp2_reg), 0x7f
+ | and Rd(tmp_reg), 0x80000000
+ | add Rd(tmp_reg), -1
+ | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
+ }
- if (type == IR_I64) {
- | adc Rd(tmp2_reg), 0
- | mov Rd(tmp_reg), edx // check sign of result
- | xor Rd(tmp_reg), Rd(tmp2_reg)
- | and Rd(tmp2_reg), 0x7f
- | and Rd(tmp_reg), 0x80000000
- | add Rd(tmp_reg), -1
- | adc Rd(tmp2_reg), 0x7fffffff // set OV flag if tmp2_reg != 0
- } else {
- | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow
+ if (def_reg != IR_REG_NONE) {
+ def_reg_hi = IR_REG_I64_HI(def_reg);
+ def_reg = IR_REG_I64_LO(def_reg);
+ if (def_reg != IR_REG_RDX) {
+ if (def_reg != IR_REG_RAX) {
+ | mov Rd(def_reg), eax
+ }
+ if (def_reg_hi != IR_REG_RDX) {
+ | mov Rd(def_reg_hi), edx
+ }
+ } else {
+ if (def_reg_hi != IR_REG_RDX) {
+ | mov Rd(def_reg_hi), edx
+ }
+ if (def_reg != IR_REG_RAX) {
+ IR_ASSERT(def_reg_hi != IR_REG_RAX);
+ | mov Rd(def_reg), eax
+ }
+ }
+
+ if (IR_REG_SPILLED(ctx->regs[def][0])) {
+ ir_emit_store_i64_lo(ctx, def, def_reg);
+ ir_emit_store_i64_hi(ctx, def, def_reg_hi);
+ }
+ } else {
+ ir_emit_store_i64_lo(ctx, def, IR_REG_RAX);
+ ir_emit_store_i64_hi(ctx, def, IR_REG_RDX);
+ }
+}
+
+static void ir_emit_umul_ov_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_ref op1 = insn->op1;
+ ir_ref op2 = insn->op2;
+ ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
+ ir_reg op1_reg = ctx->regs[def][1];
+ ir_reg op2_reg = ctx->regs[def][2];
+ ir_reg tmp_reg = ctx->regs[def][3];
+ ir_reg tmp2_reg = ctx->tmp_regs[def];
+ ir_reg def_reg_hi, op1_reg_hi, op2_reg_hi;
+ ir_mem op1_mem = IR_MEM_NONE, op1_mem_hi = IR_MEM_NONE, op2_mem = IR_MEM_NONE, op2_mem_hi = IR_MEM_NONE;
+ ir_val op1_val = {0};
+
+ if (IR_IS_CONST_REF(op1) && IR_IS_CONST_REF(op2)) {
+ op1_val.u64 = ctx->ir_base[op1].val.u64 * ctx->ir_base[op2].val.u64;
+ | mov edx, op1_val.u32_hi
+ | mov eax, op1_val.u32
+ if (ctx->ir_base[op1].val.u64 == 0 || (op1_val.u64 / ctx->ir_base[op1].val.u64) == ctx->ir_base[op2].val.u64) {
+ | clc
+ } else {
+ | stc
+ }
+ } else {
+ if (op1_reg != IR_REG_NONE) {
+ if (IR_REG_SPILLED(op1_reg)) {
+ op1_reg = IR_REG_NUM(op1_reg);
+ op1_reg_hi = IR_REG_I64_HI(op1_reg);
+ op1_reg = IR_REG_I64_LO(op1_reg);
+ ir_emit_load_i64_lo(ctx, op1_reg, op1);
+ ir_emit_load_i64_hi(ctx, op1_reg_hi, op1);
+ } else {
+ op1_reg_hi = IR_REG_I64_HI(op1_reg);
+ op1_reg = IR_REG_I64_LO(op1_reg);
+ }
+ if (op1 == op2) {
+ op2_reg_hi = op1_reg_hi;
+ op2_reg = op1_reg;
+ }
+ } else {
+ op1_reg_hi = IR_REG_NONE;
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ if (IR_REG_SPILLED(op2_reg)) {
+ op2_reg = IR_REG_NUM(op2_reg);
+ op2_reg_hi = IR_REG_I64_HI(op2_reg);
+ op2_reg = IR_REG_I64_LO(op2_reg);
+ ir_emit_load_i64_lo(ctx, op2_reg, op2);
+ ir_emit_load_i64_hi(ctx, op2_reg_hi, op2);
+ } else {
+ op2_reg_hi = IR_REG_I64_HI(op2_reg);
+ op2_reg = IR_REG_I64_LO(op2_reg);
+ }
+ } else {
+ op2_reg_hi = IR_REG_NONE;
+ }
+
+ if (op2_reg == IR_REG_NONE && (IR_IS_CONST_REF(op2) || op1_reg != IR_REG_NONE)) {
+ IR_ASSERT(op1_reg != IR_REG_NONE || !IR_IS_CONST_REF(op1));
+ SWAP_REFS(op1, op2);
+ SWAP_REGS(op1_reg, op2_reg);
+ SWAP_REGS(op1_reg_hi, op2_reg_hi);
+ }
+
+ if (op1_reg == IR_REG_NONE) {
+ if (IR_IS_CONST_REF(op1)) {
+ op1_val = ctx->ir_base[op1].val;
+ } else {
+ if (ir_rule(ctx, op1) & IR_FUSED) {
+ op1_mem = ir_fuse_load(ctx, def, op1);
+ } else {
+ op1_mem = ir_ref_spill_slot(ctx, op1);
+ }
+ op1_mem_hi = IR_MEM_I64_HI(op1_mem);
+ }
+ }
+
+ if (op2_reg == IR_REG_NONE) {
+ IR_ASSERT(!IR_IS_CONST_REF(op2));
+ if (ir_rule(ctx, op2) & IR_FUSED) {
+ op2_mem = ir_fuse_load(ctx, def, op2);
+ } else {
+ op2_mem = ir_ref_spill_slot(ctx, op2);
+ }
+ op2_mem_hi = IR_MEM_I64_HI(op2_mem);
+ }
+
+ | xor Rd(tmp2_reg), Rd(tmp2_reg)
+
+ if (op1_reg != IR_REG_NONE) {
+ | mov eax, Rd(op1_reg_hi)
+ } else if (IR_IS_CONST_REF(op1)) {
+ | mov eax, op1_val.u32_hi
+ } else {
+ | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem_hi
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ | mul Rd(op2_reg_hi) // hi(Ah * Bh) == 0
+ } else {
+ | ASM_TMEM_OP mul, dword, op2_mem_hi // hi(Ah * Bh) == 0
+ }
+
+ | adc Rd(tmp2_reg), 0
+ | add eax, -1 // lo(Ah * Bh) == 0 ; set C flag if tmp_reg != 0
+ | adc Rd(tmp2_reg), 0
+
+ if (op1_reg != IR_REG_NONE) {
+ | mov eax, Rd(op1_reg_hi)
+ } else if (IR_IS_CONST_REF(op1)) {
+ | mov eax, op1_val.u32_hi
+ } else {
+ | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem_hi
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ | mul Rd(op2_reg) // hi(Ah * Bl) == 0
+ } else {
+ | ASM_TMEM_OP mul, dword, op2_mem // hi(Ah * Bl) == 0
+ }
+
+ | adc Rd(tmp2_reg), 0
+ | mov Rd(tmp_reg), eax // c = lo(Ah * Bl)
+
+ if (op1_reg != IR_REG_NONE) {
+ | mov eax, Rd(op1_reg)
+ } else if (IR_IS_CONST_REF(op1)) {
+ | mov eax, op1_val.u32
+ } else {
+ | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem
+ }
+ if (op2_reg != IR_REG_NONE) {
+ | mul Rd(op2_reg_hi) // hi(Al * Bh) == 0
+ } else {
+ | ASM_TMEM_OP mul, dword, op2_mem_hi // hi(Al * Bh) == 0
+ }
+
+ | adc Rd(tmp2_reg), 0
+ | add Rd(tmp_reg), eax // c = lo(Ah * Bl) + lo(Al * Bh)
+ | adc Rd(tmp2_reg), 0
+
+ if (op1_reg != IR_REG_NONE) {
+ | mov eax, Rd(op1_reg)
+ } else if (IR_IS_CONST_REF(op1)) {
+ | mov eax, op1_val.u32
+ } else {
+ | ASM_TXT_TMEM_OP mov, eax, dword, op1_mem
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ | mul Rd(op2_reg) // Al * Bl
+ } else {
+ | ASM_TMEM_OP mul, dword, op2_mem // Al * Bl
}
+
+ | add edx, Rd(tmp_reg) // Al * Bl + c
+ | adc Rd(tmp2_reg), -1 // set C flag if UMUL overflow
}
if (def_reg != IR_REG_NONE) {
@@ -16634,8 +17071,8 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(tmp_reg != IR_REG_NONE);
if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2);
} else {
op1_reg = ctx->regs[insn->op1][1];
op2_reg = ctx->regs[insn->op1][2];
@@ -16796,8 +17233,8 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(tmp_reg != IR_REG_NONE);
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op1 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2);
} else {
op1_reg = ctx->regs[insn->op1][1];
op2_reg = ctx->regs[insn->op1][2];
@@ -17265,8 +17702,8 @@ static bool ir_emit_guard_cmp_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *
void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
- op1_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 1);
- op2_reg = ir_get_fused_reg(ctx, def, insn->op2 * sizeof(ir_ref) + 2);
+ op1_reg = ir_get_fused_reg(ctx, def, insn->op2, 1);
+ op2_reg = ir_get_fused_reg(ctx, def, insn->op2, 2);
} else {
op1_reg = ctx->regs[insn->op2][1];
op2_reg = ctx->regs[insn->op2][2];
@@ -17656,11 +18093,11 @@ static void ir_emit_vector_extract(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (IR_REG_SPILLED(op2_reg)) {
op2_reg = IR_REG_NUM(op2_reg);
- ir_emit_load(ctx, element_type, op2_reg, insn->op2);
+ ir_emit_load(ctx, ctx->ir_base[insn->op2].type, op2_reg, insn->op2);
}
/* extract through stack memory */
- int offset = -width;
+ int offset = 0 - width;
ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1);
ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]);
@@ -17789,7 +18226,7 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn)
int offset = width;
| movdqu [Ra(IR_REG_RSP)-offset], xmm(tmp_reg-IR_REG_FP_FIRST)
- | mov [Ra(IR_REG_RSP)-(offset-(lane%16))], Rb(op3_reg)
+ | mov [Ra(IR_REG_RSP)+((lane%16)-offset)], Rb(op3_reg)
| movdqu xmm(tmp_reg-IR_REG_FP_FIRST), [Ra(IR_REG_RSP)-offset]
}
} else if (element_type == IR_I16 || element_type == IR_U16) {
@@ -17931,12 +18368,12 @@ static void ir_emit_vector_replace(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(op2_reg != IR_REG_NONE);
if (IR_REG_SPILLED(op2_reg)) {
- op3_reg = IR_REG_NUM(op2_reg);
- ir_emit_load(ctx, element_type, op2_reg, insn->op2);
+ op2_reg = IR_REG_NUM(op2_reg);
+ ir_emit_load(ctx, ctx->ir_base[insn->op2].type, op2_reg, insn->op2);
}
/* modify through stack memory */
- int offset = -width;
+ int offset = 0 - width;
ir_mem mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1);
ir_mem mem2 = IR_MEM(IR_REG_RSP, offset, op2_reg, ir_type_size[element_type]);
@@ -18790,7 +19227,7 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_type mask_element_type = IR_VECTOR_BASE_TYPE(mask_type);
uint32_t mask_element_size = ir_type_size[mask_element_type];
uint32_t mask_len = IR_VECTOR_LENGTH(mask_type);
- int dst_offset = -width;
+ int dst_offset = 0 - width;
int src_offset = dst_offset;
uint32_t mod = 0;
ir_reg tmp_reg = ctx->tmp_regs[def];
@@ -18818,22 +19255,22 @@ static void ir_emit_vector_shuffle(ir_ctx *ctx, ir_ref def, ir_insn *insn)
ir_mem src, dst;
if (mask_element_size == 1) {
- j = *(uint8_t*)ptr;
+ j = *(uint8_t*)ptr % mod;
ptr = ((uint8_t*)ptr) + 1;
} else if (mask_element_size == 2) {
- j = *(uint16_t*)ptr;
+ j = *(uint16_t*)ptr % mod;
ptr = ((uint16_t*)ptr) + 1;
} else if (mask_element_size == 4) {
- j = *(uint32_t*)ptr;
+ j = *(uint32_t*)ptr % mod;
ptr = ((uint32_t*)ptr) + 1;
} else if (mask_element_size == 8) {
- j = *(uint64_t*)ptr;
+ j = *(uint64_t*)ptr % mod;
ptr = ((uint64_t*)ptr) + 1;
} else {
IR_ASSERT(0);
j = 0;
}
- src = IR_MEM(IR_REG_RSP, src_offset + (j % mod) * element_size, IR_REG_NONE, 1);
+ src = IR_MEM(IR_REG_RSP, src_offset + j * element_size, IR_REG_NONE, 1);
dst = IR_MEM(IR_REG_RSP, dst_offset + i * element_size, IR_REG_NONE, 1);
#if IR_X86_I64
if (element_type == IR_I64 || element_type == IR_U64) {
@@ -19039,7 +19476,6 @@ static void ir_emit_vector_op(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else {
ir_mem mem;
- IR_ASSERT(width == 16);
if (ir_rule(ctx, insn->op1) & IR_FUSED) {
mem = ir_fuse_load(ctx, def, insn->op1);
} else {
@@ -19121,7 +19557,7 @@ static void ir_emit_vector_op(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| vpcmpeqd ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST)
| ASM_TXT_TXT_TMEM_OP vpxor, ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), yword, mem
} else {
- | vxorps ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), ymm(op1_reg-IR_REG_FP_FIRST)
+ | vxorps ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST)
| vcmpps ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), 0xf
| ASM_TXT_TXT_TMEM_OP vxorps, ymm(def_reg-IR_REG_FP_FIRST), ymm(def_reg-IR_REG_FP_FIRST), yword, mem
}
@@ -21978,7 +22414,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn)
dst_mem = ir_ref_spill_slot(ctx, def);
def_reg = IR_REG_NUM(def_reg);
} else {
- offset = -width;
+ offset = 0 - width;
dst_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1);
}
@@ -21991,7 +22427,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (offset != 0) {
src1_mem = dst_mem;
} else {
- offset = -width;
+ offset = 0 - width;
src1_mem = IR_MEM(IR_REG_RSP, offset, IR_REG_NONE, 1);
}
ir_emit_store_mem_fp(ctx, type, src1_mem, op1_reg);
@@ -22144,7 +22580,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (!scalar_shift) {
if (element_size == 8) {
if (IR_IS_SIGNED_32BIT(((int64_t*)ptr2)[i])) {
- val = ((int64_t*)ptr2)[i];
+ val = ((int64_t*)ptr2)[i] & 0xffffffff;
} else {
#if IR_X86_I64
val_hi = ((uint64_t*)ptr2)[i] >> 32;
@@ -23811,7 +24247,7 @@ static void ir_emit_vector_fp2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
if (src_type == IR_DOUBLE) {
- IR_ASSERT(dst_type == IR_DOUBLE);
+ IR_ASSERT(dst_type == IR_FLOAT);
if (ctx->mflags & IR_X86_AVX) {
if (src_width == 32) {
| ASM_TXT_TMEM_OP vcvtpd2ps, ymm(def_reg-IR_REG_FP_FIRST), yword, mem
@@ -25739,7 +26175,7 @@ static void ir_emit_load_params(ir_ctx *ctx)
int_param_num++;
}
}
- if (ctx->vregs[use]) {
+ if (ctx->vregs[use] > 0) {
dst_reg = IR_REG_NUM(ctx->regs[use][0]);
IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE ||
stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos);
@@ -25761,87 +26197,6 @@ static void ir_emit_load_params(ir_ctx *ctx)
}
}
-static int ir_fix_dessa_tmps(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to, void *dessa_from_block)
-{
- ir_ref ref = ctx->cfg_blocks[(intptr_t)dessa_from_block].end;
-
- if (to == 0) {
- if (IR_IS_TYPE_INT(type)) {
- if (ctx->regs[ref][0] == IR_REG_NONE) {
- ctx->regs[ref][0] = IR_REG_RAX;
- }
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type));
- if (ctx->regs[ref][1] == IR_REG_NONE) {
- ctx->regs[ref][1] = IR_REG_XMM0;
- }
- }
- } else if (from != 0) {
- if (IR_IS_TYPE_INT(type)) {
- if (ctx->regs[ref][0] == IR_REG_NONE) {
- ctx->regs[ref][0] = IR_REG_RAX;
- }
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type));
- if (ctx->regs[ref][1] == IR_REG_NONE) {
- ctx->regs[ref][1] = IR_REG_XMM0;
- }
- }
- }
- return 1;
-}
-
-/* This remaps spill slot of the VAR, where PARAM is stored, to the PARAMs slot.
- * This eliminates copying, but more important, this fixes va_start() compilation
- * on systems, where it's defined through address calculation macro (e.g. MSVC x86).
- */
-static void ir_remap_param_spill_slot(ir_ctx *ctx, ir_ref param, int32_t spill_slot)
-{
- ir_use_list *param_use_list = &ctx->use_lists[param];
-
- if (param_use_list->count >= 1) {
- ir_ref param_store = ctx->use_edges[param_use_list->refs];
- ir_insn *param_store_insn = &ctx->ir_base[param_store];
-
- if (param_store_insn->op == IR_VSTORE) {
- ir_ref var = param_store_insn->op2;
- ir_insn *var_insn = &ctx->ir_base[var];
-
- IR_ASSERT(param_store_insn->op3 == param);
- IR_ASSERT(ctx->ir_base[var].op == IR_VAR);
-
- /* Remap all related VADDR and VLOAD */
- ir_use_list *var_use_list = &ctx->use_lists[var];
- ir_ref n = var_use_list->count;
- ir_ref *p = &ctx->use_edges[var_use_list->refs];
- for (; n > 0; p++, n--) {
- ir_ref use = *p;
- ir_insn *insn = &ctx->ir_base[use];
-
- if (insn->op == IR_VADDR) {
- if (ctx->rules[use] == IR_STATIC_ALLOCA) {
- IR_ASSERT(insn->op3 == var_insn->op3);
- insn->op3 = spill_slot;
- }
- } else if (insn->op == IR_VLOAD) {
- if (ctx->vregs[use]) {
- ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]];
- if (ival->stack_spill_pos == var_insn->op3) {
- ival->stack_spill_pos = spill_slot;
- }
- }
- }
- }
-
- /* Remap VAR itself */
- var_insn->op3 = spill_slot;
-
- /* Avoid copying to itself */
- ctx->rules[param_store] = IR_SKIPPED | IR_NOP;
- }
- }
-}
-
static void ir_fix_param_spills(ir_ctx *ctx)
{
ir_use_list *use_list = &ctx->use_lists[1];
@@ -25877,7 +26232,7 @@ static void ir_fix_param_spills(ir_ctx *ctx)
align = IR_MAX(sizeof(void*), align);
stack_offset = IR_ALIGNED_SIZE(stack_offset, align);
- ctx->value_params[insn->op3 - 1].offset = stack_start + stack_offset;
+ ctx->vregs[use] = IR_STACK_SLOT_TO_VREG(stack_start + stack_offset);
stack_offset += ctx->value_params[insn->op3 - 1].size;
stack_offset = IR_ALIGNED_SIZE(stack_offset, sizeof(void*));
continue;
@@ -25923,13 +26278,15 @@ static void ir_fix_param_spills(ir_ctx *ctx)
}
}
if (src_reg == IR_REG_NONE) {
- if (ctx->vregs[use]) {
+ if (ctx->vregs[use] > 0) {
ir_live_interval *ival = ctx->live_intervals[ctx->vregs[use]];
- if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)
- && ival->stack_spill_pos == -1) {
+ if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) {
+ IR_ASSERT(ival->stack_spill_pos == -1);
ival->stack_spill_pos = stack_start + stack_offset;
- /* Remap VAR to PARAM stack slot */
- ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
+ if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
+ /* Remap VAR to PARAM stack slot */
+ ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
+ }
}
}
if (sizeof(void*) == 8) {
@@ -26003,7 +26360,7 @@ void ir_fix_stack_frame(ir_ctx *ctx)
#ifdef IR_TARGET_X86
if (ctx->flags2 & IR_HAS_MEMCPY) {
- ir_regset tmp = ctx->used_preserved_regs;
+ ir_regset tmp = (ir_regset)ctx->used_preserved_regs;
IR_REGSET_INCL(tmp, IR_REG_RSI);
IR_REGSET_INCL(tmp, IR_REG_RDI);
ctx->used_preserved_regs = tmp;
@@ -26686,8 +27043,11 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr)
case IR_MUL_I64:
ir_emit_mul_i64(ctx, i, insn);
break;
- case IR_MUL_OV_I64:
- ir_emit_mul_ov_i64(ctx, i, insn);
+ case IR_IMUL_OV_I64:
+ ir_emit_imul_ov_i64(ctx, i, insn);
+ break;
+ case IR_UMUL_OV_I64:
+ ir_emit_umul_ov_i64(ctx, i, insn);
break;
case IR_BINOP_HELPER_I64:
ir_emit_binop_helper_i64(ctx, i, insn);
@@ -26810,7 +27170,7 @@ next_block:;
|.align 8
|=>label:
while (str[i]) {
- char c = str[i];
+ uint8_t c = (uint8_t)str[i];
|.byte c
i++;