Commit 454c7a698b9 for php
commit 454c7a698b9633d576bfd539e89108c37abc69ad
Author: Jorg Adam Sowa <jorg.sowa@gmail.com>
Date: Thu Nov 5 13:44:09 2026 +0100
Update IR (exclude ir_ra_simple) (#25180)
* Update IR and drop unused register allocators
IR commit: 4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e
Stop bundling ir_ra_simple.c and ir_ra_fast.c, which are not used.
diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh
index 74a9789f35a..9a1f8faa1be 100755
--- a/.github/scripts/download-bundled/jit-ir.sh
+++ b/.github/scripts/download-bundled/jit-ir.sh
@@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.."
tmp_dir=/tmp/php-src-download-bundled/jit-ir
rm -rf "$tmp_dir"
-revision=30a40633b0b8f3e8e66da0a25941c4199d69d38a
+revision=4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e
git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir"
@@ -32,6 +32,8 @@ rm ir_load.c
rm ir_load_llvm.c
rm ir_main.c
rm ir_mem2ssa.c
+rm ir_ra_fast.c
+rm ir_ra_simple.c
# add extra files
git restore README
diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4
index 224e9e54a2e..2a1ac4e332f 100644
--- a/ext/opcache/config.m4
+++ b/ext/opcache/config.m4
@@ -48,7 +48,6 @@ AS_VAR_IF([PHP_OPCACHE_JIT], [yes], [
jit/ir/ir_patch.c
jit/ir/ir_perf.c
jit/ir/ir_ra.c
- jit/ir/ir_ra_simple.c
jit/ir/ir_save.c
jit/ir/ir_sccp.c
jit/ir/ir_strtab.c
diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32
index f92b02e5967..cd22b363db0 100644
--- a/ext/opcache/config.w32
+++ b/ext/opcache/config.w32
@@ -25,7 +25,7 @@ if (PHP_OPCACHE_JIT == "yes") {
} else if (CHECK_HEADER("ir/ir.h", "CFLAGS_OPCACHE", PHP_OPCACHE + ";ext\\opcache\\jit")) {
var dasm_flags = (X64 ? "-D X64=1" : "") + (X64 ? " -D X64WIN=1" : "") + " -D WIN=1";
var ir_target = (X64 ? "IR_TARGET_X64" : "IR_TARGET_X86");
- var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_ra_simple.c ir_save.c \
+ var ir_src = "ir_strtab.c ir_cfg.c ir_sccp.c ir_gcm.c ir_ra.c ir_save.c \
ir_dump.c ir_check.c ir_patch.c";
DEFINE("IR_TARGET", ir_target);
diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32
index b5273a1b86e..7caf8145887 100644
--- a/ext/opcache/jit/Makefile.frag.w32
+++ b/ext/opcache/jit/Makefile.frag.w32
@@ -19,11 +19,6 @@ $(BUILD_DIR)\ext\opcache\jit\ir\ir_ra.obj: \
ext\opcache\jit\ir\ir_private.h \
ext\opcache\jit\ir\ir_x86.h
-$(BUILD_DIR)\ext\opcache\jit\ir\ir_ra_simple.obj: \
- ext\opcache\jit\ir\ir.h \
- ext\opcache\jit\ir\ir_private.h \
- ext\opcache\jit\ir\ir_x86.h
-
$(BUILD_DIR)\ext\opcache\jit\ir\ir_emit.obj: \
ext\opcache\jit\ir\ir.h \
ext\opcache\jit\ir\ir_private.h \
diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c
index 734e9086648..6cfd2a0007f 100644
--- a/ext/opcache/jit/ir/ir.c
+++ b/ext/opcache/jit/ir/ir.c
@@ -119,7 +119,20 @@ static void ir_print_double(double v, FILE *f)
char buf[128];
if (isnan(v)) {
- fprintf(f, "nan");
+ union {
+ double d;
+ long long i;
+ } u1, u2, u3;
+ u1.d = v;
+ u2.d = NAN;
+ u3.d = -NAN;
+ if (u1.i == u2.i) {
+ fprintf(f, "nan");
+ } else if (u1.i == u3.i) {
+ fprintf(f, "-nan");
+ } else {
+ fprintf(f, "nan(0x%llx)", u1.i);
+ }
} else {
snprintf(buf, sizeof(buf), "%g", v);
if (strtod(buf, NULL) != v) {
@@ -137,7 +150,20 @@ static void ir_print_float(float v, FILE *f)
char buf[128];
if (isnan(v)) {
- fprintf(f, "nan");
+ union {
+ float f;
+ int i;
+ } u1, u2, u3;
+ u1.f = v;
+ u2.f = NAN;
+ u3.f = -NAN;
+ if (u1.i == u2.i) {
+ fprintf(f, "nan");
+ } else if (u1.i == u3.i) {
+ fprintf(f, "-nan");
+ } else {
+ fprintf(f, "nan(0x%x)", u1.i);
+ }
} else {
snprintf(buf, sizeof(buf), "%g", v);
if (strtod(buf, NULL) != v) {
diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h
index 1d7bd07b381..74481865e0f 100644
--- a/ext/opcache/jit/ir/ir.h
+++ b/ext/opcache/jit/ir/ir.h
@@ -968,8 +968,9 @@ int ir_coalesce(ir_ctx *ctx);
int ir_compute_dessa_moves(ir_ctx *ctx);
int ir_reg_alloc(ir_ctx *ctx);
int ir_reg_alloc_simple(ir_ctx *ctx);
+int ir_reg_alloc_fast(ir_ctx *ctx);
-#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_simple(_ctx)
+#define ir_reg_alloc_O0(_ctx) ir_reg_alloc_fast(_ctx)
int ir_regs_number(void);
bool ir_reg_is_int(int32_t reg);
diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc
index 2bc28e59e33..965b3e9f745 100644
--- a/ext/opcache/jit/ir/ir_aarch64.dasc
+++ b/ext/opcache/jit/ir/ir_aarch64.dasc
@@ -1413,8 +1413,9 @@ binop_fp:
if (IR_IS_TYPE_VECTOR(insn->type)) {
return IR_VECTOR_OP;
}
- IR_FALLTHROUGH;
#endif
+ IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type));
+ IR_FALLTHROUGH;
case IR_BSWAP:
case IR_CTLZ:
case IR_CTTZ:
@@ -3487,29 +3488,41 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
if (ir_type_size[type] == 8) {
- | cmp Rx(op1_reg), #0
- if (op2_reg != IR_REG_NONE) {
- | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg)
+ if (shift == 63) {
+ | movz Rx(def_reg), #0x8000, lsl #48
+ | cmp Rx(def_reg), Rx(op1_reg)
+ | cset Rx(def_reg), eq
} else {
- | add Rx(def_reg), Rx(op1_reg), #offset
+ | cmp Rx(op1_reg), #0
+ if (op2_reg != IR_REG_NONE) {
+ | add Rx(def_reg), Rx(op1_reg), Rx(op2_reg)
+ } else {
+ | add Rx(def_reg), Rx(op1_reg), #offset
+ }
+ | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt
+ | asr Rx(def_reg), Rx(def_reg), #shift
}
- | csel Rx(def_reg), Rx(def_reg), Rx(op1_reg), lt
- | asr Rx(def_reg), Rx(def_reg), #shift
} else {
- | cmp Rw(op1_reg), #0
- if (op2_reg != IR_REG_NONE) {
- | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg)
- } else {
- | add Rw(def_reg), Rw(op1_reg), #offset
- }
- | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt
- if (ir_type_size[type] == 4) {
- | asr Rw(def_reg), Rw(def_reg), #shift
- } else if (ir_type_size[type] == 2) {
- | ubfx Rw(def_reg), Rw(def_reg), #shift, #16
+ if (shift == 31) {
+ | movz Rw(def_reg), #0x8000, lsl #16
+ | cmp Rw(def_reg), Rw(op1_reg)
+ | cset Rw(def_reg), eq
} else {
- IR_ASSERT(ir_type_size[type] == 1);
- | ubfx Rw(def_reg), Rw(def_reg), #shift, #8
+ | cmp Rw(op1_reg), #0
+ if (op2_reg != IR_REG_NONE) {
+ | add Rw(def_reg), Rw(op1_reg), Rw(op2_reg)
+ } else {
+ | add Rw(def_reg), Rw(op1_reg), #offset
+ }
+ | csel Rw(def_reg), Rw(def_reg), Rw(op1_reg), lt
+ if (ir_type_size[type] == 4) {
+ | asr Rw(def_reg), Rw(def_reg), #shift
+ } else if (ir_type_size[type] == 2) {
+ | ubfx Rw(def_reg), Rw(def_reg), #shift, #16
+ } else {
+ IR_ASSERT(ir_type_size[type] == 1);
+ | ubfx Rw(def_reg), Rw(def_reg), #shift, #8
+ }
}
}
if (IR_REG_SPILLED(ctx->regs[def][0])) {
@@ -3965,10 +3978,10 @@ static void ir_emit_binop_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| ASM_FP_REG_REG_REG_OP fdiv, type, def_reg, op1_reg, op2_reg
break;
case IR_MIN:
- | ASM_FP_REG_REG_REG_OP fmin, type, def_reg, op1_reg, op2_reg
+ | ASM_FP_REG_REG_REG_OP fminnm, type, def_reg, op1_reg, op2_reg
break;
case IR_MAX:
- | ASM_FP_REG_REG_REG_OP fmax, type, def_reg, op1_reg, op2_reg
+ | ASM_FP_REG_REG_REG_OP fmaxnm, type, def_reg, op1_reg, op2_reg
break;
}
if (IR_REG_SPILLED(ctx->regs[def][0])) {
@@ -9272,10 +9285,10 @@ static void ir_fix_param_spills(ir_ctx *ctx)
if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) {
IR_ASSERT(ival->stack_spill_pos == -1);
ival->stack_spill_pos = stack_start + stack_offset;
- if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
- /* Remap VAR to PARAM stack slot */
- ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
- }
+ }
+ if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
+ /* Remap VAR to PARAM stack slot */
+ ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
}
}
if (sizeof(void*) == 8) {
diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c
index 4e4db166d37..621f02a2372 100644
--- a/ext/opcache/jit/ir/ir_cfg.c
+++ b/ext/opcache/jit/ir/ir_cfg.c
@@ -1106,6 +1106,34 @@ static IR_NEVER_INLINE uint32_t ir_collect_irreducible_loops(ir_ctx *ctx, uint32
return loops;
}
+static IR_NEVER_INLINE void ir_check_loop_entry_in_preds(ir_ctx *ctx, uint32_t hdr, ir_worklist *work)
+{
+ ir_block *blocks = ctx->cfg_blocks;
+ uint32_t *edges = ctx->cfg_edges;
+
+ ir_worklist_clear(work);
+ ir_bitset_incl(work->visited, blocks[hdr].idom);
+ ir_worklist_push(work, hdr);
+ while (ir_worklist_len(work)) {
+ uint32_t b = ir_worklist_pop(work);
+ ir_block *bb = &blocks[b];
+
+ if (bb->flags & IR_BB_ENTRY) {
+ blocks[hdr].flags |= IR_BB_LOOP_WITH_ENTRY_IN_PREDS;
+ break;
+ }
+
+ uint32_t *p = &edges[bb->predecessors];
+ uint32_t count = bb->predecessors_count;
+
+ for (; count > 0; p++, count--) {
+ if (!ir_dominates(blocks, hdr, *p)) {
+ ir_worklist_push(work, *p);
+ }
+ }
+ }
+}
+
int ir_find_loops(ir_ctx *ctx)
{
uint32_t b, j, n;
@@ -1295,6 +1323,10 @@ int ir_find_loops(ir_ctx *ctx)
b = loops;
do {
bb = &blocks[b];
+ if (ctx->entries_count
+ && (bb->predecessors_count > 2 || (bb->flags & IR_BB_IRREDUCIBLE_LOOP))) {
+ ir_check_loop_entry_in_preds(ctx, b, &work);
+ }
b = bb->next_loop;
IR_ASSERT(bb->flags & IR_BB_LOOP_HEADER);
bb->loop_depth = (bb->loop_header) ? blocks[bb->loop_header].loop_depth + 1 : 1;
diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c
index bc3ae5859be..de6b53141a6 100644
--- a/ext/opcache/jit/ir/ir_emit.c
+++ b/ext/opcache/jit/ir/ir_emit.c
@@ -541,7 +541,9 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block
static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg)
{
int i;
- int8_t *pred, *loc, *types;
+ int8_t *loc; /* array of the current locations of the original source registers: initially loc[from] = from */
+ int8_t *pred; /* array of original source registers that must be copied to destinations: pred[to] = from */
+ int8_t *types; /* array of the types of the orginal source registers: types[from] = typeof(from) */
ir_reg to, from;
ir_type type;
ir_regset todo, ready, srcs;
@@ -625,18 +627,19 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_
IR_ASSERT(tmp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_reg) || pred[loc[tmp_reg]] == tmp_reg);
IR_ASSERT(tmp_fp_reg == IR_REG_NONE || !IR_REGSET_IN(srcs, tmp_fp_reg) || pred[loc[tmp_fp_reg]] == tmp_fp_reg);
- /* now we resolve all "windmill axles" - cycles (this reuires temporary registers) */
+ /* now we resolve all "windmill axles" - cycles (this requires temporary registers) */
while (todo != IR_REGSET_EMPTY) {
- to = ir_regset_pop_first(&todo);
+ to = ir_regset_pop_first(&todo); /* select a register to break the cycle */
from = pred[to];
IR_ASSERT(to != loc[from]);
- type = types[from];
+ type = types[to]; /* type of the register that needs to be moved into a temporary register */
if (IR_IS_TYPE_INT(type)) {
#ifdef IR_HAVE_SWAP_INT
if (pred[from] == to) {
- if (ir_type_size[types[to]] > ir_type_size[type]) {
- type = types[to];
+ if (ir_type_size[types[from]] > ir_type_size[type]) {
+ type = types[from];
}
+ /* resolve the cycle with 2 registers using SWAP */
ir_emit_swap(ctx, type, to, from);
IR_REGSET_EXCL(todo, from);
loc[to] = from;
@@ -646,11 +649,13 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_
#endif
IR_ASSERT(tmp_reg != IR_REG_NONE);
IR_ASSERT(tmp_reg >= IR_REG_GP_FIRST && tmp_reg <= IR_REG_GP_LAST);
+ /* break cycle, moving one of its register into a temporary register */
ir_emit_mov(ctx, type, tmp_reg, to);
loc[to] = tmp_reg;
} else {
#ifdef IR_HAVE_SWAP_FP
- if (pred[from] == to && types[to] == type) {
+ if (pred[from] == to && types[from] == type) {
+ /* resolve the cycle with 2 registers using SWAP */
ir_emit_swap_fp(ctx, type, to, from);
IR_REGSET_EXCL(todo, from);
loc[to] = from;
@@ -660,6 +665,7 @@ static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_
#endif
IR_ASSERT(tmp_fp_reg != IR_REG_NONE);
IR_ASSERT(tmp_fp_reg >= IR_REG_FP_FIRST && tmp_fp_reg <= IR_REG_FP_LAST);
+ /* break cycle, moving one of its register into a temporary register */
ir_emit_fp_mov(ctx, type, tmp_fp_reg, to);
loc[to] = tmp_fp_reg;
}
diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h
index 16dc6f536b8..c8292191b5e 100644
--- a/ext/opcache/jit/ir/ir_fold.h
+++ b/ext/opcache/jit/ir/ir_fold.h
@@ -30,7 +30,7 @@ IR_FOLD(EQ(C_DOUBLE, C_DOUBLE))
IR_FOLD(EQ(C_FLOAT, C_FLOAT))
{
- IR_FOLD_BOOL(op1_insn->val.d == op2_insn->val.d);
+ IR_FOLD_BOOL(op1_insn->val.f == op2_insn->val.f);
}
IR_FOLD(NE(C_BOOL, C_BOOL))
@@ -1001,7 +1001,8 @@ IR_FOLD(MUL_OV(C_I64, C_I64))
int64_t res;
IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type);
res = op1_insn->val.u64 * op2_insn->val.u64;
- if ((op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) {
+ if ((op1_insn->val.i64 == -1 && res == INT64_MIN)
+ || (op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) {
IR_FOLD_NEXT;
}
IR_FOLD_CONST_U(res);
@@ -2669,10 +2670,6 @@ IR_FOLD(MUL(NEG, C_I8))
IR_FOLD(MUL(NEG, C_I16))
IR_FOLD(MUL(NEG, C_I32))
IR_FOLD(MUL(NEG, C_I64))
-IR_FOLD(DIV(NEG, C_I8))
-IR_FOLD(DIV(NEG, C_I16))
-IR_FOLD(DIV(NEG, C_I32))
-IR_FOLD(DIV(NEG, C_I64))
{
op1 = op1_insn->op1;
val.u64 = (0 - op2_insn->val.u64);
@@ -2868,13 +2865,22 @@ IR_FOLD(DIV(_, C_FLOAT))
}
IR_FOLD(MUL(NEG, NEG))
-IR_FOLD(DIV(NEG, NEG))
{
op1 = op1_insn->op1;
op2 = op2_insn->op1;
IR_FOLD_RESTART;
}
+IR_FOLD(DIV(NEG, NEG))
+{
+ if (IR_IS_TYPE_FP(IR_OPT_TYPE(opt))) {
+ op1 = op1_insn->op1;
+ op2 = op2_insn->op1;
+ IR_FOLD_RESTART;
+ }
+ IR_FOLD_NEXT;
+}
+
IR_FOLD(AND(_, C_BOOL))
{
IR_FOLD_COPY(op2_insn->val.b ? op1 : op2);
diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c
index 712411774a4..54c894f27cb 100644
--- a/ext/opcache/jit/ir/ir_gcm.c
+++ b/ext/opcache/jit/ir/ir_gcm.c
@@ -100,7 +100,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca)
#endif
flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags;
- if ((flags & IR_BB_LOOP_WITH_ENTRY)
+ if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS))
&& !(ctx->binding && ir_binding_find(ctx, ref))) {
/* Don't move loop invariant code across an OSR ENTRY if we can't restore it */
return lca;
@@ -135,7 +135,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca)
break;
}
flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags;
- if ((flags & IR_BB_LOOP_WITH_ENTRY)
+ if ((flags & (IR_BB_LOOP_WITH_ENTRY|IR_BB_LOOP_WITH_ENTRY_IN_PREDS))
&& !(ctx->binding && ir_binding_find(ctx, ref))) {
break;
}
diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h
index 6fa8efa9cf7..9f91fbfba9f 100644
--- a/ext/opcache/jit/ir/ir_private.h
+++ b/ext/opcache/jit/ir/ir_private.h
@@ -1204,6 +1204,7 @@ void ir_iter_cleanup(ir_ctx *ctx);
#define IR_BB_PREV_EMPTY_ENTRY (1<<8)
#define IR_BB_OSR_ENTRY_LOADS (1<<9) /* OSR Entry-point with register LOADs */
#define IR_BB_LOOP_WITH_ENTRY (1<<10) /* set together with LOOP_HEADER if there is an ENTRY in the loop */
+#define IR_BB_LOOP_WITH_ENTRY_IN_PREDS (1<<18) /* set together with LOOP_HEADER if an ENTRY reaches the loop bypassing its immediate dominator */
/* The following flags are set by GCM */
#define IR_BB_HAS_PHI (1<<11)
diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c
index 53c3cbd8a04..f8d418d0614 100644
--- a/ext/opcache/jit/ir/ir_ra.c
+++ b/ext/opcache/jit/ir/ir_ra.c
@@ -812,6 +812,9 @@ int ir_compute_live_ranges(ir_ctx *ctx)
IR_ASSERT(use_insn->op3 == ref);
/* skip VSTORE (VAR is going to be remapped to PARAM) */
param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
+ if (ctx->use_lists[ref].count != 1) {
+ param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM;
+ }
param_ival->use_pos = param_ival->use_pos->next;
ctx->rules[use] = IR_SKIPPED | IR_NOP;
}
@@ -1530,6 +1533,9 @@ int ir_compute_live_ranges(ir_ctx *ctx)
IR_ASSERT(use_insn->op3 == ref);
/* skip VSTORE (VAR is going to be remapped to PARAM) */
param_ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
+ if (ctx->use_lists[ref].count != 1) {
+ param_ival->flags &= ~IR_LIVE_INTERVAL_MEM_PARAM;
+ }
param_ival->use_pos = param_ival->use_pos->next;
ctx->rules[use] = IR_SKIPPED | IR_NOP;
}
diff --git a/ext/opcache/jit/ir/ir_ra_simple.c b/ext/opcache/jit/ir/ir_ra_simple.c
deleted file mode 100644
index 4ecdd79986b..00000000000
--- a/ext/opcache/jit/ir/ir_ra_simple.c
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * IR - Lightweight JIT Compilation Framework
- * Simple Local Register Allocation (used for -O0)
- * This file is part of the IR Project distributed under the MIT-style LICENSE.
- * Authors: Dmitry Stogov <dmitry@php.net>
- *
- * Simple RA assign registers for each IR instruction separately and always pass
- * values between instrutios in stack spill slots.
- */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE
-#endif
-
-#include "ir.h"
-
-#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
-# include "ir_x86.h"
-#elif defined(IR_TARGET_AARCH64)
-# include "ir_aarch64.h"
-#else
-# error "Unknown IR target"
-#endif
-
-#include "ir_private.h"
-
-typedef struct {
- ir_reg_alloc_data ra_data;
- int32_t num;
- ir_regset preserved_regs;
- ir_regset clobbered[IR_SUB_REFS_COUNT];
- struct {
- uint8_t type;
- int8_t start;
- int8_t end;
- int8_t hint;
- int8_t flags;
- ir_ref root;
- ir_ref ref;
- ir_ref op;
- } regs[32];
-} ir_reg_alloc_simple_data;
-
-static void _add_scratch(ir_reg_alloc_simple_data *data, ir_reg reg, int8_t start, int8_t end)
-{
- int8_t j;
-
- if (start < 0) start = 0; // TODO: ARGVAL support ???
- IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT);
- if (reg >= IR_REG_NUM) {
- for (j = start; j < end; j++) {
- data->clobbered[j] = IR_REGSET_UNION(data->clobbered[j], ir_scratch_regset[reg - IR_REG_NUM]);
- }
- } else {
- for (j = start; j < end; j++) {
- IR_REGSET_INCL(data->clobbered[j], reg);
- }
- }
-}
-
-static void _add_reg(ir_reg_alloc_simple_data *data, ir_type type,
- int8_t start, int8_t end, ir_reg hint, int8_t flags,
- ir_ref root, ir_ref ref, ir_ref op)
-{
- IR_ASSERT(start >= 0 && end <= IR_SUB_REFS_COUNT && data->num < 32);
- data->regs[data->num].type = type;
- data->regs[data->num].start = start;
- data->regs[data->num].end = end;
- data->regs[data->num].hint = hint;
- data->regs[data->num].flags = flags;
- data->regs[data->num].root = root;
- data->regs[data->num].ref = ref;
- data->regs[data->num].op = op;
- data->num++;
-}
-
-static ir_reg _get_free_reg(ir_type type, ir_regset available)
-{
- if (IR_IS_TYPE_INT(type)) {
- available = IR_REGSET_INTERSECTION(available, IR_REGSET_GP);
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
- available = IR_REGSET_INTERSECTION(available, IR_REGSET_FP);
- }
- if (!IR_REGSET_IS_EMPTY(available)) {
- return IR_REGSET_FIRST(available);
- } else {
- return IR_REG_NONE;
- }
-}
-
-static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data *data, int j)
-{
- int n;
- ir_regset available;
- ir_reg reg;
-
- if (IR_IS_TYPE_INT(type)) {
- available = IR_REGSET_GP;
- if (ctx->flags & IR_USE_FRAME_POINTER) {
- IR_REGSET_EXCL(available, IR_REG_FRAME_POINTER);
- }
-
-#if defined(IR_TARGET_X86)
- if (ir_type_size[type] == 1) {
- /* TODO: if no registers avialivle, we may use of one this register for already allocated interval ??? */
- IR_REGSET_EXCL(available, IR_REG_RBP);
- IR_REGSET_EXCL(available, IR_REG_RSI);
- IR_REGSET_EXCL(available, IR_REG_RDI);
- }
-#endif
- } else {
- IR_ASSERT(IR_IS_TYPE_FP(type) || IR_IS_TYPE_VECTOR(type));
- available = IR_REGSET_FP;
- }
- for (n = data->regs[j].start; n < data->regs[j].end; n++) {
- available = IR_REGSET_DIFFERENCE(available, data->clobbered[n]);
- }
- if (IR_REGSET_IS_EMPTY(available)) {
- fprintf(stderr, "Internal Error: No registers available. Allocation is not possible\n");
- IR_ASSERT(0);
- exit(-1);
- }
-
- reg = IR_REGSET_FIRST(available);
- if (IR_REGSET_IN(data->preserved_regs, reg)) {
- ir_regset tmp = (ir_regset)ctx->used_preserved_regs;
- IR_REGSET_INCL(tmp, reg);
- ctx->used_preserved_regs = tmp;
- }
- return reg;
-}
-
-static bool ir_load_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref load)
-{
- ir_use_list *use_list = &ctx->use_lists[load];
- ir_ref *p, use, i, n = use_list->count;
- ir_ref last_use = IR_UNUSED;
- ir_insn *insn;
- int address_taken = -1; /* unknown */
-
- if (n) {
- /* Check if all uses of the result of VLOAD are in the same basic block (after VLOAD) */
- for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
- use = *p;
- if (use < load || use > bb->end) return 0;
- if (use > last_use) last_use = use;
- }
- /* Check if the result of VLOAD can not be clobbered between VLOAD and the last use */
- for (i = load + 1, insn = &ctx->ir_base[i]; i < last_use;) {
- if ((insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) && insn->op2 == var) {
- return 0;
- } else if (address_taken
- && (insn->op == IR_STORE
- || insn->op == IR_STORE_v
- || insn->op == IR_CALL)) {
- if (address_taken == -1) {
- ir_use_list *use_list = &ctx->use_lists[var];
- ir_ref *p, n = use_list->count;
-
- for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
- if (ctx->ir_base[*p].op == IR_VADDR) {
- return 0;
- }
- }
- address_taken = 0;
- }
- }
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- }
- }
- return 1;
-}
-
-static bool ir_store_may_reuse_var_slot(ir_ctx *ctx, ir_block *bb, ir_ref var, ir_ref store, ir_ref val)
-{
- ir_ref i, n;
- ir_insn *insn;
- int address_taken = -1; /* unknown */
-
- /* Check if the value stored by VSTORE is produced in the same basic block (before VSTORE) */
- if (val < bb->start || val > store || store > bb->end) return 0;
-
- /* Check if the instructions between the "producer" and VSTORE can not access the same VAR */
- for (i = val, insn = &ctx->ir_base[i]; i < store;) {
- if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v)
- && insn->op2 == var) {
- return 0;
- } else if (address_taken
- && (insn->op == IR_LOAD
- || insn->op == IR_LOAD_v
- || insn->op == IR_STORE
- || insn->op == IR_STORE_v
- || insn->op == IR_CALL)) {
- if (address_taken == -1) {
- ir_use_list *use_list = &ctx->use_lists[var];
- ir_ref *p, n = use_list->count;
-
- for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) {
- if (ctx->ir_base[*p].op == IR_VADDR) {
- return 0;
- }
- }
- address_taken = 0;
- }
- }
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- }
- return 1;
-}
-
-static void ir_add_fusion_data(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_reg_alloc_simple_data *data)
-{
- ir_ref stack[4];
- int stack_pos = 0;
- ir_target_constraints constraints;
- ir_insn *insn;
- uint32_t j, n, flags, def_flags;
- ir_ref *p, child;
-
- while (1) {
- IR_ASSERT(input > 0 && ctx->rules[input] & IR_FUSED);
-
- if (!(ctx->rules[input] & IR_SIMPLE)) {
- def_flags = ir_get_target_constraints(ctx, input, &constraints);
- n = constraints.tmps_count;
- while (n > 0) {
- n--;
- if (constraints.tmp_regs[n].type) {
- ir_reg flags = 0;
- ir_ref op = constraints.tmp_regs[n].num;
-
- if (op > 0 && op <= ctx->ir_base[input].inputs_count) {
- ir_ref *ops = ctx->ir_base[input].ops;
-
- if (IR_IS_CONST_REF(ops[op])) {
- /* rematerialization */
- flags = IR_REG_SPILL_LOAD;
- } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
- /* local address rematerialization */
- flags = IR_REG_SPILL_LOAD;
- }
- }
- _add_reg(data, constraints.tmp_regs[n].type,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
- IR_UNUSED, input, op);
- } else {
- _add_scratch(data, constraints.tmp_regs[n].reg,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
- }
- }
- } else {
- def_flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG;
- constraints.hints_count = 0;
- }
-
- insn = &ctx->ir_base[input];
- flags = ir_op_flags[insn->op];
- n = IR_INPUT_EDGES_COUNT(flags);
- j = 1;
- p = insn->ops + j;
- if (flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_PINNED)) {
- j++;
- p++;
- }
- for (; j <= n; j++, p++) {
- IR_ASSERT(IR_OPND_KIND(flags, j) == IR_OPND_DATA);
- child = *p;
- if (child > 0) {
- if (ctx->vregs[child] > 0) {
- if (IR_USE_FLAGS(def_flags, j) & IR_USE_MUST_BE_IN_REG) {
- ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
- int8_t use_pos = EXPECTED(reg == IR_REG_NONE) ? IR_USE_SUB_REF : IR_LOAD_SUB_REF;
-
- _add_reg(data, ctx->ir_base[child].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
- ref, input, j);
- }
- } else if (ctx->rules[child] & IR_FUSED) {
- IR_ASSERT(stack_pos < (int)(sizeof(stack)/sizeof(stack_pos)));
- stack[stack_pos++] = child;
- } else if (ctx->rules[child] == (IR_SKIPPED|IR_RLOAD)) {
- ctx->regs[input][j] = ctx->ir_base[child].op2;
- }
- }
- }
- if (!stack_pos) {
- break;
- }
- input = stack[--stack_pos];
- }
-}
-
-static void ir_allocate_var_spill_slot(ir_ctx *ctx, ir_ref var, ir_insn *var_insn)
-{
- size_t size = ir_get_type_size(var_insn->type);
- size_t align = var_insn->op3;
- int32_t offset;
-
-#if IR_SIMD
- if (IR_IS_TYPE_VECTOR(var_insn->type) && size < 4) {
- size = 4;
- align = IR_MAX(align, 4);
- }
-#endif
-
- offset = ir_allocate_spill_slot(ctx, size, align);
- ctx->vregs[var] = IR_STACK_SLOT_TO_VREG(offset);
-}
-
-int ir_reg_alloc_simple(ir_ctx *ctx)
-{
- uint32_t b;
- ir_block *bb;
- ir_insn *insn;
- ir_ref i, n, j, *p;
- uint32_t *rule, insn_flags;
- ir_target_constraints constraints;
- uint32_t def_flags;
- ir_reg reg;
- ir_regset scratch;
- ir_reg_alloc_simple_data data;
- int32_t offset;
-
- ir_compute_dessa_moves(ctx);
-
- memset(&data.ra_data, 0, sizeof(data.ra_data));
- data.ra_data.cc = ir_get_call_conv_dsc(ctx->flags);
- ctx->data = &data.ra_data;
-
- ctx->stack_frame_size = 0;
- ctx->call_stack_size = 0;
- ctx->used_preserved_regs = 0;
- ctx->used_preserved_regs = ctx->fixed_save_regset;
-
- scratch = ir_scratch_regset[data.ra_data.cc->scratch_reg - IR_REG_NUM];
- data.preserved_regs = IR_REGSET_DIFFERENCE(data.ra_data.cc->preserved_regs, ctx->fixed_save_regset);
-
- ctx->regs = ir_mem_malloc(sizeof(ir_regs) * ctx->insns_count);
- memset(ctx->regs, IR_REG_NONE, sizeof(ir_regs) * ctx->insns_count);
-
- /* vregs + tmp + fixed + SRATCH + ALL */
- ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_SET_NUM, sizeof(ir_live_interval*));
-
- if (!ctx->arena) {
- ctx->arena = ir_arena_create(16 * 1024);
- }
-
- for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) {
- IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE));
- for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
- if (*rule & (IR_FUSED|IR_SKIPPED)) {
- if ((*rule & IR_RULE_MASK) == IR_ALLOCA) {
- if (IR_VREG_IS_STACK_SLOT(ctx->vregs[i])) {
- /* spill slot already allocated */
- } else if (insn->op == IR_VAR) {
- if (ctx->use_lists[i].count > 0) {
- ir_allocate_var_spill_slot(ctx, i, insn);
- }
- } else if (insn->op == IR_ALLOCA) {
- if (ctx->use_lists[i].count > 0) {
- ir_insn *val = &ctx->ir_base[insn->op2];
-
- IR_ASSERT(IR_IS_CONST_REF(insn->op2));
- IR_ASSERT(IR_IS_TYPE_INT(val->type));
- IR_ASSERT(!IR_IS_SYM_CONST(val->op));
- IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0);
- IR_ASSERT(val->val.i64 < 0x7fffffff);
- offset = ir_allocate_spill_slot(ctx, val->val.i32, insn->op3);
- ctx->vregs[i] = IR_STACK_SLOT_TO_VREG(offset);
- }
- } else if (insn->op == IR_VADDR) {
- if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op1])) {
- ir_allocate_var_spill_slot(ctx, insn->op1, &ctx->ir_base[insn->op1]);
- }
- ctx->vregs[i] = ctx->vregs[insn->op1];
- }
- }
- } else {
- data.num = 0;
- for (j = 0; j < IR_SUB_REFS_COUNT; j++) {
- data.clobbered[j] = IR_REGSET_EMPTY;
- }
-
- def_flags = ir_get_target_constraints(ctx, i, &constraints);
- n = constraints.tmps_count;
- while (n) {
- n--;
-
- IR_ASSERT(constraints.tmp_regs[n].start >= 0 && constraints.tmp_regs[n].end < IR_SUB_REFS_COUNT);
- if (constraints.tmp_regs[n].type) {
- ir_reg flags = 0;
- ir_ref op = constraints.tmp_regs[n].num;
-
- if (op > 0 && op <= insn->inputs_count) {
- ir_ref *ops = insn->ops;
-
- if (IR_IS_CONST_REF(ops[op])) {
- /* rematerialization */
- flags = IR_REG_SPILL_LOAD;
- } else if (ctx->rules[ops[op]] == IR_STATIC_ALLOCA) {
- /* local address rematerialization */
- flags = IR_REG_SPILL_LOAD;
- }
- }
- _add_reg(&data, constraints.tmp_regs[n].type,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end, IR_REG_NONE, flags,
- IR_UNUSED, i, op);
- } else {
- _add_scratch(&data, constraints.tmp_regs[n].reg,
- constraints.tmp_regs[n].start, constraints.tmp_regs[n].end);
- }
- }
-
- if (ctx->vregs[i] > 0) {
- reg = constraints.def_reg;
- if (!ctx->live_intervals[ctx->vregs[i]]) {
- do {
- ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
- memset(ival, 0, sizeof(ir_live_interval));
- ctx->live_intervals[ctx->vregs[i]] = ival;
- ival->type = insn->type;
- ival->reg = IR_REG_NONE;
- ival->vreg = ctx->vregs[i];
- ival->stack_spill_pos = -1;
-
- if (insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) {
- if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) {
- ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]);
- }
- if (ir_load_may_reuse_var_slot(ctx, bb, insn->op2, i)) {
- ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[insn->op2]);
- reg = IR_REG_NONE;
- def_flags &= ~IR_USE_MUST_BE_IN_REG;
- break;
- }
- } else if (insn->op == IR_PARAM && reg == IR_REG_NONE) {
- /* We may reuse parameter stack slot for spilling */
- ival->flags |= IR_LIVE_INTERVAL_MEM_PARAM;
- if (ctx->use_lists[i].count > 0) {
- ir_ref use = ctx->use_edges[ctx->use_lists[i].refs];
- if (use < bb->end) {
- ir_insn *use_insn = &ctx->ir_base[use];
- if (use_insn->op == IR_VSTORE
- && !(ctx->rules[use] & IR_SKIPPED)
- /* first use of VAR is VSTORE(_, VAR, PARAM) */
- && ctx->use_edges[ctx->use_lists[use_insn->op2].refs] == use) {
- IR_ASSERT(use_insn->op3 == i);
- /* skip VSTORE (VAR is going to be remapped to PARAM) */
- ival->flags |= IR_LIVE_INTERVAL_VAR_MEM_PARAM;
- ival->use_pos = NULL;
- ctx->rules[use] = IR_SKIPPED | IR_NOP;
- }
- }
- }
- break;
- }
-
- if (ctx->use_lists[i].count == 1) {
- ir_ref use = ctx->use_edges[ctx->use_lists[i].refs];
- ir_insn *use_insn = &ctx->ir_base[use];
-
- if ((use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v)
- && use_insn->op3 == i
- && ir_store_may_reuse_var_slot(ctx, bb, use_insn->op2, use, i)) {
- if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[use_insn->op2])) {
- ir_allocate_var_spill_slot(ctx, use_insn->op2, &ctx->ir_base[use_insn->op2]);
- }
- ival->stack_spill_pos = IR_VREG_TO_STACK_SLOT(ctx->vregs[use_insn->op2]);
- break;
- }
- }
-
- size_t size = ir_get_type_size(ival->type);
-#if IR_SIMD
- if (IR_IS_TYPE_VECTOR(ival->type) && size < 4) {
- size = 4;
- }
-#endif
- ival->stack_spill_pos = ir_allocate_spill_slot(ctx, size, 0);
- } while (0);
- }
-
- if (def_flags & IR_USE_MUST_BE_IN_REG) {
- ir_live_pos def_pos;
-
- if (reg != IR_REG_NONE) {
- def_pos = IR_SAVE_SUB_REF;
- } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- def_pos = IR_USE_SUB_REF;
- } else {
- def_pos = IR_LOAD_SUB_REF;
- }
- } else if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- def_pos = IR_LOAD_SUB_REF;
- } else {
- def_pos = IR_DEF_SUB_REF;
- }
-
- _add_reg(&data, insn->type, def_pos, IR_SUB_REFS_COUNT, reg, IR_REG_SPILL_STORE,
- IR_UNUSED, i, 0);
- }
- } else if (insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) {
- if (!IR_VREG_IS_STACK_SLOT(ctx->vregs[insn->op2])) {
- ir_allocate_var_spill_slot(ctx, insn->op2, &ctx->ir_base[insn->op2]);
- }
- }
-
- n = insn->inputs_count;
- insn_flags = ir_op_flags[insn->op];
- j = 1;
- p = insn->ops + 1;
- if (insn_flags & (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_PINNED)) {
- j++;
- p++;
- }
- for (; j <= n; j++, p++) {
- ir_ref input = *p;
- ir_reg reg = (j < constraints.hints_count) ? constraints.hints[j] : IR_REG_NONE;
- ir_live_pos use_pos;
- uint32_t use_flags = IR_USE_FLAGS(def_flags, j);
-
- if (reg != IR_REG_NONE) {
-#if IR_X86_I64
- if (use_flags & IR_HINT_TWO_REGS) {
- IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_LO(reg));
- IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], IR_REG_I64_HI(reg));
- } else
-#endif
- IR_REGSET_INCL(data.clobbered[IR_LOAD_SUB_REF], reg);
- }
-
- if (input > 0) {
- if (ctx->vregs[input] > 0) {
- use_pos = IR_USE_SUB_REF;
- if (reg != IR_REG_NONE) {
- use_pos = IR_LOAD_SUB_REF;
- } else if (def_flags & IR_DEF_REUSES_OP1_REG) {
- if (j == 1) {
- if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
- use_pos = IR_USE_SUB_REF;
- } else {
- use_pos = IR_LOAD_SUB_REF;
- }
- } else if (input == insn->op1) {
- /* Input is the same as "op1" */
- use_pos = IR_LOAD_SUB_REF;
- }
- }
- if (use_flags & IR_USE_MUST_BE_IN_REG) {
- _add_reg(&data, ctx->ir_base[input].type, IR_LOAD_SUB_REF, use_pos, reg, IR_REG_SPILL_LOAD,
- IR_UNUSED, i, j);
- }
- } else {
- if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) {
- ir_add_fusion_data(ctx, i, input, &data);
- } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) {
- ctx->regs[i][j] = ctx->ir_base[input].op2;
- }
- }
- }
- }
-
- for (j = 0; j < data.num; j++) {
- ir_regset available = scratch;
-#if IR_X86_I64
- ir_reg reg2;
-#endif
-
- for (n = data.regs[j].start; n < data.regs[j].end; n++) {
- available = IR_REGSET_DIFFERENCE(available, data.clobbered[n]);
- }
- reg = data.regs[j].hint;
-#if IR_X86_I64
- reg2 = IR_REG_NONE;
- if (reg != IR_REG_NONE && (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64)) {
- reg2 = IR_REG_I64_HI(reg);
- reg = IR_REG_I64_LO(reg);
- }
-#endif
- if (reg == IR_REG_NONE || !IR_REGSET_IN(available, reg)) {
- reg = _get_free_reg(data.regs[j].type, available);
- if (UNEXPECTED(reg == IR_REG_NONE)) {
- reg = _get_free_reg2(ctx, data.regs[j].type, &data, j);
- }
- }
- for (n = data.regs[j].start; n < data.regs[j].end; n++) {
- IR_REGSET_INCL(data.clobbered[n], reg);
- }
-#if IR_X86_I64
- if (data.regs[j].type == IR_I64 || data.regs[j].type == IR_U64) {
- IR_REGSET_EXCL(available, reg);
- if (reg2 == IR_REG_NONE || !IR_REGSET_IN(available, reg2)) {
- reg2 = _get_free_reg(data.regs[j].type, available);
- if (UNEXPECTED(reg2 == IR_REG_NONE)) {
- reg2 = _get_free_reg2(ctx, data.regs[j].type, &data, j);
- }
- }
- for (n = data.regs[j].start; n < data.regs[j].end; n++) {
- IR_REGSET_INCL(data.clobbered[n], reg2);
- }
- if (reg > reg2) {
- SWAP_REGS(reg, reg2);
- }
- reg = IR_REG_I64_PAIR(reg, reg2);
- }
-#endif
- reg = reg | data.regs[j].flags;
- if (data.regs[j].op == 4 && insn->inputs_count < 4) {
- if (!ctx->tmp_regs) {
- ctx->tmp_regs = ir_mem_malloc(ctx->insns_count);
- memset(ctx->tmp_regs, -1, ctx->insns_count);
- }
- ctx->tmp_regs[data.regs[j].ref] = reg;
- } else if (!data.regs[j].root || ctx->regs[data.regs[j].ref][data.regs[j].op] == IR_REG_NONE) {
- ctx->regs[data.regs[j].ref][data.regs[j].op] = reg;
- } else if (ctx->regs[data.regs[j].ref][data.regs[j].op] != reg) {
- ctx->rules[data.regs[j].ref] |= IR_FUSED_REG;
- ir_set_fused_reg(ctx, data.regs[j].root, data.regs[j].ref, data.regs[j].op, reg);
- }
- }
- }
-
- n = ir_insn_len(insn);
- i += n;
- insn += n;
- rule += n;
- }
- if (bb->flags & IR_BB_DESSA_MOVES) {
-#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
- ctx->regs[bb->end][0] = IR_REG_RAX;
- ctx->regs[bb->end][1] = IR_REG_XMM0;
-#elif defined(IR_TARGET_AARCH64)
- ctx->regs[bb->end][0] = IR_REG_X0;
- ctx->regs[bb->end][1] = IR_REG_V0;
-#else
-# error "Unknown IR target"
-#endif
- }
- }
-
-#ifdef IR_TARGET_X86
- if (ctx->flags2 & IR_HAS_FP_RET_SLOT) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[IR_DOUBLE], 0);
- } else if ((ctx->ret_type == IR_FLOAT || ctx->ret_type == IR_DOUBLE)
- && data.ra_data.cc->fp_ret_reg == IR_REG_NONE) {
- ctx->ret_slot = ir_allocate_spill_slot(ctx, ir_type_size[ctx->ret_type], 0);
- } else {
- ctx->ret_slot = -1;
- }
-#endif
-
- ctx->flags |= IR_NO_STACK_COMBINE;
- ir_fix_stack_frame(ctx);
- ctx->data = NULL;
-
- return 1;
-}
diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c
index 3ad6997b071..2ca202add5e 100644
--- a/ext/opcache/jit/ir/ir_sccp.c
+++ b/ext/opcache/jit/ir/ir_sccp.c
@@ -2673,16 +2673,16 @@ static bool ir_is_zero(const ir_ctx *ctx, ir_ref ref)
&& ctx->ir_base[ref].val.u32 == 0;
}
-static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref)
+static bool ir_is_min_max_const(const ir_ctx *ctx, const ir_insn *cond, ir_ref ref)
{
if (cond->op == IR_ULE) {
/* (x <= 3 ? 4 : x) => (x < 4 ? 4 : x) => max(x, 4) */
/* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */
if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)
&& !IR_IS_SYM_CONST(ctx->ir_base[ref].op)
+ && ctx->ir_base[ref].val.u64 != 0
&& ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1
&& ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) {
- cond->op2 = ref;
return 1;
}
} else if (cond->op == IR_UGE) {
@@ -2690,9 +2690,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref)
/* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */
if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)
&& !IR_IS_SYM_CONST(ctx->ir_base[ref].op)
+ && ctx->ir_base[ref].val.u64 != UINT64_MAX
&& ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 + 1
&& ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) {
- cond->op2 = ref;
return 1;
}
} else if (cond->op == IR_LE) {
@@ -2700,9 +2700,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref)
/* (x <= 3 ? x : 4) => (x < 4 ? x : 4) => min(x, 4) */
if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)
&& !IR_IS_SYM_CONST(ctx->ir_base[ref].op)
- && ctx->ir_base[cond->op2].val.u64 == ctx->ir_base[ref].val.u64 - 1
+ && ctx->ir_base[ref].val.i64 != INT64_MIN
+ && ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 - 1
&& ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) {
- cond->op2 = ref;
return 1;
}
} else if (cond->op == IR_GE) {
@@ -2710,9 +2710,9 @@ static bool ir_fix_min_max_const(ir_ctx *ctx, ir_insn *cond, ir_ref ref)
/* (x >= 3 ? x : 2) => (x > 2 ? x : 2) => max(x, 2) */
if (!IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)
&& !IR_IS_SYM_CONST(ctx->ir_base[ref].op)
+ && ctx->ir_base[ref].val.i64 != INT64_MAX
&& ctx->ir_base[cond->op2].val.i64 == ctx->ir_base[ref].val.i64 + 1
&& ctx->ir_base[cond->op2].type == ctx->ir_base[ref].type) {
- cond->op2 = ref;
return 1;
}
}
@@ -2758,18 +2758,19 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
}
if (is_cmp
+ && !IR_IS_TYPE_FP(type)
&& ((insn->op2 == cond->op1
&& (insn->op3 == cond->op2
|| (IR_IS_CONST_REF(cond->op2)
&& (IR_IS_CONST_REF(insn->op3)
&& IR_IS_TYPE_INT(insn->type)
- && ir_fix_min_max_const(ctx, cond, insn->op3)))))
+ && ir_is_min_max_const(ctx, cond, insn->op3)))))
|| (insn->op3 == cond->op1
&& (insn->op2 == cond->op2
|| (IR_IS_CONST_REF(cond->op2)
&& (IR_IS_CONST_REF(insn->op2)
&& IR_IS_TYPE_INT(insn->type)
- && ir_fix_min_max_const(ctx, cond, insn->op2))))))) {
+ && ir_is_min_max_const(ctx, cond, insn->op2))))))) {
/* MAX/MIN
*
* prev prev
@@ -2804,11 +2805,16 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
ir_remove_unused_vars(ctx, start2_ref, end2_ref);
}
- insn->op = (
- (is_less ? cond->op1 : cond->op2)
- ==
- ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3)
- ) ? IR_MIN : IR_MAX;
+ if (start1->op != IR_IF_TRUE) {
+ IR_ASSERT(start1->op == IR_IF_FALSE);
+ is_less = !is_less;
+ }
+ if (cond->op1 != insn->op2) {
+ IR_ASSERT(cond->op1 == insn->op3);
+ is_less = !is_less;
+ }
+
+ insn->op = is_less ? IR_MIN : IR_MAX;
insn->inputs_count = 2;
if (insn->op2 > insn->op3) {
insn->op1 = insn->op2;
@@ -2939,7 +2945,12 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
}
return 1;
- } else if (insn->op2 <= root_ref && insn->op3 <= root_ref
+ } else if (((insn->op2 <= root_ref && insn->op3 <= root_ref)
+ || ((cond->op >= IR_EQ && cond->op <= IR_UNORDERED)
+ && ((insn->op2 == cond->op1 && (insn->op3 == cond->op2 || insn->op3 <= root_ref))
+ || (insn->op2 == cond->op2 && (insn->op3 == cond->op1 || insn->op3 <= root_ref))
+ || (insn->op3 == cond->op1 && insn->op2 <= root_ref)
+ || (insn->op3 == cond->op2 && insn->op2 <= root_ref))))
&& cond->op != IR_OVERFLOW
// TODO: temporary disable IF-conversion for RLOAD.
// We don't track anti-dependencies in GCM and Local Scheduling.
diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc
index f33c89f574f..463be9e750d 100644
--- a/ext/opcache/jit/ir/ir_x86.dasc
+++ b/ext/opcache/jit/ir/ir_x86.dasc
@@ -1622,6 +1622,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = {
_(BINOP_INT) \
_(BINOP_SSE2) \
_(BINOP_AVX) \
+ _(MIN_MAX_FP) \
_(SHIFT) \
_(SHIFT_CONST) \
_(COPY_INT) \
@@ -1636,6 +1637,7 @@ const ir_call_conv_dsc ir_call_conv_x86_fastcall = {
_(COND_TEST_BIT) \
_(COND_CMP_INT) \
_(COND_CMP_FP) \
+ _(COND_MIN_MAX_FP) \
_(GUARD_CMP_INT) \
_(GUARD_CMP_FP) \
_(GUARD_TEST_INT) \
@@ -1998,6 +2000,28 @@ op2_const:
case IR_COND_CMP_FP:
flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG;
break;
+ case IR_COND_MIN_MAX_FP:
+ flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG;
+ if (!(ctx->mflags & IR_X86_AVX)) {
+ insn = &ctx->ir_base[ref];
+ if (!IR_IS_CONST_REF(insn->op3) || ctx->vregs[insn->op3] > 0) {
+ /* def_reg can't be the same as op3_reg */
+ flags |= IR_DEF_CONFLICTS_WITH_INPUT_REGS;
+ }
+ }
+ break;
+ case IR_MIN_MAX_FP:
+ flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS;
+ insn = &ctx->ir_base[ref];
+ constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
+ n = 1;
+ if (IR_IS_CONST_REF(insn->op1)) {
+ n = ir_add_const_tmp_reg(ctx, insn->op1, 1, n, constraints);
+ }
+ if (IR_IS_CONST_REF(insn->op2)) {
+ n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
+ }
+ break;
case IR_VSTORE_INT:
flags = IR_OP3_MUST_BE_IN_REG;
insn = &ctx->ir_base[ref];
@@ -4496,7 +4520,7 @@ binop_fp:
goto binop_vector;
#endif
} else {
- goto binop_fp;
+ return IR_MIN_MAX_FP;
}
break;
case IR_COPY:
@@ -5047,6 +5071,11 @@ store_int:
return IR_TWO_REGS | IR_COND_I64_CMP_FP;
}
#endif
+ if ((op1_insn->op >= IR_LT && op1_insn->op <= IR_GT) &&
+ ((op1_insn->op1 == insn->op2 && op1_insn->op2 == insn->op3) ||
+ (op1_insn->op1 == insn->op3 && op1_insn->op2 == insn->op2))) {
+ return IR_COND_MIN_MAX_FP;
+ }
return IR_COND_CMP_FP;
}
#if IR_X86_I64
@@ -7543,6 +7572,30 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
| add Rd(def_reg), Rd(op1_reg)
|.if X64
|| }
+|.endif
+ | ASM_REG_IMM_OP sar, type, def_reg, shift
+ } else if (shift == (uint32_t)ir_type_size[type] * 8 - 1) {
+|.if X64
+|| if (ir_type_size[type] == 8) {
+ | mov64 Rq(def_reg), 0x8000000000000000
+ | cmp Rq(def_reg), Rq(op1_reg)
+ | sete Rb(def_reg)
+ | movzx Rd(def_reg), Rb(def_reg)
+|| } else {
+|.endif
+ | xor Rd(def_reg), Rd(def_reg)
+ | cmp Rd(op1_reg), 0x80000000
+|.if X64
+ | sete Rb(def_reg)
+|| }
+|.else
+ if (def_reg < IR_REG_R3) {
+ | sete Rb(def_reg)
+ } else {
+ | jne >1
+ | mov Rd(def_reg), 1
+ |1:
+ }
|.endif
} else {
|.if X64
@@ -7564,8 +7617,8 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
|.endif
| ASM_REG_REG_OP test, type, op1_reg, op1_reg
| ASM_REG_REG_OP2 cmovns, type, def_reg, op1_reg
+ | ASM_REG_IMM_OP sar, type, def_reg, shift
}
- | ASM_REG_IMM_OP sar, type, def_reg, shift
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, type, def, def_reg);
@@ -8614,12 +8667,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
case IR_DIV:
| ASM_SSE2_REG_REG_OP divs, type, def_reg, op2_reg
break;
- case IR_MIN:
- | ASM_SSE2_REG_REG_OP mins, type, def_reg, op2_reg
- break;
- case IR_MAX:
- | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op2_reg
- break;
}
} else if (IR_IS_CONST_REF(op2)) {
int label = ir_get_const_label(ctx, op2);
@@ -8639,12 +8686,6 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
case IR_DIV:
| ASM_SSE2_REG_TXT_OP divs, type, def_reg, [=>label]
break;
- case IR_MIN:
- | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label]
- break;
- case IR_MAX:
- | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label]
- break;
}
} else {
ir_mem mem;
@@ -8669,14 +8710,88 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
case IR_DIV:
| ASM_SSE2_REG_MEM_OP divs, type, def_reg, mem
break;
- case IR_MIN:
- | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem
- break;
- case IR_MAX:
- | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem
- break;
}
}
+
+ if (IR_REG_SPILLED(ctx->regs[def][0])) {
+ ir_emit_store(ctx, insn->type, def, def_reg);
+ }
+}
+
+static void ir_emit_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_type type = insn->type;
+ ir_ref op1 = insn->op1;
+ ir_ref op2 = insn->op2;
+ ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
+ ir_reg op1_reg = ctx->regs[def][1];
+ ir_reg op2_reg = ctx->regs[def][2];
+ ir_reg tmp_reg = ctx->regs[def][3];
+
+ IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE);
+
+ if (IR_REG_SPILLED(op1_reg)) {
+ op1_reg = IR_REG_NUM(op1_reg);
+ ir_emit_load(ctx, type, op1_reg, op1);
+ }
+ if (IR_REG_SPILLED(op2_reg)) {
+ op2_reg = IR_REG_NUM(op2_reg);
+ if (op1 != op2) {
+ ir_emit_load(ctx, type, op2_reg, op2);
+ }
+ }
+
+ if (!(ctx->mflags & IR_X86_AVX)) {
+ ir_emit_fp_mov(ctx, type, def_reg, op1_reg);
+ }
+
+ if (type == IR_DOUBLE) {
+ if (ctx->mflags & IR_X86_AVX) {
+ if (insn->op == IR_MAX) {
+ | vmaxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ } else {
+ | vminsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ }
+ | vcmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7
+ | vblendvpd xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ } else {
+ if (insn->op == IR_MAX) {
+ | maxsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ } else {
+ | minsd xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ }
+ | movapd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ | cmpsd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7
+ | andpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ | andnpd xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST)
+ | orpd xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ }
+ } else {
+ IR_ASSERT(type == IR_FLOAT);
+ if (ctx->mflags & IR_X86_AVX) {
+ if (insn->op == IR_MAX) {
+ | vmaxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ } else {
+ | vminss xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ }
+ | vcmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST), 7
+ | vblendvps xmm(def_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ } else {
+ if (insn->op == IR_MAX) {
+ | maxss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ } else {
+ | minss xmm(def_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ }
+ | movaps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op2_reg-IR_REG_FP_FIRST)
+ | cmpss xmm(tmp_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST), 7
+ | andps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ | andnps xmm(tmp_reg-IR_REG_FP_FIRST), xmm(op1_reg-IR_REG_FP_FIRST)
+ | orps xmm(def_reg-IR_REG_FP_FIRST), xmm(tmp_reg-IR_REG_FP_FIRST)
+ }
+ }
+
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
@@ -9469,6 +9584,20 @@ static void ir_emit_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint
}
}
+static bool ir_insns_may_change_cmp_flags(ir_ctx *ctx, ir_ref from, ir_ref to)
+{
+ ir_ref ref;
+
+ for (ref = ctx->prev_ref[to]; ref != from; ref = ctx->prev_ref[ref]) {
+ if (!(ctx->rules[ref] & (IR_FUSED|IR_SKIPPED))
+ && ctx->ir_base[ref].op != IR_VAR
+ && ctx->ir_base[ref].op != IR_PHI) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block)
{
ir_insn *cmp_insn = &ctx->ir_base[insn->op2];
@@ -9518,7 +9647,9 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) {
prev_insn = &ctx->ir_base[prev_insn->op1];
prev_insn = &ctx->ir_base[prev_insn->op2];
- if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) {
+ if (prev_insn->op1 == cmp_insn->op1
+ && prev_insn->op2 == cmp_insn->op2
+ && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) {
same_comparison = true;
}
}
@@ -9666,7 +9797,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn)
op3_reg = IR_REG_NUM(op3_reg);
ir_emit_load(ctx, type, op3_reg, op3);
if (op1 == op3) {
- op1_reg = op2_reg;
+ op1_reg = op3_reg;
}
}
if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) {
@@ -10295,6 +10426,137 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
}
}
+static void ir_emit_cond_min_max_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
+{
+ ir_backend_data *data = ctx->data;
+ dasm_State **Dst = &data->dasm_state;
+ ir_type type = insn->type;
+ ir_insn *cmp_insn = &ctx->ir_base[insn->op1];
+ ir_ref op2 = insn->op2;
+ ir_ref op3 = insn->op3;
+ ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]);
+ ir_reg op2_reg = ctx->regs[def][2];
+ ir_reg op3_reg = ctx->regs[def][3];
+ ir_reg cmp_op1_reg, cmp_op2_reg;
+ ir_op op;
+
+ IR_ASSERT(def_reg != IR_REG_NONE);
+
+ if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) {
+ cmp_op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1);
+ cmp_op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2);
+ } else {
+ cmp_op1_reg = ctx->regs[insn->op1][1];
+ cmp_op2_reg = ctx->regs[insn->op1][2];
+ }
+ if (cmp_insn->op1 == op2) {
+ IR_ASSERT(cmp_insn->op2 == op3);
+ op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MIN : IR_MAX;
+ if (op2_reg != IR_REG_NONE) {
+ IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op2_reg));
+ op2_reg |= cmp_op1_reg; /* merge spill load flag */
+ }
+ if (op3_reg != IR_REG_NONE) {
+ IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op3_reg));
+ op3_reg |= cmp_op2_reg; /* merge spill load flag */
+ }
+ } else {
+ IR_ASSERT(cmp_insn->op1 == op3 && cmp_insn->op2 == op2);
+ op = (cmp_insn->op == IR_LT || cmp_insn->op == IR_LE) ? IR_MAX : IR_MIN;
+ if (op2_reg != IR_REG_NONE) {
+ IR_ASSERT(IR_REG_NUM(cmp_op2_reg) == IR_REG_NUM(op2_reg));
+ op2_reg |= cmp_op2_reg; /* merge spill load flag */
+ }
+ if (op3_reg != IR_REG_NONE) {
+ IR_ASSERT(IR_REG_NUM(cmp_op1_reg) == IR_REG_NUM(op3_reg));
+ op3_reg |= cmp_op1_reg; /* merge spill load flag */
+ }
+ }
+
+ if (op2_reg != IR_REG_NONE) {
+ if (IR_REG_SPILLED(op2_reg)) {
+ op2_reg = IR_REG_NUM(op2_reg);
+ ir_emit_load(ctx, type, op2_reg, op2);
+ }
+ }
+ if (op3_reg != IR_REG_NONE) {
+ if (IR_REG_SPILLED(op3_reg)) {
+ op3_reg = IR_REG_NUM(op3_reg);
+ if (op2 != op3) {
+ ir_emit_load(ctx, type, op3_reg, op3);
+ }
+ }
+ }
+
+ IR_ASSERT((ctx->mflags & IR_X86_AVX) || def_reg != op3_reg);
+
+ if (op2_reg != IR_REG_NONE) {
+ if (!(ctx->mflags & IR_X86_AVX) && def_reg != op2_reg) {
+ ir_emit_fp_mov(ctx, type, def_reg, op2_reg);
+ }
+ } else {
+ ir_emit_load(ctx, type, def_reg, op2);
+ op2_reg = def_reg;
+ }
+
+ if (op3_reg != IR_REG_NONE) {
+ if (ctx->mflags & IR_X86_AVX) {
+ if (op == IR_MIN) {
+ | ASM_AVX_REG_REG_REG_OP vmins, type, def_reg, op2_reg, op3_reg
+ } else {
+ | ASM_AVX_REG_REG_REG_OP vmaxs, type, def_reg, op2_reg, op3_reg
+ }
+ } else {
+ if (op == IR_MIN) {
+ | ASM_SSE2_REG_REG_OP mins, type, def_reg, op3_reg
+ } else {
+ | ASM_SSE2_REG_REG_OP maxs, type, def_reg, op3_reg
+ }
+ }
+ } else if (IR_IS_CONST_REF(op3)) {
+ int label = ir_get_const_label(ctx, op3);
+
+ if (ctx->mflags & IR_X86_AVX) {
+ if (op == IR_MIN) {
+ | ASM_AVX_REG_REG_TXT_OP vmins, type, def_reg, op2_reg, [=>label]
+ } else {
+ | ASM_AVX_REG_REG_TXT_OP vmaxs, type, def_reg, op2_reg, [=>label]
+ }
+ } else {
+ if (op == IR_MIN) {
+ | ASM_SSE2_REG_TXT_OP mins, type, def_reg, [=>label]
+ } else {
+ | ASM_SSE2_REG_TXT_OP maxs, type, def_reg, [=>label]
+ }
+ }
+ } else {
+ ir_mem mem;
+
+ if (ir_rule(ctx, op3) & IR_FUSED) {
+ mem = ir_fuse_load(ctx, def, op3);
+ } else {
+ mem = ir_ref_spill_slot(ctx, op3);
+ }
+ if (ctx->mflags & IR_X86_AVX) {
+ if (op == IR_MIN) {
+ | ASM_AVX_REG_REG_MEM_OP vmins, type, def_reg, op2_reg, mem
+ } else {
+ | ASM_AVX_REG_REG_MEM_OP vmaxs, type, def_reg, op2_reg, mem
+ }
+ } else {
+ if (op == IR_MIN) {
+ | ASM_SSE2_REG_MEM_OP mins, type, def_reg, mem
+ } else {
+ | ASM_SSE2_REG_MEM_OP maxs, type, def_reg, mem
+ }
+ }
+ }
+
+ if (IR_REG_SPILLED(ctx->regs[def][0])) {
+ ir_emit_store(ctx, type, def, def_reg);
+ }
+}
+
static void ir_emit_return_void(ir_ctx *ctx)
{
ir_backend_data *data = ctx->data;
@@ -15009,7 +15271,9 @@ static void ir_emit_cmp_and_branch_i64(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i
if (ir_rule(ctx, prev_insn->op1) == IR_CMP_AND_BRANCH_INT) {
prev_insn = &ctx->ir_base[prev_insn->op1];
prev_insn = &ctx->ir_base[prev_insn->op2];
- if (prev_insn->op1 == cmp_insn->op1 && prev_insn->op2 == cmp_insn->op2) {
+ if (prev_insn->op1 == cmp_insn->op1
+ && prev_insn->op2 == cmp_insn->op2
+ && !ir_insns_may_change_cmp_flags(ctx, insn->op1, def)) {
same_comparison = true;
}
}
@@ -17070,7 +17334,7 @@ static void ir_emit_cond_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(tmp_reg != IR_REG_NONE);
- if (!IR_IS_CONST_REF(insn->op2) && UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
+ if (!IR_IS_CONST_REF(insn->op1) && UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) {
op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1);
op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2);
} else {
@@ -17232,7 +17496,7 @@ static void ir_emit_cond_i64_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
IR_ASSERT(tmp_reg != IR_REG_NONE);
- if (UNEXPECTED(ctx->rules[insn->op2] & IR_FUSED_REG)) {
+ if (UNEXPECTED(ctx->rules[insn->op1] & IR_FUSED_REG)) {
op1_reg = ir_get_fused_reg(ctx, def, insn->op1, 1);
op2_reg = ir_get_fused_reg(ctx, def, insn->op1, 2);
} else {
@@ -26183,6 +26447,12 @@ static void ir_emit_load_params(ir_ctx *ctx)
ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset);
}
if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) {
+#if IR_X86_I64
+ if (insn->type == IR_I64 || insn->type == IR_U64) {
+ ir_emit_store_i64_lo(ctx, use, IR_REG_I64_LO(dst_reg));
+ ir_emit_store_i64_hi(ctx, use, IR_REG_I64_HI(dst_reg));
+ } else
+#endif
ir_emit_store(ctx, insn->type, use, dst_reg);
}
}
@@ -26283,10 +26553,10 @@ static void ir_fix_param_spills(ir_ctx *ctx)
if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) {
IR_ASSERT(ival->stack_spill_pos == -1);
ival->stack_spill_pos = stack_start + stack_offset;
- if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
- /* Remap VAR to PARAM stack slot */
- ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
- }
+ }
+ if (ival->flags & IR_LIVE_INTERVAL_VAR_MEM_PARAM) {
+ /* Remap VAR to PARAM stack slot */
+ ir_remap_param_spill_slot(ctx, use, stack_start + stack_offset);
}
}
if (sizeof(void*) == 8) {
@@ -26636,6 +26906,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr)
case IR_BINOP_AVX:
ir_emit_binop_avx(ctx, i, insn);
break;
+ case IR_MIN_MAX_FP:
+ ir_emit_min_max_fp(ctx, i, insn);
+ break;
case IR_MUL_INT:
case IR_DIV_INT:
case IR_MOD_INT:
@@ -26759,6 +27032,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr)
case IR_COND_CMP_FP:
ir_emit_cond_cmp_fp(ctx, i, insn);
break;
+ case IR_COND_MIN_MAX_FP:
+ ir_emit_cond_min_max_fp(ctx, i, insn);
+ break;
case IR_SWITCH:
ir_emit_switch(ctx, b, i, insn);
break;