Commit 112e11120f for perl
commit 112e11120f23d025ab50b51c1aca67ef67b63e62
Author: Yves Orton <demerphq@gmail.com>
Date: Sat Oct 17 12:41:00 2026 +0200
regex: avoid redundant trie conversions and position tracking
We already have the folder's UTF-8 bytes, so there is no need to decode
each character and encode it again. Feed those bytes to the trie
directly. Track pending octets separately from pending folded characters
so multi-character folds keep the right input positions.
For nonfolded UTF-8 AC matches, subtract the word's byte length to find
its start. This avoids the position ring and its heap allocation for
long words. Folded input still needs the ring, as its length can vary.
Once the acceptance chains are built, reuse each word's accept-state
slot for its byte length. We no longer need the accept state then, so
the word records need not grow.
diff --git a/regcomp.h b/regcomp.h
index 82d7a89074..633fce5768 100644
--- a/regcomp.h
+++ b/regcomp.h
@@ -1323,7 +1323,10 @@ typedef struct {
* accepted word is #2, and the previous accepted
* word is #3 */
U32 len; /* how many chars long is this word? */
- U32 accept; /* accept state for this word */
+ union {
+ U32 accept; /* accept state while building acceptance chains */
+ U32 len_utf8; /* non-folded word length in trie octets after building */
+ } u;
} reg_trie_wordinfo;
diff --git a/regcomp_trie.c b/regcomp_trie.c
index 82ddebfd0b..aeacb3d8b1 100644
--- a/regcomp_trie.c
+++ b/regcomp_trie.c
@@ -921,7 +921,7 @@ Perl_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch,
latin1_word_count++;
trie->wordinfo[curword].prev = 0;
trie->wordinfo[curword].len = wordlen;
- trie->wordinfo[curword].accept = state;
+ trie->wordinfo[curword].u.accept = state;
/* If this branch continues past its exact node, preserve the
* continuation and the capture-state values associated with
@@ -1712,7 +1712,7 @@ Perl_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch,
prev = 0;
if (trie->wordinfo[word].prev)
continue;
- state = trie->wordinfo[word].accept;
+ state = trie->wordinfo[word].u.accept;
while (state) {
state = prev_states[state];
if (!state)
@@ -1723,6 +1723,26 @@ Perl_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch,
}
trie->wordinfo[word].prev = prev;
}
+ if (!TRIE_IS_FOLDED(trie)) {
+ /* Repurpose the per-word accept slot to store UTF-8 octet
+ * lengths now that we have finished building acceptance chains.
+ * We reuse the predecessor array for path depths, which give
+ * the octet lengths. Each state follows its parent in allocation
+ * order, so we can replace the predecessors in place.
+ *
+ * AC conversion excludes tries with an extracted prefix, so its
+ * word lengths can include the complete path from state 1.
+ */
+ prev_states[1] = 0;
+ for (state = 2; state < trie->statecount; state++) {
+ assert(prev_states[state] < state);
+ prev_states[state] = prev_states[prev_states[state]] + 1;
+ }
+ for (word = 1; word <= trie->wordcount; word++) {
+ const U32 accept = trie->wordinfo[word].u.accept;
+ trie->wordinfo[word].u.len_utf8 = prev_states[accept];
+ }
+ }
Safefree(prev_states);
}
diff --git a/regexec.c b/regexec.c
index 9edc55130e..9c0ef28a8b 100644
--- a/regexec.c
+++ b/regexec.c
@@ -1884,15 +1884,14 @@ Perl_re_intuit_start(pTHX_
#define DECL_TRIE_TYPE(scan) \
- const enum { trie_plain, trie_utf8, trie_utf8_fold, \
+ const enum { trie_plain, trie_utf8_fold, \
trie_latin_utf8_fold, trie_utf8_exactfa_fold, \
trie_latin_utf8_exactfa_fold, \
- trie_utf8l, trie_flu8, trie_flu8_latin } \
+ trie_flu8, trie_flu8_latin } \
trie_type = ((IS_EXACT_OR_NF(FLAGS(scan)) \
- || IS_EXACT_REQ8_OR_NF_REQ8(FLAGS(scan))) \
- ? (utf8_target ? trie_utf8 : trie_plain) \
- : (FLAGS(scan) == EXACTL) \
- ? (utf8_target ? trie_utf8l : trie_plain)\
+ || IS_EXACT_REQ8_OR_NF_REQ8(FLAGS(scan)) \
+ || FLAGS(scan) == EXACTL) \
+ ? trie_plain \
: (FLAGS(scan) == EXACTFAA) \
? (utf8_target \
? trie_utf8_exactfa_fold \
@@ -1909,10 +1908,11 @@ Perl_re_intuit_start(pTHX_
* 'foldbuf+sizeof(foldbuf)' */
#define REXEC_TRIE_READ_CHAR(trie_type, trie, uc, uc_end, \
uscan, len, uvc, octet, foldlen, foldbuf, \
- uniflags, octet_scan, octets_remaining, \
+ octet_scan, octets_remaining, \
octet_buffer) \
STMT_START { \
STRLEN skiplen; \
+ const U8 *trie_bytes; \
if (TRIE_RAW_INPUT_MODE(trie, utf8_target)) { \
uvc = (U8)*uc; \
octet = uvc; \
@@ -1936,11 +1936,11 @@ STMT_START { \
case trie_utf8_fold: \
do_trie_utf8_fold: \
if ( foldlen > 0 ) { \
- (void) utf8_to_uv_flags( (const U8*) uscan, uscan + foldlen, \
- &uvc, &len, \
- (uniflags|UTF8_DIE_IF_MALFORMED)); \
- foldlen -= len; \
- uscan += len; \
+ trie_bytes = uscan; \
+ skiplen = UTF8SKIP(trie_bytes); \
+ DEBUG_TRIE_EXECUTE_r(uvc = valid_utf8_to_uv(trie_bytes, NULL)); \
+ foldlen -= skiplen; \
+ uscan += skiplen; \
len = 0; \
} else { \
uvc = toFOLD_utf8_flags_( (const U8*) uc, uc_end, foldbuf, \
@@ -1949,6 +1949,7 @@ STMT_START { \
skiplen = UVCHR_SKIP( uvc ); \
foldlen -= skiplen; \
uscan = foldbuf + skiplen; \
+ trie_bytes = foldbuf; \
} \
break; \
case trie_flu8_latin: \
@@ -1960,11 +1961,11 @@ STMT_START { \
case trie_latin_utf8_fold: \
do_trie_latin_utf8_fold: \
if ( foldlen > 0 ) { \
- (void) utf8_to_uv_flags( (const U8*) uscan, uscan + foldlen, \
- &uvc, &len, \
- (uniflags|UTF8_DIE_IF_MALFORMED)); \
- foldlen -= len; \
- uscan += len; \
+ trie_bytes = uscan; \
+ skiplen = UTF8SKIP(trie_bytes); \
+ DEBUG_TRIE_EXECUTE_r(uvc = valid_utf8_to_uv(trie_bytes, NULL)); \
+ foldlen -= skiplen; \
+ uscan += skiplen; \
len = 0; \
} else { \
len = 1; \
@@ -1972,40 +1973,28 @@ STMT_START { \
skiplen = UVCHR_SKIP( uvc ); \
foldlen -= skiplen; \
uscan = foldbuf + skiplen; \
+ trie_bytes = foldbuf; \
} \
break; \
- case trie_utf8l: \
- CHECK_AND_WARN_PROBLEMATIC_LOCALE_; \
- if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \
- CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG_(uc, uc_end); \
- } \
- /* FALLTHROUGH */ \
- case trie_utf8: \
- (void) utf8_to_uv_flags( (const U8*) uc, uc_end, &uvc, &len, \
- (uniflags|UTF8_DIE_IF_MALFORMED)); \
- break; \
- case trie_plain: \
+ case trie_plain: { \
+ const native_octet_utf8_t *native_octet; \
+ /* Non-folded UTF-8 input takes the raw-octet path above. */ \
+ assert(!utf8_target); \
uvc = (UV)*uc; \
len = 1; \
+ native_octet = &PL_native_octet_utf8[(U8)uvc]; \
+ trie_bytes = native_octet->bytes; \
+ skiplen = native_octet->len; \
+ break; \
} \
- { \
- const native_octet_utf8_t *native_octet; \
- U8 *end; \
- if (!utf8_target && FITS_IN_8_BITS(uvc)) { \
- /* Non-UTF-8 input is a native octet. Use its cached UTF-8 \
- * representation for the trie transition stream. */ \
- native_octet = &PL_native_octet_utf8[(U8)uvc]; \
- octet = native_octet->bytes[0]; \
- octet_scan = native_octet->bytes + 1; \
- octets_remaining = native_octet->len - 1; \
- } else { \
- end = uvchr_to_utf8(octet_buffer, uvc); \
- octet = octet_buffer[0]; \
- octet_scan = octet_buffer + 1; \
- octets_remaining = (STRLEN)(end - octet_buffer - 1); \
- } \
} \
- } \
+ /* Folded characters use the folder's UTF-8 bytes directly. */ \
+ /* Keep pending characters separate so foldlen tracks multichar */ \
+ /* expansions. */ \
+ octet = trie_bytes[0]; \
+ octet_scan = trie_bytes + 1; \
+ octets_remaining = skiplen - 1; \
+ } \
} STMT_END
#define DUMP_EXEC_POS(li,s,doutf8,depth) \
@@ -3492,12 +3481,10 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
#ifndef DEBUGGING
const U32 root_base = aho->states[1].trans.base;
#endif
- /* Native, non-folded characters each occupy one target byte.
- * An accepting state follows a complete encoded character, so
- * uc - wordlen gives the original start even when reading that
- * character required several trie octets. UTF-8 and folded
- * targets need a ring mapping characters to target positions. */
- const bool need_points = utf8_target || TRIE_IS_FOLDED(trie);
+ /* Non-folded words have fixed target byte lengths: character
+ * counts for native input and trie octet counts for UTF-8.
+ * Folded words need a ring because expansions change lengths. */
+ const bool need_points = TRIE_IS_FOLDED(trie);
U8 **points;
/* For a shorter maxlen, points are stored in a stack buffer. */
@@ -3548,7 +3535,6 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
*/
while (s <= last_start) {
- const U32 uniflags = UTF8_ALLOW_DEFAULT;
U8 *uc = (U8*)s;
U32 octet;
U32 base = 1;
@@ -3636,8 +3622,10 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
&& (!utf8_target || !UTF8_IS_CONTINUATION(*uc));
if ( word ) {
- const STRLEN wordlen = trie->wordinfo[word].len;
- assert(wordlen <= maxlen);
+ const STRLEN wordlen = !need_points && utf8_target
+ ? trie->wordinfo[word].u.len_utf8
+ : trie->wordinfo[word].len;
+ assert(trie->wordinfo[word].len <= maxlen);
U8 *lpos = !need_points ? uc - wordlen
: points[pointpos >= wordlen
? pointpos - wordlen
@@ -3652,7 +3640,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
REXEC_TRIE_READ_CHAR(trie_type, trie, uc,
(U8 *) strend, uscan, len, uvc,
octet, foldlen, foldbuf,
- uniflags, octet_scan,
+ octet_scan,
octets_remaining, octet_buffer);
DEBUG_TRIE_EXECUTE_r({
@@ -3750,9 +3738,11 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
}
}
if ( aho->states[ state ].wordnum ) {
- const STRLEN wordlen
- = trie->wordinfo[aho->states[state].wordnum].len;
- assert(wordlen <= maxlen);
+ const U32 word = aho->states[state].wordnum;
+ const STRLEN wordlen = !need_points && utf8_target
+ ? trie->wordinfo[word].u.len_utf8
+ : trie->wordinfo[word].len;
+ assert(trie->wordinfo[word].len <= maxlen);
U8 *lpos = !need_points ? uc - wordlen
: points[pointpos >= wordlen
? pointpos - wordlen
@@ -7404,7 +7394,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
REXEC_TRIE_READ_CHAR(trie_type, trie, uc,
(U8 *) loceol, uscan,
len, uvc, octet, foldlen,
- foldbuf, uniflags, octet_scan,
+ foldbuf, octet_scan,
octets_remaining, octet_buffer);
if (TRIE_RAW_INPUT_MODE(trie, utf8_target) \
? (!utf8_target || !UTF8_IS_CONTINUATION(*uc)) \
@@ -7562,9 +7552,9 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
while (foldlen) {
if (!--chars)
break;
- (void) utf8_to_uv_flags((U8*)uscan,
- uscan + foldlen, &uvc, &len,
- uniflags|UTF8_DIE_IF_MALFORMED);
+ /* The folder produces valid UTF-8. Only its
+ * character boundaries are needed here. */
+ len = UTF8SKIP(uscan);
uscan += len;
foldlen -= len;
}
diff --git a/t/re/pat.t b/t/re/pat.t
index 2aa9f6b75e..46989c9a2e 100644
--- a/t/re/pat.t
+++ b/t/re/pat.t
@@ -28,7 +28,7 @@ skip_all_without_unicode_tables();
my $has_locales = locales_enabled('LC_CTYPE');
my $utf8_locale = find_utf8_ctype_locale();
-plan tests => 2290 + 6 * $Config{uvsize}; # Update when adding/deleting tests.
+plan tests => 2438 + 6 * $Config{uvsize}; # Update when adding/deleting tests.
run_tests() unless caller;
@@ -88,6 +88,20 @@ sub run_tests {
[ 'empty alternative', "\x{100}x|", 'abc' ],
[ 'wide-only alternatives', "\x{100}x|\x{200}y", 'plain' ],
[ 'mixed alternatives', "\x{100}x|cat", 'catcat' ],
+ [ 'UTF-8 failure suffix', "\x{100}ab|b\x{1f600}",
+ "\x{100}ab\x{100}acb\x{1f600}" ],
+ [ 'UTF-8 unequal byte lengths', "\x{e9}x|\x{100}x|\x{1f600}x",
+ "\x{1f600}x\x{e9}x\x{100}x" ],
+ [ 'UTF-8 accepting suffix', "\x{100}x|x\x{1f600}|\x{1f600}",
+ "\x{100}x\x{1f600}\x{1f600}" ],
+ [ 'UTF-8 duplicate paths', "\x{100}x(?=A)|\x{100}x(?=B)|\x{200}y",
+ "\x{100}xZ\x{100}xB\x{200}y" ],
+ [ 'UTF-8 heap length',
+ ("\x{100}" x 40) . 'b|b' . ("\x{1f600}" x 39) . 'c',
+ ("\x{100}" x 160) . 'b' . ("\x{1f600}" x 39) . 'c' ],
+ [ 'UTF-8 embedded NUL', "\x{100}\0x|\x{200}\0y",
+ "\x{100}\0Z\x{200}\0y\x{100}\0x" ],
+ [ 'UTF-8 invariant target suffix', "\x{100}abc|bc|cat", 'zzabccat' ],
[ 'ring wraps', 'aaaaab|baaaac', ( 'a' x 100 ) . 'baaaac' ],
[ 'short word after wraps', 'abababx|babc', ( 'ab' x 100 ) . 'c' ],
[
@@ -153,6 +167,39 @@ sub run_tests {
}
}
+ {
+ my @folded_backtracks = (
+ ['ss', "\x{df}"],
+ ['ffi', "\x{fb03}"],
+ ["\x{3b9}\x{308}\x{301}", "\x{390}"],
+ ["\x{565}\x{582}", "\x{587}"],
+ ["\x{3b9}\x{308}\x{301}\x{565}\x{582}", "\x{390}\x{587}"],
+ );
+ for my $case (@folded_backtracks) {
+ my ($literal, $input) = @$case;
+ my $pattern = "${literal}x(?=NO)|${literal}xy(?=YES)|${literal}xyz|zoo";
+ for my $charset ('u', 'a', 'aa', 'd') {
+ my ($trie_re, $plain_re);
+ {
+ local ${^RE_TRIE_MAXBUF} = 65536;
+ $trie_re = qr/\A(?i$charset:$pattern)(?#folded backtracking trie)/;
+ }
+ {
+ local ${^RE_TRIE_MAXBUF} = -1;
+ $plain_re = qr/\A(?i$charset:$pattern)(?#folded backtracking branches)/;
+ }
+ for my $upgrade (0, 1) {
+ my $target = "${input}xyYES";
+ utf8::upgrade($target) if $upgrade;
+ my $expected = $target =~ $plain_re ? "$-[0]:$+[0]" : 'no match';
+ my $got = $target =~ $trie_re ? "$-[0]:$+[0]" : 'no match';
+ is($got, $expected,
+ "folded trie backtracking agrees with branches: /$charset, upgrade=$upgrade");
+ }
+ }
+ }
+ }
+
{
my @folded_starts = (
['ssx', "sSx"],
@@ -167,6 +214,12 @@ sub run_tests {
["\x{3b9}\x{308}\x{301}x", "\x{390}x"],
['sx', "\x{df}x"],
['ssx', "s\x{df}x"],
+ ["\x{17f}\x{17f}tail", "\x{df}tail"],
+ ["\x{565}\x{582}tail", "\x{587}tail"],
+ ["\x{10428}tail", "\x{10400}tail"],
+ ["\x{565}\x{582}\x{10428}tail", "\x{587}\x{10400}tail"],
+ ["\x{3b9}\x{308}\x{301}\x{565}\x{582}tail",
+ "\x{390}\x{587}tail"],
);
for my $case (@folded_starts) {