Commit 3b93382d6d for perl
commit 3b93382d6de9d278d13113e0675b1b5496e4cc41
Author: Yves Orton <demerphq@gmail.com>
Date: Sat Oct 10 21:09:06 2026 +0200
regex: use start-octet maps for folded Aho-Corasick tries
For folded words we need a map of the original target characters, not
their folded bytes. Use inverse folds to find those characters,
including characters whose multi-character folds can start a word. The
map may admit extra candidates, as the matcher checks the actual folding
rules.
Leave both BMH tables empty for folded tries. Offsets in a folded word
do not tell us how far we can skip in the original input. We can use the
start map and repeat scanner at the root when no encoded or folded input
is pending.
Resolves https://github.com/Perl/perl5/issues/24935
diff --git a/pod/perldelta.pod b/pod/perldelta.pod
index d2f932b013..868225c5d9 100644
--- a/pod/perldelta.pod
+++ b/pod/perldelta.pod
@@ -103,6 +103,11 @@ blocks of octets that cannot start an alternative. These filters apply to
eligible case-sensitive literals, with separate shift tables for Latin-1 and
UTF-8 targets.
+Eligible case-insensitive literal alternatives use a start-octet map that
+includes their possible folds. They do not use the shift tables, because
+folding can change encoded lengths. Locale-dependent folding uses the
+existing scanner.
+
=back
=head1 Modules and Pragmata
diff --git a/regcomp.h b/regcomp.h
index c5422d17d9..82d7a89074 100644
--- a/regcomp.h
+++ b/regcomp.h
@@ -1426,7 +1426,8 @@ struct reg_ac_data_ {
* BMH table at 256, and the start map at 512. Map entries use mask 0x01
* for Latin-1 starts and 0x02 for UTF-8 starts. Each minlen is the
* shortest eligible literal length in that encoding's octets, capped
- * at 255. Zero means that encoding has no usable BMH table.
+ * at 255. Zero means that encoding has no usable BMH table. Folded
+ * tries use only the start map and leave both lengths zero.
* The allocation is immutable and
* shared by regex clones through this AC structure's refcount. */
U8 *prefilter;
diff --git a/regcomp_trie.c b/regcomp_trie.c
index afa117ee5e..626e701d6e 100644
--- a/regcomp_trie.c
+++ b/regcomp_trie.c
@@ -1777,6 +1777,69 @@ S_ac_path_as_latin1(const U8 *path, const U32 path_len, const U32 source_chars,
return TRUE;
}
+/* The map describes original target octets, rather than trie transitions.
+ * A wide codepoint has only a UTF-8 entry. Native Latin-1 codepoints have
+ * an entry for each target encoding. */
+static void
+S_ac_map_start_codepoint(pTHX_ U8 *map, const UV cp)
+{
+ U8 encoded[UTF8_MAXBYTES + 1];
+
+ if (cp <= 255)
+ map[(U8)cp] |= REG_AC_PREFILTER_LATIN1_MASK;
+ uv_to_utf8(encoded, cp);
+ map[encoded[0]] |= REG_AC_PREFILTER_UTF8_MASK;
+}
+
+/* Folded trie paths are UTF-8 encoded. Visit only the first codepoint of
+ * each path, not every accepting word sharing that initial codepoint. */
+static void
+S_ac_map_folded_starts(pTHX_ const reg_trie_data *trie, const U32 state,
+ U8 *path, const U32 path_len, U8 *map,
+ SV **first_folds)
+{
+ const U32 base = trie->states[state].trans.base;
+
+ for (U32 ch = trie->states[state].min_octet;
+ base && ch <= trie->states[state].max_octet; ch++)
+ {
+ const U32 next = S_trie_trans_state(trie, state, base, ch, 0);
+
+ if (!next)
+ continue;
+
+ assert(path_len < UTF8_MAXBYTES);
+ path[path_len] = (U8)ch;
+ if (path_len + 1 == UTF8SKIP(path)) {
+ U8 folded[UTF8_MAXBYTES_CASE + 1];
+ STRLEN folded_len;
+ U32 first;
+ const U32 *remaining;
+ const UV cp = valid_utf8_to_uv(path, NULL);
+ const UV initial = to_uni_fold(cp, folded, &folded_len);
+ const Size_t count = inverse_folds_(initial, &first, &remaining);
+
+ /* Unicode folding, together with native /d fold pairs, covers all
+ * possible starts under /d and /aa. It may admit false positives,
+ * but that is fine as AC rejects them using the actual folding rules. */
+ S_ac_map_start_codepoint(aTHX_ map, cp);
+ if (cp <= 255 && (trie->prop_flags & TRIE_FOLD_NATIVE))
+ S_ac_map_start_codepoint(aTHX_ map, PL_fold[(U8)cp]);
+ S_ac_map_start_codepoint(aTHX_ map, initial);
+ if (count) {
+ S_ac_map_start_codepoint(aTHX_ map, first);
+ for (Size_t i = 1; i < count; i++)
+ S_ac_map_start_codepoint(aTHX_ map, remaining[i - 1]);
+ }
+ *first_folds = add_cp_to_invlist(*first_folds, initial);
+ }
+ else {
+ S_ac_map_folded_starts(aTHX_ trie, next, path, path_len + 1,
+ map, first_folds);
+ }
+ }
+}
+
#ifdef DEBUGGING
static void
S_dump_ac_prefilter_table(pTHX_ const char *name, const U8 *table,
@@ -2040,6 +2103,43 @@ Perl_construct_ahocorasick_from_trie(pTHX_ RExC_state_t *pRExC_state, regnode *s
Safefree(path_depths);
Safefree(parents);
}
+ else if (TRIE_IS_FOLDED(trie)
+ && FLAGS(source) != EXACTFLU8
+ && trie->startstate == 1 && !trie->states[1].wordnum
+ && !trie->prefixlen_octets && !trie->prefixlen_chars)
+ {
+ SV *first_folds = NULL;
+ U8 path[UTF8_MAXBYTES + 1];
+ U8 *start_map;
+ UV start, end;
+
+ /* BMH offsets in folded data do not describe the original target.
+ * Leave both anchor lengths zero and use only the start map. Locale
+ * folding is excluded because its rules can change at execution. */
+ aho->prefilter = (U8 *)PerlMemShared_calloc(
+ REG_AC_PREFILTER_SIZE, sizeof(U8));
+ start_map = aho->prefilter + REG_AC_PREFILTER_MAP_OFFSET;
+ S_ac_map_folded_starts(aTHX_ trie, 1, path, 0, start_map, &first_folds);
+
+ /* A single target codepoint can supply several folded characters.
+ * Add every such codepoint whose fold begins with a possible first
+ * folded character. We need not compare the rest of its fold: false
+ * positives are safe, including folds extending beyond a trie word
+ * into its following regex nodes. */
+ invlist_iterinit(PL_HasMultiCharFold);
+ while (invlist_iternext(PL_HasMultiCharFold, &start, &end)) {
+ for (UV cp = start; cp <= end; cp++) {
+ U8 folded[UTF8_MAXBYTES_CASE + 1];
+ STRLEN folded_len;
+ const UV initial = to_uni_fold(cp, folded, &folded_len);
+
+ if (first_folds && invlist_contains_cp_(first_folds, initial))
+ S_ac_map_start_codepoint(aTHX_ start_map, cp);
+ }
+ }
+ SvREFCNT_dec(first_folds);
+ DEBUG_TRIE_COMPILE_r(S_dump_ac_prefilters(aTHX_ aho, depth));
+ }
/* The breadth-first work queue can contain each state at most once, so
* numstates entries suffice. q_read and q_write address it as a
* circular queue while q_write advances monotonically. */
diff --git a/regexec.c b/regexec.c
index 65ab1d4624..8587c94559 100644
--- a/regexec.c
+++ b/regexec.c
@@ -3560,21 +3560,24 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
/* A prefilter may skip only from the AC root. Once
* AC has consumed a prefix, its failure state must
* retain that partial-match information. */
- if (has_bmh && !foldlen && !octets_remaining) {
+ if (prefilter && !foldlen && !octets_remaining) {
/* BMH positions its anchor by encoded octet
- * length, but candidate starts must be character
- * boundaries in UTF-8 targets. */
- while (uc <= (U8*)last_start) {
- U8 distance;
+ * length. Folded tries have no BMH table and
+ * use only the map of original target starts.
+ * Both require character boundaries in UTF-8. */
+ while (uc < (U8*)strend && uc <= (U8*)last_start) {
+ U8 distance = 0;
if (utf8_target && UTF8_IS_CONTINUATION(*uc)) {
uc++;
pointpos = 0;
continue;
}
- if ((STRLEN)((U8*)strend - uc) < bmh_minlen)
- break;
- distance = bmh_shift[uc[bmh_minlen - 1]];
+ if (has_bmh) {
+ if ((STRLEN)((U8*)strend - uc) < bmh_minlen)
+ break;
+ distance = bmh_shift[uc[bmh_minlen - 1]];
+ }
/* For shifts zero and one, a start-map miss
* proves independently that no literal starts
@@ -3593,7 +3596,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
/* The helper consumes at least the
* rejected octet at uc. */
uc = (U8 *)S_scan_repeated_uv_blocks(
- uc, (U8*)strend, (U8*)last_start,
+ uc, (U8*)strend,
+ MIN((U8*)last_start, (U8*)strend - 1),
start_map, start_mask, utf8_target);
pointpos = 0;
continue;
@@ -3603,7 +3607,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s,
uc += distance;
pointpos = 0;
}
- if (uc > (U8*)last_start)
+ if (uc >= (U8*)strend || uc > (U8*)last_start)
break;
}
}
diff --git a/t/re/pat.t b/t/re/pat.t
index 11146e1206..7dce0a05d2 100644
--- a/t/re/pat.t
+++ b/t/re/pat.t
@@ -28,7 +28,7 @@ skip_all_without_unicode_tables();
my $has_locales = locales_enabled('LC_CTYPE');
my $utf8_locale = find_utf8_ctype_locale();
-plan tests => 1588; # Update this when adding/deleting tests.
+plan tests => 1780; # Update this when adding/deleting tests.
run_tests() unless caller;
@@ -36,6 +36,66 @@ run_tests() unless caller;
# Tests start here.
#
sub run_tests {
+ {
+ my @folded_starts = (
+ ['ssx', "sSx"],
+ ['ssx', "\x{df}x"],
+ ['ssx', "\x{17f}sx"],
+ ['ssx', "\x{1e9e}x"],
+ ['ssx', "\x{1df95}x"],
+ ['kx', "\x{212a}x"],
+ ["\x{b5}x", "\x{39c}x"],
+ ['ffx', "\x{fb00}x"],
+ ['ffix', "\x{fb03}x"],
+ ["\x{3b9}\x{308}\x{301}x", "\x{390}x"],
+ ['sx', "\x{df}x"],
+ ['ssx', "s\x{df}x"],
+ );
+
+ for my $case (@folded_starts) {
+ my ($literal, $input) = @$case;
+ my $case_name = join(' ', map { sprintf '%X', ord $_ }
+ split //, "$literal:$input");
+
+ for my $charset ('u', 'a', 'aa', 'd') {
+ my ($trie_re, $plain_re);
+
+ {
+ local ${^RE_TRIE_MAXBUF} = 65536;
+ $trie_re = qr/(?i$charset:$literal|zoo)(?#with trie)/;
+ }
+
+ {
+ local ${^RE_TRIE_MAXBUF} = -1;
+ $plain_re = qr/(?i$charset:$literal|zoo)(?#without trie)/;
+ }
+
+ for my $upgrade (0, 1) {
+ my $target = ('Q' x 31) . $input;
+ utf8::upgrade($target) if $upgrade;
+ my @expected;
+ my @got;
+
+ while ($target =~ /$plain_re/g) {
+ push @expected, "$-[0]:$+[0]";
+ }
+
+ while ($target =~ /$trie_re/g) {
+ push @got, "$-[0]:$+[0]";
+ }
+
+ is(join(',', @got), join(',', @expected),
+ "folded start map agrees with branches under /i$charset, upgrade=$upgrade, case=$case_name");
+
+ my $miss = 'Q' x 77;
+ utf8::upgrade($miss) if $upgrade;
+ ok($miss !~ /$trie_re/,
+ "folded start map rejects a repeated run under /i$charset, upgrade=$upgrade, case=$case_name");
+ }
+ }
+ }
+ }
+
{
my $haystack = ("A" x 5000) . "cat";
ok($haystack =~ /walrus|cat/, 'AC finds a literal after a long rejected run');
@@ -258,8 +318,8 @@ sub run_tests {
ok($retry_target =~ /(?:abc|bcd)X/ && $-[0] == 1 && $+[0] == 5,
"AC retries an overlapping start after full-match rejection in $encoding");
- # Prefilters are disabled for folds, but the point-ring changes
- # apply there too. One sharp s contributes two folded characters.
+ # Folded tries use only the start map. One sharp s contributes
+ # two folded characters, so ring offsets count original input.
for my $case (["\x{df}x", 0, 2],
["s\x{df}x", 1, 3],
["s\x{df}ssx", 2, 5])