Commit 1b72dea75a for perl
commit 1b72dea75a56a45f87c85e5ddd5e98dabaa6e272
Author: Yves Orton <demerphq@gmail.com>
Date: Thu Oct 29 10:39:12 2026 +0100
regex: skip irrelevant multi-character fold scans
Skip the multi-character-fold closure scan when no folded trie start can
begin such a fold. Generate a classifier from the first code point of
each multi-character Case_Folding mapping, and track whether any trie
start needs the scan. Keep the existing closure calculation when one
does.
The classifier consumes an already folded UTF string. It identifies the
first character of a multi-character fold, rather than a character whose
own fold expands to multiple characters.
In a local comparison on CPU 2 using GCC 13.3, -O2, no debugging and no
threads, the four-alternative fixture without a relevant fold start fell
from a median 10.821 to 7.354 microseconds per compilation. The run used
24 alternating-order sample pairs targeting 0.4 seconds per baseline
sample. These synthetic measurements include compilation and
destruction, with setup excluded.
diff --git a/regcharclass.h b/regcharclass.h
index 15515a09a4..0d4db21255 100644
--- a/regcharclass.h
+++ b/regcharclass.h
@@ -1749,6 +1749,55 @@ character sequence actually occupies.
: 0 ) )
#endif
+/*
+ MULTI_CHAR_FOLD_START: The first character of a multi-character fold for ASCII/Latin1
+
+ %regcharclass_multi_char_folds::multi_char_fold_starts()
+*/
+/*
+=for apidoc_section $casing
+=for apidoc QT|STRLEN|is_MULTI_CHAR_FOLD_START_utf8|const U8 * s
+This character can begin the full fold of a single character that folds to
+multiple characters. The input is already folded.
+
+This returns non-zero if the
+sequence of the next few bytes starting at C<s>
+form a valid UTF-8 representation of such a character.
+
+Otherwise, it returns zero.
+
+The value returned gives the number of bytes the matched
+character actually occupies.
+
+You need to be sure that the bytes at C<s> comprise at least
+one complete character before calling this; otherwise this
+could read beyond its end.
+
+
+
+=cut
+*/
+
+
+#if defined(PERL_CORE) || defined(PERL_EXT_RE_BUILD)
+/*** GENERATED CODE ***/
+# define is_MULTI_CHAR_FOLD_START_utf8(s) \
+( (STRLEN)( ( 'a' == ((const U8*)s)[0] || 'f' == ((const U8*)s)[0] || inRANGE_helper_(U8, ((const U8*)s)[0], 'h', 'j') || inRANGE_helper_(U8, ((const U8*)s)[0], 's', 't') || 'w' == ((const U8*)s)[0] || 'y' == ((const U8*)s)[0] ) ? 1\
+: ( 0xCA == ((const U8*)s)[0] ) ? \
+ ( ( 0xBC == ((const U8*)s)[1] ) ? 2 : 0 ) \
+: ( 0xCE == ((const U8*)s)[0] ) ? \
+ ( ( ( ( ((const U8*)s)[1] & 0xFD ) == 0xAC ) || ( ( ((const U8*)s)[1] & 0xF7 ) == 0xB1 ) || ((const U8*)s)[1] == 0xB7 ) ? 2 : 0 )\
+: ( 0xCF == ((const U8*)s)[0] ) ? \
+ ( ( ( ( ((const U8*)s)[1] & 0xFB ) == 0x81 ) || ((const U8*)s)[1] == 0x89 || ((const U8*)s)[1] == 0x8E ) ? 2 : 0 )\
+: ( 0xD5 == ((const U8*)s)[0] ) ? \
+ ( ( 0xA5 == ((const U8*)s)[1] || 0xB4 == ((const U8*)s)[1] || 0xBE == ((const U8*)s)[1] ) ? 2 : 0 )\
+: ( 0xE1 == ((const U8*)s)[0] ) ? \
+ ( ( 0xBC == ((const U8*)s)[1] ) ? \
+ ( ( ( ((const U8*)s)[2] & 0xD8 ) == 0x80 ) ? 3 : 0 ) \
+ : ( ( 0xBD == ((const U8*)s)[1] ) && ( ( ( ((const U8*)s)[2] & 0xF8 ) == 0xA0 ) || ( ( ((const U8*)s)[2] & 0xFB ) == 0xB0 ) || ((const U8*)s)[2] == 0xBC ) ) ? 3 : 0 )\
+: 0 ) )
+#endif
+
/*
FOLDS_TO_MULTI: characters that fold to multi-char strings for ASCII/Latin1
@@ -3300,6 +3349,31 @@ character actually occupies.
: 0 ) )
#endif
+/*
+ MULTI_CHAR_FOLD_START: The first character of a multi-character fold for EBCDIC 1047
+
+ %regcharclass_multi_char_folds::multi_char_fold_starts()
+*/
+
+#if defined(PERL_CORE) || defined(PERL_EXT_RE_BUILD)
+/*** GENERATED CODE ***/
+# define is_MULTI_CHAR_FOLD_START_utf8(s) \
+( (STRLEN)( ( ( ( ((const U8*)s)[0] & 0xEF ) == 'a' ) || ( ( ((const U8*)s)[0] & 0xDF ) == 'f' ) || ( ( ((const U8*)s)[0] & 0xFE ) == 'h' ) || ( ( ((const U8*)s)[0] & 0xFE ) == 's' ) || ((const U8*)s)[0] == 'y' ) ? 1\
+: ( 0xAB == ((const U8*)s)[0] ) ? \
+ ( ( 0x70 == ((const U8*)s)[1] ) ? 2 : 0 ) \
+: ( 0xB4 == ((const U8*)s)[0] ) ? \
+ ( ( ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[1]) & 0xFD ) == 0xAC ) || ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[1]) & 0xF7 ) == 0xB1 ) || ((const U8*)s)[1] == 0x66 ) ? 2 : 0 )\
+: ( 0xB5 == ((const U8*)s)[0] ) ? \
+ ( ( ( ( ((const U8*)s)[1] & 0xFB ) == 0x42 ) || ((const U8*)s)[1] == 0x4A || ((const U8*)s)[1] == 0x55 ) ? 2 : 0 )\
+: ( 0xB8 == ((const U8*)s)[0] ) ? \
+ ( ( ( 0x52 == ((const U8*)s)[1] ) && ( 0x46 == ((const U8*)s)[2] || 0x63 == ((const U8*)s)[2] || 0x72 == ((const U8*)s)[2] ) ) ? 3 : 0 )\
+: ( 0xBF == ((const U8*)s)[0] ) ? \
+ ( ( inRANGE_helper_(U8, ((const U8*)s)[1], 0x67, 0x68) ) ? \
+ ( ( inRANGE_helper_(U8, ((const U8*)s)[2], 0x41, 0x48) ) ? 3 : 0 ) \
+ : ( ( 0x6A == ((const U8*)s)[1] ) && ( ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[2]) & 0xF8 ) == 0xA0 ) || ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[2]) & 0xFB ) == 0xB0 ) || ((const U8*)s)[2] == 0x70 ) ) ? 3 : 0 )\
+: 0 ) )
+#endif
+
/*
FOLDS_TO_MULTI: characters that fold to multi-char strings for EBCDIC 1047
@@ -4672,6 +4746,31 @@ character actually occupies.
: 0 ) )
#endif
+/*
+ MULTI_CHAR_FOLD_START: The first character of a multi-character fold for EBCDIC 037
+
+ %regcharclass_multi_char_folds::multi_char_fold_starts()
+*/
+
+#if defined(PERL_CORE) || defined(PERL_EXT_RE_BUILD)
+/*** GENERATED CODE ***/
+# define is_MULTI_CHAR_FOLD_START_utf8(s) \
+( (STRLEN)( ( ( ( ((const U8*)s)[0] & 0xEF ) == 'a' ) || ( ( ((const U8*)s)[0] & 0xDF ) == 'f' ) || ( ( ((const U8*)s)[0] & 0xFE ) == 'h' ) || ( ( ((const U8*)s)[0] & 0xFE ) == 's' ) || ((const U8*)s)[0] == 'y' ) ? 1\
+: ( 0xAA == ((const U8*)s)[0] ) ? \
+ ( ( 0x6A == ((const U8*)s)[1] ) ? 2 : 0 ) \
+: ( 0xB3 == ((const U8*)s)[0] ) ? \
+ ( ( ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[1]) & 0xFD ) == 0xAC ) || ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[1]) & 0xF7 ) == 0xB1 ) || ((const U8*)s)[1] == 0x65 ) ? 2 : 0 )\
+: ( 0xB4 == ((const U8*)s)[0] ) ? \
+ ( ( ( ( ((const U8*)s)[1] & 0xFB ) == 0x42 ) || ((const U8*)s)[1] == 0x4A || ((const U8*)s)[1] == 0x55 ) ? 2 : 0 )\
+: ( 0xB7 == ((const U8*)s)[0] ) ? \
+ ( ( ( 0x52 == ((const U8*)s)[1] ) && ( 0x46 == ((const U8*)s)[2] || 0x62 == ((const U8*)s)[2] || 0x71 == ((const U8*)s)[2] ) ) ? 3 : 0 )\
+: ( 0xBF == ((const U8*)s)[0] ) ? \
+ ( ( inRANGE_helper_(U8, ((const U8*)s)[1], 0x66, 0x67) ) ? \
+ ( ( inRANGE_helper_(U8, ((const U8*)s)[2], 0x41, 0x48) ) ? 3 : 0 ) \
+ : ( ( 0x69 == ((const U8*)s)[1] ) && ( ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[2]) & 0xF8 ) == 0xA0 ) || ( ( NATIVE_UTF8_TO_I8(((const U8*)s)[2]) & 0xFB ) == 0xB0 ) || ((const U8*)s)[2] == 0x6A ) ) ? 3 : 0 )\
+: 0 ) )
+#endif
+
/*
FOLDS_TO_MULTI: characters that fold to multi-char strings for EBCDIC 037
@@ -4942,6 +5041,6 @@ character actually occupies.
* 94644b4e53ef4f2590ef86a261dcd22ffadf79247fcc7f31c4ebf5b19111fd25 lib/unicore/mktables
* a0079d7556b20c2de1fdc3d492797a91b8ec8a06006f94460006185cb6380962 lib/unicore/version
* 0a6b5ab33bb1026531f816efe81aea1a8ffcd34a27cbea37dd6a70a63d73c844 regen/charset_translations.pl
- * 2f36f8fba175fbda370ce07b6539a26ede81b6c1489df04e62f74ca9c3cecef1 regen/regcharclass.pl
- * 1ce43daf54162d0a267412453ce2a613f60f3a7631365775cc52d34213251a83 regen/regcharclass_multi_char_folds.pl
+ * cf707f1b19b018e6cf8a3e2228f3448ec31b20843d89e388beb471e809d0b5f5 regen/regcharclass.pl
+ * ea16ffd33f6130e7e38a027c21bdea415a7d96efb3741d7cd731243df769581b regen/regcharclass_multi_char_folds.pl
* ex: set ro ft=c: */
diff --git a/regcomp_trie.c b/regcomp_trie.c
index ae2767fc69..f8891af4ea 100644
--- a/regcomp_trie.c
+++ b/regcomp_trie.c
@@ -1838,7 +1838,7 @@ S_ac_map_start_codepoint(pTHX_ U8 *map, const UV cp)
static void
S_ac_map_folded_starts(pTHX_ const reg_trie_data *trie, const U32 state,
U8 *path, const U32 path_len, U8 *map,
- SV **first_folds)
+ SV **first_folds, bool *has_multichar_start)
{
const U32 base = trie->states[state].trans.base;
@@ -1874,10 +1874,12 @@ S_ac_map_folded_starts(pTHX_ const reg_trie_data *trie, const U32 state,
S_ac_map_start_codepoint(aTHX_ map, remaining[i - 1]);
}
*first_folds = add_cp_to_invlist(*first_folds, initial);
+ if (is_MULTI_CHAR_FOLD_START_utf8(folded))
+ *has_multichar_start = TRUE;
}
else {
S_ac_map_folded_starts(aTHX_ trie, next, path, path_len + 1,
- map, first_folds);
+ map, first_folds, has_multichar_start);
}
}
}
@@ -2151,6 +2153,7 @@ Perl_construct_ahocorasick_from_trie(pTHX_ RExC_state_t *pRExC_state, regnode *s
&& !trie->prefixlen_octets && !trie->prefixlen_chars)
{
SV *first_folds = NULL;
+ bool has_multichar_start = FALSE;
U8 path[UTF8_MAXBYTES + 1];
U8 *start_map;
UV start, end;
@@ -2161,22 +2164,28 @@ Perl_construct_ahocorasick_from_trie(pTHX_ RExC_state_t *pRExC_state, regnode *s
aho->prefilter = (U8 *)PerlMemShared_calloc(
REG_AC_PREFILTER_SIZE, sizeof(U8));
start_map = aho->prefilter + REG_AC_PREFILTER_MAP_OFFSET;
- S_ac_map_folded_starts(aTHX_ trie, 1, path, 0, start_map, &first_folds);
+ S_ac_map_folded_starts(aTHX_ trie, 1, path, 0, start_map,
+ &first_folds, &has_multichar_start);
/* A single target codepoint can supply several folded characters.
* Add every such codepoint whose fold begins with a possible first
* folded character. We need not compare the rest of its fold: false
* positives are safe, including folds extending beyond a trie word
* into its following regex nodes. */
- invlist_iterinit(PL_HasMultiCharFold);
- while (invlist_iternext(PL_HasMultiCharFold, &start, &end)) {
- for (UV cp = start; cp <= end; cp++) {
- U8 folded[UTF8_MAXBYTES_CASE + 1];
- STRLEN folded_len;
- const UV initial = to_uni_fold(cp, folded, &folded_len);
-
- if (first_folds && invlist_contains_cp_(first_folds, initial))
- S_ac_map_start_codepoint(aTHX_ start_map, cp);
+ if (has_multichar_start) {
+ invlist_iterinit(PL_HasMultiCharFold);
+ while (invlist_iternext(PL_HasMultiCharFold, &start, &end)) {
+ for (UV cp = start; cp <= end; cp++) {
+ U8 folded[UTF8_MAXBYTES_CASE + 1];
+ STRLEN folded_len;
+ const UV initial = to_uni_fold(cp, folded, &folded_len);
+
+ if (first_folds
+ && invlist_contains_cp_(first_folds, initial))
+ {
+ S_ac_map_start_codepoint(aTHX_ start_map, cp);
+ }
+ }
}
}
SvREFCNT_dec(first_folds);
diff --git a/regen/regcharclass.pl b/regen/regcharclass.pl
index 6b0ba5882f..d03043705a 100755
--- a/regen/regcharclass.pl
+++ b/regen/regcharclass.pl
@@ -2205,6 +2205,12 @@ Sometimes we are interested in just the first two of these.
#=> LATIN1 :safe
#%regcharclass_multi_char_folds::multi_char_folds('l', 'fm')
+MULTI_CHAR_FOLD_START: Q : The first character of a multi-character fold
+This character can begin the full fold of a single character that folds to
+multiple characters. The input is already folded.
+=> UTF8 :fast
+%regcharclass_multi_char_folds::multi_char_fold_starts()
+
FOLDS_TO_MULTI: Q : characters that fold to multi-char strings
$multi_overview
=> UTF8 :fast
diff --git a/regen/regcharclass_multi_char_folds.pl b/regen/regcharclass_multi_char_folds.pl
index 360eb2610a..4d63ea8223 100644
--- a/regen/regcharclass_multi_char_folds.pl
+++ b/regen/regcharclass_multi_char_folds.pl
@@ -171,4 +171,11 @@ sub multi_char_folds ($$) {
return %output_folds;
}
+sub multi_char_fold_starts {
+ my (undef, $folds, $format) = prop_invmap("Case_Folding");
+ die "Unexpected Case_Folding format" unless $format eq 'al';
+ my %heads = map { $_->[0] => 1 } grep { ref $_ } @$folds;
+ return map { sprintf('"\\x{%X}"', $_) => $_ } sort { $a <=> $b } keys %heads;
+}
+
1