Commit 3c69dcf for zlib
commit 3c69dcf78c1cac3ea4c9a2b0063d463634c3b304
Author: Mark Adler <git@madler.net>
Date: Sun Nov 15 11:08:42 2026 -0800
Unroll the byte loops in the braided crc32_z() calculation.
The loops over crc32 braid bytes had a comment: "This loop should
get unrolled." Unfortunately the compilers never read my comments,
and in fact gcc -O2 would not unroll the loop. Though gcc -O3 and
clang -O2 would unroll. While the build scripts in zlib use -O3, it
is often built out in the wild using gcc -O2, resulting in about a
factor of two hit on the speed of crc32().
Now the loops are explicitly unrolled in the code, so the compiler
no longer has a say in the matter. Also a single expression is now
used for the exclusive-or's to get the next CRC value, instead of a
sequence of ^= expressions, one for each byte. This increased the
speed of clang-compiled code by as much as 20%.
diff --git a/crc32.c b/crc32.c
index 32b1d04..3e418e8 100644
--- a/crc32.c
+++ b/crc32.c
@@ -620,6 +620,24 @@ local z_word_t crc_word_big(z_word_t data) {
return data;
}
+/*
+ The CRC of one braid's word w, using braid table t: the table entries for
+ its W bytes, combined in pairs. This is written out rather than looped over
+ the bytes, since gcc at -O2 does not unroll such a loop, which leaves the
+ shift and the table offset to be computed for each byte, and is up to twice
+ as slow.
+ */
+#define CRCB(t, w, k) t[k][((w) >> ((k) << 3)) & 0xff]
+#if W == 8
+# define CRCWORD(t, w) (((CRCB(t, w, 0) ^ CRCB(t, w, 1)) ^ \
+ (CRCB(t, w, 2) ^ CRCB(t, w, 3))) ^ \
+ ((CRCB(t, w, 4) ^ CRCB(t, w, 5)) ^ \
+ (CRCB(t, w, 6) ^ CRCB(t, w, 7))))
+#else
+# define CRCWORD(t, w) ((CRCB(t, w, 0) ^ CRCB(t, w, 1)) ^ \
+ (CRCB(t, w, 2) ^ CRCB(t, w, 3)))
+#endif
+
#endif
/* ========================================================================= */
@@ -641,7 +659,6 @@ uLong ZEXPORT crc32_z(uLong crc, const unsigned char FAR *buf, z_size_t len) {
z_size_t blks;
z_word_t const *words;
unsigned endian;
- int k;
/* Compute the CRC up to a z_word_t boundary. */
while (len && ((z_size_t)buf & (W - 1)) != 0) {
@@ -727,42 +744,23 @@ uLong ZEXPORT crc32_z(uLong crc, const unsigned char FAR *buf, z_size_t len) {
#endif
words += N;
- /* Compute and update the CRC for each word. The loop should
- get unrolled. */
- crc0 = crc_braid_table[0][word0 & 0xff];
+ /* Compute and update the CRC for each word. */
+ crc0 = CRCWORD(crc_braid_table, word0);
#if N > 1
- crc1 = crc_braid_table[0][word1 & 0xff];
+ crc1 = CRCWORD(crc_braid_table, word1);
#if N > 2
- crc2 = crc_braid_table[0][word2 & 0xff];
+ crc2 = CRCWORD(crc_braid_table, word2);
#if N > 3
- crc3 = crc_braid_table[0][word3 & 0xff];
+ crc3 = CRCWORD(crc_braid_table, word3);
#if N > 4
- crc4 = crc_braid_table[0][word4 & 0xff];
+ crc4 = CRCWORD(crc_braid_table, word4);
#if N > 5
- crc5 = crc_braid_table[0][word5 & 0xff];
-#endif
-#endif
+ crc5 = CRCWORD(crc_braid_table, word5);
#endif
#endif
-#endif
- for (k = 1; k < W; k++) {
- crc0 ^= crc_braid_table[k][(word0 >> (k << 3)) & 0xff];
-#if N > 1
- crc1 ^= crc_braid_table[k][(word1 >> (k << 3)) & 0xff];
-#if N > 2
- crc2 ^= crc_braid_table[k][(word2 >> (k << 3)) & 0xff];
-#if N > 3
- crc3 ^= crc_braid_table[k][(word3 >> (k << 3)) & 0xff];
-#if N > 4
- crc4 ^= crc_braid_table[k][(word4 >> (k << 3)) & 0xff];
-#if N > 5
- crc5 ^= crc_braid_table[k][(word5 >> (k << 3)) & 0xff];
#endif
#endif
#endif
-#endif
-#endif
- }
}
/*
@@ -849,42 +847,23 @@ uLong ZEXPORT crc32_z(uLong crc, const unsigned char FAR *buf, z_size_t len) {
#endif
words += N;
- /* Compute and update the CRC for each word. The loop should
- get unrolled. */
- crc0 = crc_braid_big_table[0][word0 & 0xff];
-#if N > 1
- crc1 = crc_braid_big_table[0][word1 & 0xff];
-#if N > 2
- crc2 = crc_braid_big_table[0][word2 & 0xff];
-#if N > 3
- crc3 = crc_braid_big_table[0][word3 & 0xff];
-#if N > 4
- crc4 = crc_braid_big_table[0][word4 & 0xff];
-#if N > 5
- crc5 = crc_braid_big_table[0][word5 & 0xff];
-#endif
-#endif
-#endif
-#endif
-#endif
- for (k = 1; k < W; k++) {
- crc0 ^= crc_braid_big_table[k][(word0 >> (k << 3)) & 0xff];
+ /* Compute and update the CRC for each word. */
+ crc0 = CRCWORD(crc_braid_big_table, word0);
#if N > 1
- crc1 ^= crc_braid_big_table[k][(word1 >> (k << 3)) & 0xff];
+ crc1 = CRCWORD(crc_braid_big_table, word1);
#if N > 2
- crc2 ^= crc_braid_big_table[k][(word2 >> (k << 3)) & 0xff];
+ crc2 = CRCWORD(crc_braid_big_table, word2);
#if N > 3
- crc3 ^= crc_braid_big_table[k][(word3 >> (k << 3)) & 0xff];
+ crc3 = CRCWORD(crc_braid_big_table, word3);
#if N > 4
- crc4 ^= crc_braid_big_table[k][(word4 >> (k << 3)) & 0xff];
+ crc4 = CRCWORD(crc_braid_big_table, word4);
#if N > 5
- crc5 ^= crc_braid_big_table[k][(word5 >> (k << 3)) & 0xff];
+ crc5 = CRCWORD(crc_braid_big_table, word5);
#endif
#endif
#endif
#endif
#endif
- }
}
/*