Commit 4b8fece0f1 for openssl.org
commit 4b8fece0f1bc9bcc693773f721fbb4acadc1ad43
Author: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Date: Mon Jun 29 22:46:21 2026 +0200
mfail: add sampled and count-only modes to test driver
Sampled tests cap allocation-failure injection at a fixed number of
sampled points, running exhaustively when the allocation count is below
that. Non-sampled tests fall back to counting only on non-cached-fetch
builds, where exhaustive injection is impractical.
Assisted-by: Claude:claude-opus-4-8
Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Mon Jul 13 09:43:50 2026
(Merged from https://github.com/openssl/openssl/pull/31780)
diff --git a/test/README.md b/test/README.md
index 4bc558817f..1575fb69dc 100644
--- a/test/README.md
+++ b/test/README.md
@@ -196,6 +196,13 @@ return 0 when a failure was triggered. The `ADD_MFAIL_ALL_TESTS` and
`ADD_MFAIL_ALL_NO_CHECK_TESTS` variants apply the same cycle to each index
of a parameterised test, the same way `ADD_ALL_TESTS` does.
+The `ADD_MFAIL_SAMPLED_TEST(fn, cnt)` variant (and its `NO_CHECK` and `ALL`
+forms) caps injection at `cnt` points: it injects at every allocation when
+there are at most `cnt` of them, and otherwise samples `cnt` points spread
+across the run. This keeps tests with very many allocations bounded. On
+`no-cached-fetch` builds, where allocation counts explode, non-sampled tests
+run the counting phase only and skip injection; sampled tests still run.
+
An mfail test returns 1 on success or 0 on failure; under `NO_CHECK` a 0 is
tolerated since a function may legitimately fail when an allocation fails.
Returning -1 forces a failure that is reported even under `NO_CHECK`, for
@@ -212,6 +219,11 @@ Behavior is controlled with the following environment variables:
OPENSSL_TEST_MFAIL_SLOW=N Slow threshold (default 1000).
+ OPENSSL_TEST_MFAIL_COUNT=N Override the sampled point count, taking
+ precedence over the per-test value.
+
+ OPENSSL_TEST_MFAIL_COUNT_ONLY=1 Run the counting phase only, never inject.
+
OPENSSL_TEST_MFAIL_POINT=N Run only failure point N (0-indexed),
useful for debugging a specific failure.
diff --git a/test/mfail/mfail.c b/test/mfail/mfail.c
index 254a6a965f..ec391c8e85 100644
--- a/test/mfail/mfail.c
+++ b/test/mfail/mfail.c
@@ -38,6 +38,9 @@ static struct {
int single_point;
int start_point;
int env_count;
+ int count_only;
+ int count_only_env;
+ int sample_count;
int slow_threshold;
int print_bt;
int mode;
@@ -155,11 +158,13 @@ int mfail_install(int optional)
mf.single_point = env_int("OPENSSL_TEST_MFAIL_POINT", -1);
mf.start_point = env_int("OPENSSL_TEST_MFAIL_START", 0);
mf.env_count = env_int("OPENSSL_TEST_MFAIL_COUNT", 0);
+ mf.count_only_env = env_is_true("OPENSSL_TEST_MFAIL_COUNT_ONLY");
mf.slow_threshold = env_int("OPENSSL_TEST_MFAIL_SLOW", 1000);
mf.print_bt = env_is_true("OPENSSL_TEST_MFAIL_BACKTRACE");
/* if optional and nothing configured, then no point installing hooks */
- if (optional && mf.env_count <= 0 && mf.single_point < 0)
+ if (optional && mf.env_count <= 0 && mf.single_point < 0
+ && !mf.count_only_env)
return 0;
if (!CRYPTO_set_mem_functions(mf_malloc, mf_realloc, mf_free))
@@ -208,6 +213,11 @@ static int compute_point(int i, int total, int n, int seq, int start)
}
void mfail_init(int seq, int flags)
+{
+ mfail_init_ex(seq, flags, 0);
+}
+
+void mfail_init_ex(int seq, int flags, int count)
{
mf.seq = seq;
mf.iter_index = 0;
@@ -221,10 +231,13 @@ void mfail_init(int seq, int flags)
mf.counting = 0;
mf.slow_skipped = 0;
mf.no_check = flags & MFAIL_FLAG_NO_CHECK;
+ mf.count_only = mf.count_only_env || (flags & MFAIL_FLAG_COUNT_ONLY) != 0;
+ /* env count overrides the requested per-test sample count */
+ mf.sample_count = mf.env_count > 0 ? mf.env_count : count;
if (mf.single_point >= 0) {
mf.mode = MFAIL_MODE_SINGLE;
- } else if ((flags & MFAIL_FLAG_COUNT) && mf.env_count > 0) {
+ } else if ((flags & MFAIL_FLAG_COUNT) && mf.sample_count > 0) {
mf.mode = MFAIL_MODE_SAMPLED;
} else {
mf.mode = MFAIL_MODE_EXHAUSTIVE;
@@ -240,7 +253,13 @@ int mfail_has_next(void)
switch (mf.phase) {
case MFAIL_PHASE_COUNTING:
mf.total = mf.alloc_count;
- if (mf.skip_slow && mf.total > mf.slow_threshold) {
+ if (mf.count_only) {
+ mf.phase = MFAIL_PHASE_DONE;
+ break;
+ }
+ /* sampled runs are bounded, so slow-skip only applies otherwise */
+ if (mf.skip_slow && mf.mode != MFAIL_MODE_SAMPLED
+ && mf.total > mf.slow_threshold) {
mf.slow_skipped = 1;
mf.phase = MFAIL_PHASE_DONE;
break;
@@ -256,7 +275,7 @@ int mfail_has_next(void)
mf.phase = MFAIL_PHASE_DONE;
}
} else { /* mf.mode is MFAIL_MODE_SAMPLED */
- mf.n = mf.env_count;
+ mf.n = mf.sample_count;
if (mf.n > mf.total)
mf.n = mf.total;
if (mf.n > 0 && mf.total > mf.start_point) {
@@ -336,6 +355,11 @@ int mfail_was_slow_skipped(void)
return mf.slow_skipped;
}
+int mfail_is_count_only(void)
+{
+ return mf.count_only;
+}
+
int mfail_get_count(void)
{
return mf.alloc_count;
diff --git a/test/mfail/mfail.h b/test/mfail/mfail.h
index 6c16e4fe65..c93be44fca 100644
--- a/test/mfail/mfail.h
+++ b/test/mfail/mfail.h
@@ -13,6 +13,8 @@
/* Flags for mfail_init(). */
#define MFAIL_FLAG_COUNT (1 << 0)
#define MFAIL_FLAG_NO_CHECK (1 << 1)
+/* Force count-only run for this init */
+#define MFAIL_FLAG_COUNT_ONLY (1 << 2)
/* Modes */
#define MFAIL_MODE_EXHAUSTIVE 0
@@ -30,6 +32,8 @@ int mfail_install(int optional);
int mfail_is_installed(void);
/* Initialize the mfail for test case runs */
void mfail_init(int seq, int flags);
+/* As mfail_init() but caps injection at |count| sampled points */
+void mfail_init_ex(int seq, int flags, int count);
/* Check for the failure loop if another fail execution should be done */
int mfail_has_next(void);
/* Start the failure triggering block */
@@ -40,6 +44,8 @@ void mfail_end(void);
int mfail_was_triggered(void);
/* Check if the inject phase was skipped because it got over slow threshold */
int mfail_was_slow_skipped(void);
+/* Check if mfail counts allocations and not inject */
+int mfail_is_count_only(void);
/* If the counting was executed, get the total number of allocations */
int mfail_get_count(void);
/* Get the total number of failure points */
diff --git a/test/testutil.h b/test/testutil.h
index f606ff1f75..7e23f53827 100644
--- a/test/testutil.h
+++ b/test/testutil.h
@@ -73,17 +73,32 @@
/* Per-test flags for add_mfail_test() */
#define MFAIL_TEST_NO_CHECK (1 << 0)
+#define MFAIL_TEST_SAMPLED (1 << 1)
#define ADD_MFAIL_TEST(test_fn) \
- add_mfail_test(#test_fn, test_fn, 0)
+ add_mfail_test(#test_fn, test_fn, 0, 0)
#define ADD_MFAIL_NO_CHECK_TEST(test_fn) \
- add_mfail_test(#test_fn, test_fn, MFAIL_TEST_NO_CHECK)
+ add_mfail_test(#test_fn, test_fn, MFAIL_TEST_NO_CHECK, 0)
+
+/* Caps injection at |cnt| points (exhaustive when allocations <= cnt) */
+#define ADD_MFAIL_SAMPLED_TEST(test_fn, cnt) \
+ add_mfail_test(#test_fn, test_fn, MFAIL_TEST_SAMPLED, cnt)
+#define ADD_MFAIL_SAMPLED_NO_CHECK_TEST(test_fn, cnt) \
+ add_mfail_test(#test_fn, test_fn, \
+ MFAIL_TEST_NO_CHECK | MFAIL_TEST_SAMPLED, cnt)
/* Runs the exhaustive mfail cycle for each 0 <= idx < num */
#define ADD_MFAIL_ALL_TESTS(test_fn, num) \
- add_mfail_all_tests(#test_fn, test_fn, num, 0)
+ add_mfail_all_tests(#test_fn, test_fn, num, 0, 0)
#define ADD_MFAIL_ALL_NO_CHECK_TESTS(test_fn, num) \
- add_mfail_all_tests(#test_fn, test_fn, num, MFAIL_TEST_NO_CHECK)
+ add_mfail_all_tests(#test_fn, test_fn, num, MFAIL_TEST_NO_CHECK, 0)
+
+/* Sampled variants of the above */
+#define ADD_MFAIL_SAMPLED_ALL_TESTS(test_fn, num, cnt) \
+ add_mfail_all_tests(#test_fn, test_fn, num, MFAIL_TEST_SAMPLED, cnt)
+#define ADD_MFAIL_SAMPLED_ALL_NO_CHECK_TESTS(test_fn, num, cnt) \
+ add_mfail_all_tests(#test_fn, test_fn, num, \
+ MFAIL_TEST_NO_CHECK | MFAIL_TEST_SAMPLED, cnt)
/*
* A variant of the same without TAP output.
@@ -256,9 +271,9 @@ void add_test(const char *test_case_name, int (*test_fn)(void));
void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num,
int subtest);
void add_mfail_test(const char *test_case_name, int (*test_fn)(void),
- int flags);
+ int flags, int sampled);
void add_mfail_all_tests(const char *test_case_name, int (*test_fn)(int idx),
- int num, int flags);
+ int num, int flags, int sampled);
#define MFAIL_start mfail_start
#define MFAIL_end mfail_end
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 54aa2a061a..bfcb65b7bc 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -39,6 +39,7 @@ typedef struct test_info {
unsigned int subtest : 1;
unsigned int mfail : 1;
int mfail_flags;
+ int mfail_sampled;
} TEST_INFO;
static TEST_INFO all_tests[1024];
@@ -84,7 +85,8 @@ void add_all_tests(const char *test_case_name, int (*test_fn)(int idx),
num_test_cases += num;
}
-void add_mfail_test(const char *test_case_name, int (*test_fn)(void), int flags)
+void add_mfail_test(const char *test_case_name, int (*test_fn)(void), int flags,
+ int sampled)
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
@@ -92,12 +94,13 @@ void add_mfail_test(const char *test_case_name, int (*test_fn)(void), int flags)
all_tests[num_tests].num = -1;
all_tests[num_tests].mfail = 1;
all_tests[num_tests].mfail_flags = flags;
+ all_tests[num_tests].mfail_sampled = sampled;
++num_tests;
++num_test_cases;
}
void add_mfail_all_tests(const char *test_case_name, int (*test_fn)(int idx),
- int num, int flags)
+ int num, int flags, int sampled)
{
assert(num_tests != OSSL_NELEM(all_tests));
all_tests[num_tests].test_case_name = test_case_name;
@@ -106,6 +109,7 @@ void add_mfail_all_tests(const char *test_case_name, int (*test_fn)(int idx),
all_tests[num_tests].subtest = 1;
all_tests[num_tests].mfail = 1;
all_tests[num_tests].mfail_flags = flags;
+ all_tests[num_tests].mfail_sampled = sampled;
++num_tests;
++num_test_cases;
}
@@ -321,8 +325,24 @@ static int mfail_run_test(const TEST_INFO *t, int idx)
int injections = 0;
int allocations = 0;
int no_check = (t->mfail_flags & MFAIL_TEST_NO_CHECK) != 0;
+ int sampled = (t->mfail_flags & MFAIL_TEST_SAMPLED) != 0;
+ int init_flags = no_check ? MFAIL_FLAG_NO_CHECK : 0;
clock_t start = clock();
+ if (sampled)
+ /* cap injection at mfail_sampled points (exhaustive below that) */
+ init_flags |= MFAIL_FLAG_COUNT;
+#ifdef OPENSSL_NO_CACHED_FETCH
+ else
+ /*
+ * The non-cached does too many allocations, which results in a
+ * significant slowdown of the tests. It does not provide much value,
+ * as it also requires NO_CHECK variant, so just run counting
+ * correctness check and skip the memory failure injection part.
+ */
+ init_flags |= MFAIL_FLAG_COUNT_ONLY;
+#endif
+
level += 4;
test_adjust_streams_tap_level(level);
test_printf_stdout("Subtest: %s[%d]\n", t->test_case_name, idx);
@@ -330,7 +350,7 @@ static int mfail_run_test(const TEST_INFO *t, int idx)
test_flush_stdout();
test_flush_tapout();
- mfail_init(0, no_check ? MFAIL_FLAG_NO_CHECK : 0);
+ mfail_init_ex(idx, init_flags, t->mfail_sampled);
while (mfail_has_next()) {
int phase = mfail_get_phase();
@@ -383,6 +403,8 @@ static int mfail_run_test(const TEST_INFO *t, int idx)
test_verdict(TEST_SKIP_CODE, "2 - injection (mfail not installed)");
else if (mfail_env_skip_all())
test_verdict(TEST_SKIP_CODE, "2 - injection (mfail skip-all set)");
+ else if (mfail_is_count_only())
+ test_verdict(TEST_SKIP_CODE, "2 - injection (count only)");
else if (mfail_was_slow_skipped())
test_verdict(TEST_SKIP_CODE,
"2 - injection (%d allocations exceeds slow threshold %d)",