Commit 5fe24f6dd3 for aom
commit 5fe24f6dd3ecdea27746396d61aca149732e1ae9
Author: Hang Nan <nanx95726@gmail.com>
Date: Wed Aug 19 19:13:12 2026 +0800
Add AV1 encoder public API fuzzer
Add av1_enc_fuzzer, a libFuzzer target for the AV1 encoder public API,
modeled after the existing av1_dec_fuzzer in this directory. The target
exercises:
- aom_codec_enc_init() with good/realtime/all-intra usage modes
- encoder controls via AOM_CODEC_CONTROL_TYPECHECKED
- one-frame and two-frame encode flows
- mid-stream aom_codec_enc_config_set() reconfiguration
- row-mt and all-intra encoder paths
- packet draining and aom_codec_destroy() teardown
Also add a build script for the new fuzzer, matching the decoder fuzzer
build script in this directory.
The fuzzer returns 0 for all inputs and ignores invalid encoder
configurations.
Change-Id: I16f9ebc38d6785e081a8ca80825bc7e63bbcfec5
diff --git a/examples/av1_enc_fuzzer.cc b/examples/av1_enc_fuzzer.cc
new file mode 100644
index 0000000000..fed3e6d841
--- /dev/null
+++ b/examples/av1_enc_fuzzer.cc
@@ -0,0 +1,329 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+/*
+ * See build_av1_enc_fuzzer.sh for building instructions.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "aom/aom_encoder.h"
+#include "aom/aom_image.h"
+#include "aom/aomcx.h"
+
+namespace {
+
+constexpr unsigned int kMaxDimension = 1024;
+constexpr size_t kMinHeaderSize = 16;
+
+// Bit masks derived from the mode_flags byte. The low two bits are used to
+// select the encoding usage (see PickUsage).
+enum ModeFlag {
+ kLossless = 1u << 2,
+ kRowMt = 1u << 3,
+ kUseSecondResolution = 1u << 4,
+ kKeyFrameSecond = 1u << 5,
+ kErrorResilient = 1u << 6,
+};
+
+// Number of frames to encode for a given cpu_used value. Lower cpu_used
+// values encode more slowly, so fewer frames are used to stay within the
+// fuzzer's execution time budget (libFuzzer defaults to one second per
+// input). This can be refined based on measured fuzzer performance.
+constexpr unsigned int kNumFramesForCpuUsed[12] = {
+ 50, // cpu_used 0
+ 50, // cpu_used 1
+ 50, // cpu_used 2
+ 60, // cpu_used 3
+ 60, // cpu_used 4
+ 70, // cpu_used 5
+ 70, // cpu_used 6
+ 80, // cpu_used 7
+ 80, // cpu_used 8
+ 90, // cpu_used 9
+ 90, // cpu_used 10
+ 100 // cpu_used 11
+};
+
+struct FuzzReader {
+ const uint8_t *data;
+ size_t size;
+};
+
+uint8_t ReadU8(FuzzReader *reader) {
+ if (reader->size == 0) return 0;
+ const uint8_t value = *reader->data++;
+ --reader->size;
+ return value;
+}
+
+uint16_t ReadU16(FuzzReader *reader) {
+ if (reader->size < 2) return 0;
+ const uint16_t value = static_cast<uint16_t>(reader->data[0]) |
+ static_cast<uint16_t>(reader->data[1] << 8);
+ reader->data += 2;
+ reader->size -= 2;
+ return value;
+}
+
+unsigned int UsageToIndex(unsigned int usage) {
+ switch (usage) {
+ case AOM_USAGE_GOOD_QUALITY: return 0;
+ case AOM_USAGE_REALTIME: return 1;
+ case AOM_USAGE_ALL_INTRA: return 2;
+ default: return 0;
+ }
+}
+
+unsigned int PickUsage(uint8_t raw) {
+ switch (raw % 3) {
+ case 0: return AOM_USAGE_GOOD_QUALITY;
+ case 1: return AOM_USAGE_REALTIME;
+ default: return AOM_USAGE_ALL_INTRA;
+ }
+}
+
+aom_rc_mode PickRcMode(uint8_t raw) {
+ switch (raw % 4) {
+ case 0: return AOM_VBR;
+ case 1: return AOM_CBR;
+ case 2: return AOM_CQ;
+ default: return AOM_Q;
+ }
+}
+
+unsigned int PickDimension(uint16_t raw) { return 1u + (raw % kMaxDimension); }
+
+int PickCpuUsed(unsigned int usage, uint8_t raw) {
+ const int max_cpu_used = usage == AOM_USAGE_REALTIME ? 11 : 9;
+ return raw % (max_cpu_used + 1);
+}
+
+void DrainPackets(aom_codec_ctx_t *codec) {
+ aom_codec_iter_t iter = nullptr;
+ while (aom_codec_get_cx_data(codec, &iter) != nullptr) {
+ }
+}
+
+bool InitDefaultConfig(aom_codec_iface_t *iface, unsigned int usage,
+ aom_codec_enc_cfg_t *cfg) {
+ const unsigned int usages[3] = { AOM_USAGE_GOOD_QUALITY, AOM_USAGE_REALTIME,
+ AOM_USAGE_ALL_INTRA };
+ const unsigned int requested_index = UsageToIndex(usage);
+ for (unsigned int i = 0; i < 3; ++i) {
+ const unsigned int index = (requested_index + i) % 3;
+ if (aom_codec_enc_config_default(iface, cfg, usages[index]) ==
+ AOM_CODEC_OK) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Deterministic PRNG so that the frame content is derived from the fuzzer
+// input without requiring the reader to supply a full plane-sized chunk of
+// data for every frame.
+uint32_t Rand(uint32_t *state) {
+ *state = *state * 1664525u + 1013904223u;
+ return *state;
+}
+
+void FillPlane(uint8_t *plane, int stride, unsigned int width,
+ unsigned int height, uint32_t *state) {
+ for (unsigned int row = 0; row < height; ++row) {
+ for (unsigned int col = 0; col < width; ++col) {
+ plane[static_cast<size_t>(row) * stride + col] =
+ static_cast<uint8_t>(Rand(state) >> 24);
+ }
+ }
+}
+
+bool BuildImage(FuzzReader *reader, unsigned int width, unsigned int height,
+ aom_image_t *image, unsigned int frame_index) {
+ if (aom_img_alloc(image, AOM_IMG_FMT_I420, width, height, 1) == nullptr) {
+ return false;
+ }
+
+ // Seed a deterministic PRNG from the fuzzer input and the frame index so
+ // that each encoded frame differs without consuming a plane-sized chunk of
+ // input per frame.
+ uint32_t state = 0;
+ for (unsigned int i = 0; i < 4; ++i) {
+ state = (state << 8) | ReadU8(reader);
+ }
+ const uint32_t golden_ratio = 0x9e3779b9u;
+ state ^= frame_index * golden_ratio;
+
+ const unsigned int uv_width = (width + 1) / 2;
+ const unsigned int uv_height = (height + 1) / 2;
+ FillPlane(image->planes[AOM_PLANE_Y], image->stride[AOM_PLANE_Y], width,
+ height, &state);
+ FillPlane(image->planes[AOM_PLANE_U], image->stride[AOM_PLANE_U], uv_width,
+ uv_height, &state);
+ FillPlane(image->planes[AOM_PLANE_V], image->stride[AOM_PLANE_V], uv_width,
+ uv_height, &state);
+ return true;
+}
+
+void ApplyControls(aom_codec_ctx_t *codec, unsigned int usage,
+ uint8_t mode_flags, uint8_t cpu_used_raw, uint8_t ctl0,
+ uint8_t ctl1) {
+ const int cpu_used = PickCpuUsed(usage, cpu_used_raw);
+ const unsigned int lossless = (mode_flags & kLossless) != 0;
+ const unsigned int row_mt = (mode_flags & kRowMt) != 0;
+ const unsigned int aq_mode = ctl0 % 4;
+ const unsigned int deltaq_mode = ctl1 % 4;
+ const unsigned int tile_columns = (ctl0 >> 4) & 0x3;
+ const unsigned int tile_rows = (ctl1 >> 4) & 0x3;
+ const unsigned int enable_cdef = (ctl0 >> 2) & 1;
+ const unsigned int enable_restoration = (ctl1 >> 2) & 1;
+
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AOME_SET_CPUUSED, cpu_used);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_LOSSLESS, lossless);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_ROW_MT, row_mt);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_AQ_MODE, aq_mode);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_DELTAQ_MODE, deltaq_mode);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_TILE_COLUMNS,
+ tile_columns);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_TILE_ROWS, tile_rows);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_ENABLE_CDEF, enable_cdef);
+ (void)AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1E_SET_ENABLE_RESTORATION,
+ enable_restoration);
+}
+
+bool EncodeFrame(aom_codec_ctx_t *codec, FuzzReader *reader, unsigned int width,
+ unsigned int height, aom_codec_pts_t pts,
+ aom_enc_frame_flags_t flags) {
+ aom_image_t image;
+ memset(&image, 0, sizeof(image));
+
+ if (!BuildImage(reader, width, height, &image, pts)) {
+ return false;
+ }
+
+ const aom_codec_err_t enc_ret =
+ aom_codec_encode(codec, &image, pts, 1, flags);
+ if (enc_ret == AOM_CODEC_OK) {
+ DrainPackets(codec);
+ }
+
+ aom_img_free(&image);
+ return true;
+}
+
+} // namespace
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ if (iface == nullptr || size < kMinHeaderSize) return 0;
+
+ FuzzReader reader = { data, size };
+ const uint8_t mode_flags = ReadU8(&reader);
+ const uint8_t cpu_used_raw = ReadU8(&reader);
+ const uint8_t ctl0 = ReadU8(&reader);
+ const uint8_t ctl1 = ReadU8(&reader);
+ const unsigned int usage = PickUsage(mode_flags);
+ const unsigned int width0 = PickDimension(ReadU16(&reader));
+ const unsigned int height0 = PickDimension(ReadU16(&reader));
+ const unsigned int bitrate = 1u + (ReadU16(&reader) % 4000u);
+ const unsigned int width1 = PickDimension(ReadU16(&reader));
+ const unsigned int height1 = PickDimension(ReadU16(&reader));
+ const unsigned int threads0 = ReadU8(&reader) % 9;
+ const unsigned int threads1 = ReadU8(&reader) % 9;
+
+ aom_codec_enc_cfg_t cfg;
+ if (!InitDefaultConfig(iface, usage, &cfg)) return 0;
+
+ cfg.g_usage = usage;
+ cfg.g_w = width0;
+ cfg.g_h = height0;
+ cfg.g_threads = threads0;
+ cfg.g_forced_max_frame_width = kMaxDimension;
+ cfg.g_forced_max_frame_height = kMaxDimension;
+ cfg.g_timebase.num = 1;
+ cfg.g_timebase.den = 1000000;
+ cfg.g_pass = AOM_RC_ONE_PASS;
+ cfg.g_lag_in_frames = 0;
+ cfg.g_error_resilient =
+ (mode_flags & kErrorResilient) ? AOM_ERROR_RESILIENT_DEFAULT : 0;
+ cfg.rc_end_usage = PickRcMode(ctl0);
+ cfg.rc_target_bitrate = bitrate;
+
+ aom_codec_ctx_t codec;
+ memset(&codec, 0, sizeof(codec));
+ if (aom_codec_enc_init(&codec, iface, &cfg, 0) != AOM_CODEC_OK) return 0;
+
+ const int cpu_used = PickCpuUsed(usage, cpu_used_raw);
+ ApplyControls(&codec, usage, mode_flags, cpu_used_raw, ctl0, ctl1);
+
+ const unsigned int num_frames = kNumFramesForCpuUsed[cpu_used];
+ bool config_failed = false;
+ for (unsigned int frame = 0; frame < num_frames; ++frame) {
+ aom_enc_frame_flags_t flags = 0;
+ if (frame == 1 && (mode_flags & kKeyFrameSecond)) {
+ flags |= AOM_EFLAG_FORCE_KF;
+ }
+
+ unsigned int width = width0;
+ unsigned int height = height0;
+ if (mode_flags & kUseSecondResolution) {
+ // Vary the switch frame based on the input so different encodings
+ // exercise the resolution change at different points in the stream.
+ // num_frames >= 50 for every cpu_used, so switch_frame and
+ // restore_frame stay within [1, num_frames - 1].
+ const unsigned int switch_frame = 1u + (ctl0 % (num_frames / 4u));
+ const unsigned int restore_frame =
+ switch_frame + 1u + (ctl1 % (num_frames / 4u));
+ if (frame == switch_frame) {
+ cfg.g_w = width1;
+ cfg.g_h = height1;
+ cfg.g_threads = threads1;
+ cfg.rc_end_usage = PickRcMode(ctl1);
+ if (aom_codec_enc_config_set(&codec, &cfg) != AOM_CODEC_OK) {
+ config_failed = true;
+ break;
+ }
+ width = width1;
+ height = height1;
+ } else if (frame == restore_frame) {
+ cfg.g_w = width0;
+ cfg.g_h = height0;
+ cfg.g_threads = threads0;
+ cfg.rc_end_usage = PickRcMode(ctl0);
+ if (aom_codec_enc_config_set(&codec, &cfg) != AOM_CODEC_OK) {
+ config_failed = true;
+ break;
+ }
+ width = width0;
+ height = height0;
+ }
+ }
+
+ if (!EncodeFrame(&codec, &reader, width, height, frame, flags)) {
+ config_failed = true;
+ break;
+ }
+ }
+
+ if (!config_failed) {
+ const aom_codec_err_t flush_ret =
+ aom_codec_encode(&codec, nullptr, 0, 0, 0);
+ if (flush_ret == AOM_CODEC_OK) {
+ DrainPackets(&codec);
+ }
+ }
+
+ (void)aom_codec_destroy(&codec);
+ return 0;
+}
diff --git a/examples/build_av1_enc_fuzzer.sh b/examples/build_av1_enc_fuzzer.sh
new file mode 100755
index 0000000000..a11de1652f
--- /dev/null
+++ b/examples/build_av1_enc_fuzzer.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+#
+# This source code is subject to the terms of the BSD 2 Clause License and
+# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+# was not distributed with this source code in the LICENSE file, you can
+# obtain it at www.aomedia.org/license/software. If the Alliance for Open
+# Media Patent License 1.0 was not distributed with this source code in the
+# PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+#
+###############################################################################
+# Fuzzer for libaom AV1 encoder public API.
+# =========================================
+# Requirements
+# ---------------------
+# Clang6.0 or above (must support -fsanitize=fuzzer -fsanitize=fuzzer-no-link)
+#
+# References:
+# ---------------------
+# http://llvm.org/docs/LibFuzzer.html
+# https://github.com/google/oss-fuzz
+#
+# Steps to build / run
+# ---------------------
+
+set -eu
+
+# Have a copy of AOM and a build directory ready.
+if [[ $# -ne 2 ]]; then
+ echo "Pass in the AOM source tree as first argument, and a build directory "
+ echo "as the second argument. The AOM source tree can be obtained via: "
+ echo " git clone https://aomedia.googlesource.com/aom"
+ exit 2
+fi
+if [[ -z "${CC:-}" ]]; then
+ echo "Set the CC environment variable to point to your C compiler."
+ exit 2
+fi
+if [[ -z "${CXX:-}" ]]; then
+ echo "Set the CXX environment variable to point to your C++ compiler."
+ exit 2
+fi
+
+AOM_DIR=$1
+BUILD_DIR=$2
+# Run CMake with address sanitizer enabled and build the codec.
+# Enable DO_RANGE_CHECK_CLAMP to suppress the noise of integer overflows
+# in the transform functions. Also set memory limits.
+EXTRA_C_FLAGS='-UNDEBUG -DDO_RANGE_CHECK_CLAMP=1 -DAOM_MAX_ALLOCABLE_MEMORY=1073741824'
+cd "${BUILD_DIR}"
+cmake "${AOM_DIR}" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONFIG_PIC=1 \
+ -DCONFIG_AV1_ENCODER=1 \
+ -DENABLE_APPS=0 -DENABLE_EXAMPLES=0 -DENABLE_DOCS=0 -DENABLE_TESTS=0 \
+ -DAOM_EXTRA_C_FLAGS="${EXTRA_C_FLAGS}" \
+ -DAOM_EXTRA_CXX_FLAGS="${EXTRA_C_FLAGS}" -DSANITIZE=fuzzer-no-link,address
+
+# Build the codec.
+make -j$(nproc)
+
+# Build the av1 encoder fuzzer
+$CXX -std=c++17 -I${AOM_DIR} -I${BUILD_DIR} \
+ -g -fsanitize=fuzzer,address \
+ ${AOM_DIR}/examples/av1_enc_fuzzer.cc -o ${BUILD_DIR}/av1_enc_fuzzer \
+ ${BUILD_DIR}/libaom.a
+
+echo "Fuzzer built at ${BUILD_DIR}/av1_enc_fuzzer."
+echo "Create a corpus directory, and run:"
+echo " av1_enc_fuzzer CORPUS_DIR"