Commit 4ba7d6ba5 for clamav.net

commit 4ba7d6ba5ae7a0bd99c0e254685d24d0a7ce5eb7
Author: John Humlick <15677335+jhumlick@users.noreply.github.com>
Date:   Wed Nov 18 16:45:47 2026 -0800

    libclamav: Fix XAR limit exits and regression tests

    Mark incomplete XAR scans non-cacheable on limit exits and stop after
    MaxFileSize truncation. Add a same-engine cache regression and strengthen
    gzip extraction tests for partial writes, multi-page input, and aggregate
    scan limits.

    CLAM-3094

diff --git a/libclamav/xar.c b/libclamav/xar.c
index 280748672..0f94e1d3c 100644
--- a/libclamav/xar.c
+++ b/libclamav/xar.c
@@ -586,6 +586,8 @@ int cli_scanxar(cli_ctx *ctx)
      * enforced against the actual decompressed TOC length below. */
     rc = cli_checklimits("cli_scanxar", ctx, 0, 0, 0);
     if (rc != CL_SUCCESS) {
+        if (ctx->abort_scan || rc == CL_ETIMEOUT || rc == CL_EMAXFILES)
+            emax_reached(ctx);
         return rc;
     }

@@ -665,6 +667,7 @@ int cli_scanxar(cli_ctx *ctx)
             break;

         int do_extract_cksum = 1;
+        int maxfilesize_truncated = 0;
         unsigned char *blockp;
         void *a_sc, *e_sc;
         void *a_mc, *e_mc;
@@ -680,8 +683,10 @@ int cli_scanxar(cli_ctx *ctx)

         /* Do not create or extract another member after a file-count limit. */
         rc = cli_checklimits("cli_scanxar", ctx, 0, 0, 0);
-        if (rc == CL_ETIMEOUT || rc == CL_EMAXFILES || ctx->abort_scan)
+        if (rc == CL_ETIMEOUT || rc == CL_EMAXFILES || ctx->abort_scan) {
+            emax_reached(ctx);
             goto exit_reader;
+        }

         at = offset + hdr.toc_length_compressed + hdr.size;

@@ -744,6 +749,7 @@ int cli_scanxar(cli_ctx *ctx)

                         if (out_size > UINT64_MAX - bytes) {
                             cli_dbgmsg("cli_scanxar: decompressed output size overflow.\n");
+                            emax_reached(ctx);
                             rc = CL_EMAXSIZE;
                             break;
                         }
@@ -760,8 +766,10 @@ int cli_scanxar(cli_ctx *ctx)
                                 uint64_t remaining = (out_size < ctx->engine->maxfilesize) ?
                                                          (ctx->engine->maxfilesize - out_size) :
                                                          0;
-                                if (remaining < allowed)
+                                if (remaining < allowed) {
                                     allowed = remaining;
+                                    maxfilesize_truncated = 1;
+                                }
                             }

                             if (ctx->engine->maxscansize) {
@@ -982,12 +990,24 @@ int cli_scanxar(cli_ctx *ctx)
                 }
         } /* end of switch */

-        if (ctx->abort_scan || rc == CL_ETIMEOUT || rc == CL_EMAXFILES)
+        if (ctx->abort_scan || rc == CL_ETIMEOUT || rc == CL_EMAXFILES) {
+            emax_reached(ctx);
             goto exit_tmpfile;
+        }
+
+        /* A truncated member is not scanned and therefore cannot consume a
+         * MaxFiles slot. Stop here so an archive cannot force repeated partial
+         * extractions of oversized members. */
+        if (maxfilesize_truncated) {
+            emax_reached(ctx);
+            goto exit_tmpfile;
+        }

         if (rc == CL_EMAXSIZE && ctx->engine->maxscansize &&
-            ctx->scansize >= ctx->engine->maxscansize)
+            ctx->scansize >= ctx->engine->maxscansize) {
+            emax_reached(ctx);
             goto exit_tmpfile;
+        }

         if (a_hash_ctx != NULL) {
             xar_hash_final(a_hash_ctx, a_hash_result, a_hash);
@@ -1031,6 +1051,8 @@ int cli_scanxar(cli_ctx *ctx)

             rc = cli_magic_scan_desc(fd, tmpname, ctx, NULL, LAYER_ATTRIBUTES_NONE); /// TODO: collect file names in xar_get_toc_data_values()
             if (rc != CL_SUCCESS) {
+                if (ctx->abort_scan || rc == CL_ETIMEOUT || rc == CL_EMAXFILES || rc == CL_EMAXSIZE)
+                    emax_reached(ctx);
                 goto exit_tmpfile;
             }
         }
diff --git a/unit_tests/check_clamav.c b/unit_tests/check_clamav.c
index 02d593649..3b138e606 100644
--- a/unit_tests/check_clamav.c
+++ b/unit_tests/check_clamav.c
@@ -12,6 +12,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <dirent.h>
+#include <zlib.h>
 #if HAVE_UNISTD_H
 #include <unistd.h>
 #endif
@@ -309,6 +310,108 @@ START_TEST(test_cl_scanfile_allscan)
 }
 END_TEST

+START_TEST(test_xar_incomplete_scan_is_not_clean_cached)
+{
+    const char *virname = NULL;
+    const char *member_path = OBJDIR PATHSEP "input" PATHSEP "clamav_hdb_scanfiles" PATHSEP "clam.exe";
+    char xar_path[PATH_MAX];
+    char toc[8192];
+    unsigned char header[28] = {0};
+    unsigned char *member = NULL;
+    unsigned char *compressed_toc = NULL;
+    unsigned long member_size, compressed_toc_size;
+    unsigned long scanned = 0;
+    size_t toc_size;
+    int i, toc_length, added;
+    FILE *member_file = NULL, *xar_file = NULL;
+    struct cl_scan_options options;
+    int ret;
+
+    member_file = fopen(member_path, "rb");
+    ck_assert_msg(member_file != NULL, "open %s", member_path);
+    ck_assert_msg(fseek(member_file, 0, SEEK_END) == 0, "seek %s", member_path);
+    long member_length = ftell(member_file);
+    ck_assert_msg(member_length > 0, "empty %s", member_path);
+    ck_assert_msg(fseek(member_file, 0, SEEK_SET) == 0, "rewind %s", member_path);
+    member_size = (unsigned long)member_length;
+    member = malloc(member_size);
+    ck_assert_msg(member != NULL, "allocate XAR member");
+    ck_assert_msg(fread(member, 1, member_size, member_file) == member_size, "read %s", member_path);
+    fclose(member_file);
+    member_file = NULL;
+
+    toc_length = snprintf(toc, sizeof(toc), "<xar><toc>");
+    ck_assert_msg(toc_length > 0 && (size_t)toc_length < sizeof(toc), "start XAR TOC");
+    for (i = 0; i < 20; i++) {
+        added = snprintf(toc + toc_length, sizeof(toc) - (size_t)toc_length,
+                         "<file><name>clean%d.txt</name><type>file</type><data>"
+                         "<length>5</length><offset>%d</offset><size>5</size>"
+                         "</data></file>", i, i * 5);
+        ck_assert_msg(added > 0 && (size_t)added < sizeof(toc) - (size_t)toc_length,
+                      "append clean member to XAR TOC");
+        toc_length += added;
+    }
+    added = snprintf(toc + toc_length, sizeof(toc) - (size_t)toc_length,
+                     "<file><name>clam.exe</name><type>file</type><data>"
+                     "<length>%lu</length><offset>100</offset><size>%lu</size>"
+                     "</data></file></toc></xar>", member_size, member_size);
+    ck_assert_msg(added > 0 && (size_t)added < sizeof(toc) - (size_t)toc_length,
+                  "append signature member to XAR TOC");
+    toc_length += added;
+    toc_size = (size_t)toc_length;
+    compressed_toc_size = compressBound(toc_size);
+    compressed_toc = malloc(compressed_toc_size);
+    ck_assert_msg(compressed_toc != NULL, "allocate compressed XAR TOC");
+    ck_assert_msg(compress(compressed_toc, &compressed_toc_size,
+                           (const Bytef *)toc, toc_size) == Z_OK,
+                  "compress XAR TOC");
+
+    /* XAR's header fields are big-endian. */
+    header[0] = 0x78; header[1] = 0x61; header[2] = 0x72; header[3] = 0x21;
+    header[5] = 28;
+    for (i = 0; i < 8; i++) {
+        header[8 + i] = (unsigned char)(compressed_toc_size >> (56 - 8 * i));
+        header[16 + i] = (unsigned char)(toc_size >> (56 - 8 * i));
+    }
+
+    snprintf(xar_path, sizeof(xar_path), "%s" PATHSEP "xar-cache-regression.xar", OBJDIR);
+    xar_file = fopen(xar_path, "wb");
+    ck_assert_msg(xar_file != NULL, "create %s", xar_path);
+    ck_assert_msg(fwrite(header, 1, sizeof(header), xar_file) == sizeof(header), "write XAR header");
+    ck_assert_msg(fwrite(compressed_toc, 1, compressed_toc_size, xar_file) == compressed_toc_size,
+                  "write compressed XAR TOC");
+    for (i = 0; i < 20; i++)
+        ck_assert_msg(fwrite("clean", 1, 5, xar_file) == 5, "write clean XAR member %d", i);
+    ck_assert_msg(fwrite(member, 1, member_size, xar_file) == member_size, "write second XAR member");
+    ck_assert_msg(fclose(xar_file) == 0, "close %s", xar_path);
+    xar_file = NULL;
+    free(compressed_toc);
+    free(member);
+
+    memset(&options, 0, sizeof(options));
+    options.parse |= ~0;
+
+    /* MaxFiles allows the first member but blocks later members, so this
+     * incomplete archive result must not be cached as clean. */
+    ck_assert_msg(cl_engine_set_num(g_engine, CL_ENGINE_MAX_FILES, 2) == CL_SUCCESS,
+                  "set low MaxFiles");
+    ret = cl_scanfile(xar_path, &virname, &scanned, g_engine, &options);
+    ck_assert_msg(ret == CL_CLEAN, "first constrained XAR scan: %s", cl_strerror(ret));
+
+    ck_assert_msg(cl_engine_set_num(g_engine, CL_ENGINE_MAX_FILES, 100) == CL_SUCCESS,
+                  "raise MaxFiles");
+    ck_assert_msg(cl_engine_get_num(g_engine, CL_ENGINE_MAX_FILES, NULL) == 100,
+                  "MaxFiles update did not take effect");
+    virname = NULL;
+    scanned = 0;
+    ret = cl_scanfile(xar_path, &virname, &scanned, g_engine, &options);
+    ck_assert_msg(ret == CL_VIRUS, "second XAR scan should detect skipped member: %s", cl_strerror(ret));
+    ck_assert_msg(virname && !strcmp(virname, "ClamAV-Test-File.UNOFFICIAL"),
+                  "virusname: %s", virname ? virname : "(null)");
+    cli_unlink(xar_path);
+}
+END_TEST
+
 START_TEST(test_cl_scanfile_callback)
 {
     const char *virname = NULL;
@@ -1501,6 +1604,7 @@ static Suite *test_cl_suite(void)
     tcase_add_loop_test(tc_cl_scan, test_cl_scandesc_allscan, 0, expect);
     tcase_add_loop_test(tc_cl_scan, test_cl_scanfile, 0, expect);
     tcase_add_loop_test(tc_cl_scan, test_cl_scanfile_allscan, 0, expect);
+    tcase_add_test(tc_cl_scan, test_xar_incomplete_scan_is_not_clean_cached);
     tcase_add_loop_test(tc_cl_scan, test_cl_scandesc_callback, 0, expect);
     tcase_add_loop_test(tc_cl_scan, test_cl_scandesc_callback_allscan, 0, expect);
     tcase_add_loop_test(tc_cl_scan, test_cl_scanfile_callback, 0, expect);
diff --git a/unit_tests/clamscan/xar_test.py b/unit_tests/clamscan/xar_test.py
index 445e9a5c1..cffe59068 100644
--- a/unit_tests/clamscan/xar_test.py
+++ b/unit_tests/clamscan/xar_test.py
@@ -82,6 +82,16 @@ class TC(testcase.TestCase):
     def xar_compress(data):
         return zlib.compress(data)

+    @staticmethod
+    def deterministic_payload(length, start_counter=0):
+        return b''.join(
+            hashlib.sha256(struct.pack('>I', counter)).digest()
+            for counter in range(
+                start_counter,
+                start_counter + (length + hashlib.sha256().digest_size - 1) // hashlib.sha256().digest_size,
+            )
+        )[:length]
+
     def test_declared_large_actual_small_toc_is_scanned(self):
         self.step_name('Test XAR limits use the actual TOC size')

@@ -160,8 +170,12 @@ class TC(testcase.TestCase):
     def test_gzip_extraction_respects_maxfilesize(self):
         self.step_name('Verify that XAR gzip extraction does not write beyond MaxFileSize')

-        maxfilesize = 64 * 1024
-        payload = b'A' * (1024 * 1024)
+        maxfilesize = 65 * 1024
+        # Keep the compressed archive below MaxFileSize while making the
+        # compressed member span several fmap input pages.
+        seed_a = self.deterministic_payload(20 * 1024)
+        seed_b = self.deterministic_payload(20 * 1024, start_counter=10000)
+        payload = seed_a * 16 + seed_b * 16
         compressed = self.xar_compress(payload)
         toc = (
             b'<?xml version="1.0" encoding="UTF-8"?>'
@@ -191,7 +205,7 @@ class TC(testcase.TestCase):
         tempdir = Path(tempfile.mkdtemp(prefix='xar-gzip-maxfilesize-', dir=TC.path_tmp))
         output = self.scan_xar(
             testfile,
-            '--max-filesize=64K --max-scansize=16M --alert-exceeds-max=no '
+            '--max-filesize=65K --max-scansize=16M --alert-exceeds-max=no '
             '--leave-temps --tempdir={}'.format(tempdir),
         )

@@ -200,18 +214,58 @@ class TC(testcase.TestCase):
         assert tempfiles
         extracted = [path for path in tempfiles if path.stat().st_size == maxfilesize]
         assert len(extracted) == 1
-        assert extracted[0].read_bytes() == b'A' * maxfilesize
+        assert extracted[0].read_bytes() == payload[:maxfilesize]
+
+    def test_gzip_extraction_stops_after_oversized_member(self):
+        self.step_name('Verify that XAR stops after a MaxFileSize-truncated member')
+
+        maxfilesize = 129 * 1024
+        seed_a = self.deterministic_payload(20 * 1024)
+        seed_b = self.deterministic_payload(20 * 1024, start_counter=10000)
+        payload = seed_a * 16 + seed_b * 16
+        compressed_members = [self.xar_compress(payload) for _ in range(2)]
+        toc_entries = []
+        offset = 0
+        for index, compressed in enumerate(compressed_members):
+            toc_entries.append(
+                (
+                    f'<file><name>large{index}.bin</name><type>file</type><data>'
+                    f'<length>{len(compressed)}</length><offset>{offset}</offset>'
+                    f'<size>{len(payload)}</size>'
+                    '<encoding style="application/x-gzip"/></data></file>'
+                ).encode()
+            )
+            offset += len(compressed)
+        toc = b'<xar><toc>' + b''.join(toc_entries) + b'</toc></xar>'
+        compressed_toc = zlib.compress(toc)
+        testfile = TC.path_tmp / 'xar-gzip-two-oversized.xar'
+        testfile.write_bytes(
+            struct.pack('>IHHQQI', 0x78617221, 28, 0, len(compressed_toc), len(toc), 0)
+            + compressed_toc + b''.join(compressed_members)
+        )
+        tempdir = Path(tempfile.mkdtemp(prefix='xar-gzip-two-oversized-', dir=TC.path_tmp))
+        output = self.scan_xar(
+            testfile,
+            '--max-files=3 --max-filesize=129K --max-scansize=10M '
+            '--alert-exceeds-max=no --leave-temps --tempdir={}'.format(tempdir),
+        )
+
+        assert output.ec == 0
+        extracted = [path for path in tempdir.rglob('*') if path.is_file()]
+        partial_members = [path for path in extracted if path.stat().st_size == maxfilesize]
+        assert len(partial_members) == 1
+        assert partial_members[0].read_bytes() == payload[:maxfilesize]

     def test_gzip_extraction_respects_aggregate_maxscansize(self):
         self.step_name('Verify that XAR gzip extraction counts truncated members against MaxScanSize')

-        maxfilesize = 64 * 1024
+        maxfilesize = 2 * 1024 * 1024
         maxscansize = 96 * 1024
-        payload = b'A' * (1024 * 1024)
-        compressed_members = [self.xar_compress(payload) for _ in range(2)]
+        payloads = (b'A' * (64 * 1024), b'B' * (1024 * 1024))
+        compressed_members = [self.xar_compress(payload) for payload in payloads]
         toc_entries = []
         offset = 0
-        for index, compressed in enumerate(compressed_members):
+        for index, (payload, compressed) in enumerate(zip(payloads, compressed_members)):
             toc_entries.append(
                 (
                     f'<file><name>payload{index}.bin</name><type>file</type><data>'
@@ -248,19 +302,28 @@ class TC(testcase.TestCase):
         tempdir = Path(tempfile.mkdtemp(prefix='xar-gzip-maxscansize-', dir=TC.path_tmp))
         output = self.scan_xar(
             testfile,
-            '--max-filesize=64K --max-scansize=96K --alert-exceeds-max=no '
-            '--leave-temps --tempdir={}'.format(tempdir),
+            '--max-filesize={} --max-scansize=96K --alert-exceeds-max=no '
+            '--leave-temps --tempdir={}'.format(maxfilesize, tempdir),
         )

         assert output.ec == 0
-        extracted = [
-            path for path in tempdir.rglob('*') if path.is_file() and path.stat().st_size > 1024
-        ]
+        extracted = []
+        for path in tempdir.rglob('*'):
+            if not path.is_file() or path.stat().st_size <= 1024:
+                continue
+            data = path.read_bytes()
+            if data == b'A' * len(data) or data == b'B' * len(data):
+                extracted.append(path)
         assert len(extracted) == 2
         total_extracted = sum(path.stat().st_size for path in extracted)
-        assert maxfilesize < total_extracted <= maxscansize
-        assert all(path.read_bytes() == b'A' * path.stat().st_size for path in extracted)
-        assert max(path.stat().st_size for path in extracted) == maxfilesize
+        sizes = sorted(path.stat().st_size for path in extracted)
+        assert 64 * 1024 < total_extracted <= maxscansize
+        assert sizes[0] > 0
+        assert sizes[1] == 64 * 1024
+        assert all(
+            path.read_bytes() == (b'A' if path.stat().st_size == 64 * 1024 else b'B') * path.stat().st_size
+            for path in extracted
+        )

     def test_gzip_extraction_respects_maxfiles(self):
         self.step_name('Verify that XAR gzip extraction stops at MaxFiles')