Commit 60c87b5e417 for woocommerce
commit 60c87b5e41729ed28bd8f3e144c944a335a716a7
Author: malinajirka <malinajirka@gmail.com>
Date: Tue Jun 30 10:44:16 2026 +0200
Detect and recover stuck POS catalog feed generation jobs (#65919)
* Detect and recover stuck POS catalog feed generation jobs
When a large catalog feed is generated, the whole walk runs inside a single
Action Scheduler action. If the server kills that process (request/PHP timeout
or out of memory) before it finishes, the catch block never runs, so the status
option stays pinned at `in_progress` forever. Nothing reconciles it:
validate_status() only had a staleness check for the `scheduled` state, and
force_regeneration() refused to touch an `in_progress` job. The catalog could
then never be regenerated.
Add a heartbeat (`updated_at`) that is refreshed when a job is scheduled, when
it transitions to `in_progress`, and after every processed batch. validate_status()
now treats an `in_progress` job whose heartbeat is older than a filterable timeout
(woocommerce_product_feed_in_progress_timeout, default 5 minutes) as invalid, so
both status polling and force_regeneration() automatically reschedule a fresh job.
A genuinely running job keeps its heartbeat fresh and is left untouched.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
* Extend per-batch PHP time limit during POS catalog feed generation
ProductWalker already supported resetting PHP's execution time limit per batch
via add_time_limit(), but AsyncGenerator never called it, so generation relied
on whatever max_execution_time the host applied to the Action Scheduler request
(often as low as 30s). Reset the time limit after each batch via the new
woocommerce_product_feed_batch_time_limit filter (default 5 minutes) so a low
max_execution_time does not abort a large catalog mid-walk.
This is a mitigation, not a guarantee: it does not extend Action Scheduler's
300s failure period or a hard host request timeout. The heartbeat-based recovery
remains the safety net when a job is killed regardless of the cause.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
* Raise memory and execution time limits during POS catalog feed generation
The previous change reset PHP's execution time limit after each batch, but the
walker only does so once a batch completes, so the initial product query and the
first batch still ran under whatever (often low) limit the Action Scheduler
request started with. Raise the time limit up front as well, and raise the memory
limit before the walk begins, since large catalogs are memory heavy and out of
memory is a likely cause of the process being killed.
These raises are scoped to the generating process and never lower an existing
higher limit. They cannot override a hard host/server request timeout or Action
Scheduler's failure period, which remain the ceilings that only chunked
processing can remove.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
* Clear the feed timeout filter in tearDown to avoid leaking between tests
test_validate_status_in_progress_timeout_is_filterable() registered the
woocommerce_product_feed_in_progress_timeout filter and removed it only on the
line after the assertion. If that assertion failed, the remove_filter() call
was skipped and the fn() => 10 override leaked into the rest of the process,
making later tests see a 10-second stale threshold instead of the 15-minute
default — a confusing cross-test cascade.
Remove the filter in tearDown() instead, which runs even when a test fails, and
drop the now-redundant inline remove_filter(). Mirrors how the suite already
cleans up other feed filters.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
diff --git a/plugins/woocommerce/changelog/fix-pos-catalog-stuck-in-progress b/plugins/woocommerce/changelog/fix-pos-catalog-stuck-in-progress
new file mode 100644
index 00000000000..79b80beffcd
--- /dev/null
+++ b/plugins/woocommerce/changelog/fix-pos-catalog-stuck-in-progress
@@ -0,0 +1,3 @@
+Significance: patch
+Type: fix
+Comment: Recover POS catalog feed generation jobs that were killed (server timeout or out of memory) and left stuck in the in_progress state, and raise the execution time and memory limits during generation.
diff --git a/plugins/woocommerce/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGenerator.php b/plugins/woocommerce/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGenerator.php
index 7e25266beb7..29ea697e1d0 100644
--- a/plugins/woocommerce/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGenerator.php
+++ b/plugins/woocommerce/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGenerator.php
@@ -112,6 +112,7 @@ class AsyncGenerator {
$status = array(
'scheduled_at' => time(),
+ 'updated_at' => time(),
'completed_at' => null,
'state' => self::STATE_SCHEDULED,
'progress' => 0,
@@ -160,12 +161,44 @@ class AsyncGenerator {
return;
}
- $status['state'] = self::STATE_IN_PROGRESS;
+ $status['state'] = self::STATE_IN_PROGRESS;
+ $status['updated_at'] = time();
update_option( $option_key, $status );
try {
+ // Large catalogs are memory heavy, so give the process as much headroom as the
+ // host allows before the heavy lifting begins. This only raises the limit for the
+ // current process and never lowers an already higher limit.
+ wp_raise_memory_limit( 'admin' );
+
+ /**
+ * Filters the per-batch PHP execution time limit (in seconds) for product feed generation.
+ *
+ * The execution time limit is set to this value up front and reset to it after each processed
+ * batch, so that a low `max_execution_time` does not abort generation part-way through a large
+ * catalog. Return 0 to leave the time limit untouched.
+ *
+ * This only affects PHP's own execution timeout. It does not extend Action Scheduler's
+ * failure period (`action_scheduler_failure_period`, 300 seconds by default) nor any hard
+ * server/host request timeout, so it is a mitigation rather than a guarantee for very large
+ * catalogs.
+ *
+ * @param int $batch_time_limit The per-batch time limit in seconds.
+ *
+ * @since 11.0.0
+ */
+ $batch_time_limit = (int) apply_filters( 'woocommerce_product_feed_batch_time_limit', 5 * MINUTE_IN_SECONDS );
+
+ // Raise the time limit up front too: the walker only resets it after each batch, so the
+ // initial product query and the first batch would otherwise run under whatever (possibly
+ // very low) limit the Action Scheduler request started with.
+ if ( $batch_time_limit > 0 ) {
+ wc_set_time_limit( $batch_time_limit );
+ }
+
$feed = $this->integration->create_feed();
$walker = ProductWalker::from_integration( $this->integration, $feed );
+ $walker->add_time_limit( $batch_time_limit );
// Add dynamic args to the mapper.
$args = $status['args'] ?? array();
@@ -250,6 +283,8 @@ class AsyncGenerator {
return $status;
case self::STATE_IN_PROGRESS:
+ // A genuinely running job (its heartbeat is still fresh, otherwise validate_status()
+ // above would have restarted it) cannot be interrupted mid-flight.
throw new \Exception( 'Feed generation is already in progress and cannot be stopped.' );
case self::STATE_COMPLETED:
@@ -314,11 +349,12 @@ class AsyncGenerator {
* @return array Updated status of the feed generation.
*/
private function update_feed_progress( array $status, WalkerProgress $progress ): array {
- $status['progress'] = $progress->total_count > 0
+ $status['progress'] = $progress->total_count > 0
? round( ( $progress->processed_items / $progress->total_count ) * 100, 2 )
: 0;
- $status['processed'] = $progress->processed_items;
- $status['total'] = $progress->total_count;
+ $status['processed'] = $progress->processed_items;
+ $status['total'] = $progress->total_count;
+ $status['updated_at'] = time();
return $status;
}
@@ -377,6 +413,32 @@ class AsyncGenerator {
return false;
}
+ /**
+ * If the job is in progress but has not updated its heartbeat within the timeout, the
+ * process was most likely killed (server/host timeout or out of memory) before it could
+ * mark itself as failed. Without this check, such a job would stay `in_progress` forever
+ * and no new feed could ever be generated.
+ *
+ * The heartbeat (`updated_at`) is refreshed when the job starts and after every processed
+ * batch, so an active job keeps it fresh while a killed one does not.
+ */
+ if ( self::STATE_IN_PROGRESS === $status['state'] ) {
+ $last_activity = $status['updated_at'] ?? $status['scheduled_at'] ?? 0;
+
+ /**
+ * Allows the timeout for a feed to remain in `in_progress` state without a heartbeat
+ * update to be changed. Past this point the job is treated as stuck and regenerated.
+ *
+ * @param int $stuck_time The stuck time in seconds.
+ * @return int The stuck time in seconds.
+ * @since 11.0.0
+ */
+ $in_progress_timeout = apply_filters( 'woocommerce_product_feed_in_progress_timeout', 5 * MINUTE_IN_SECONDS );
+ if ( time() - $last_activity > $in_progress_timeout ) {
+ return false;
+ }
+ }
+
// All good.
return true;
}
diff --git a/plugins/woocommerce/tests/php/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGeneratorTest.php b/plugins/woocommerce/tests/php/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGeneratorTest.php
index 728fc658505..b6d6fc75c9a 100644
--- a/plugins/woocommerce/tests/php/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGeneratorTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/ProductFeed/Integrations/POSCatalog/AsyncGeneratorTest.php
@@ -60,6 +60,9 @@ class AsyncGeneratorTest extends \WC_Unit_Test_Case {
parent::tearDown();
delete_option( self::OPTION_KEY );
+ // Always clear the timeout filter here so a failed assertion in a test that registers it cannot
+ // leak it into later tests in the same process (tearDown runs even when a test fails).
+ remove_all_filters( 'woocommerce_product_feed_in_progress_timeout' );
$this->test_container->reset_all_replacements();
}
@@ -136,4 +139,94 @@ class AsyncGeneratorTest extends \WC_Unit_Test_Case {
$this->assertTrue( $method->invoke( $this->sut, $status ) );
}
+
+ /**
+ * Test that validate_status treats an in-progress job with a stale heartbeat as invalid.
+ *
+ * This is the recovery path for jobs whose process was killed (server timeout or out of
+ * memory) before they could mark themselves as failed.
+ */
+ public function test_validate_status_returns_false_for_stale_in_progress_feed() {
+ $status = array(
+ 'state' => AsyncGenerator::STATE_IN_PROGRESS,
+ 'scheduled_at' => time() - HOUR_IN_SECONDS,
+ 'updated_at' => time() - HOUR_IN_SECONDS,
+ );
+
+ $method = ( new ReflectionClass( $this->sut ) )->getMethod( 'validate_status' );
+ $method->setAccessible( true );
+
+ $this->assertFalse( $method->invoke( $this->sut, $status ) );
+ }
+
+ /**
+ * Test that validate_status keeps an in-progress job with a fresh heartbeat valid.
+ */
+ public function test_validate_status_returns_true_for_active_in_progress_feed() {
+ $status = array(
+ 'state' => AsyncGenerator::STATE_IN_PROGRESS,
+ 'scheduled_at' => time() - HOUR_IN_SECONDS,
+ 'updated_at' => time(),
+ );
+
+ $method = ( new ReflectionClass( $this->sut ) )->getMethod( 'validate_status' );
+ $method->setAccessible( true );
+
+ $this->assertTrue( $method->invoke( $this->sut, $status ) );
+ }
+
+ /**
+ * Test that validate_status treats an in-progress job missing a heartbeat as invalid
+ * when it was scheduled long ago (e.g. a job stuck before the heartbeat was introduced).
+ */
+ public function test_validate_status_returns_false_for_in_progress_feed_without_heartbeat() {
+ $status = array(
+ 'state' => AsyncGenerator::STATE_IN_PROGRESS,
+ 'scheduled_at' => time() - HOUR_IN_SECONDS,
+ );
+
+ $method = ( new ReflectionClass( $this->sut ) )->getMethod( 'validate_status' );
+ $method->setAccessible( true );
+
+ $this->assertFalse( $method->invoke( $this->sut, $status ) );
+ }
+
+ /**
+ * Test that the heartbeat timeout for in-progress jobs is filterable.
+ */
+ public function test_validate_status_in_progress_timeout_is_filterable() {
+ $status = array(
+ 'state' => AsyncGenerator::STATE_IN_PROGRESS,
+ 'scheduled_at' => time() - 30,
+ 'updated_at' => time() - 30,
+ );
+
+ $method = ( new ReflectionClass( $this->sut ) )->getMethod( 'validate_status' );
+ $method->setAccessible( true );
+
+ // With a 10 second timeout, a 30 second old heartbeat is considered stale. The filter is cleaned
+ // up in tearDown() so it cannot leak into later tests if the assertion fails.
+ add_filter( 'woocommerce_product_feed_in_progress_timeout', fn() => 10 );
+ $this->assertFalse( $method->invoke( $this->sut, $status ) );
+ }
+
+ /**
+ * Test that feed generation records a heartbeat in the resulting status.
+ */
+ public function test_feed_generation_action_records_heartbeat() {
+ // Make sure at least one product is present so a batch is processed.
+ WC_Helper_Product::create_simple_product();
+
+ update_option( self::OPTION_KEY, array( 'state' => AsyncGenerator::STATE_SCHEDULED ) );
+
+ $mock_mapper = $this->createMock( ProductMapper::class );
+ $mock_mapper->method( 'map_product' )->willReturn( array() );
+ $this->mock_integration->method( 'get_product_mapper' )->willReturn( $mock_mapper );
+
+ $this->sut->feed_generation_action( self::OPTION_KEY );
+
+ $updated_status = get_option( self::OPTION_KEY );
+ $this->assertSame( AsyncGenerator::STATE_COMPLETED, $updated_status['state'] );
+ $this->assertArrayHasKey( 'updated_at', $updated_status );
+ }
}